doctor_reviews/ZocDoc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "de99db84",
   "metadata": {},
   "source": [
    "# Use Splash to Request ZocDoc Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89593cca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:45.063787Z",
     "start_time": "2021-09-02T02:14:42.905564Z"
    }
   },
   "outputs": [],
   "source": [
    "from scrapy.http import TextResponse\n",
    "import requests\n",
    "from urllib.parse import urlencode\n",
    "\n",
    "\n",
    "# you need start the docker\n",
    "# url = 'https://www.zocdoc.com/doctor/ghulam-abbas-md-291989'\n",
    "url = 'https://www.zocdoc.com/doctor/dane-salazar-md-246564'\n",
    "\n",
    "# r = requests.get('http://localhost:8050/render.html', params={'url': url, 'wait':0})\n",
    "proxyapi = '68cd99bcfa3c0879b87ea3052f61da4e'\n",
    "r = requests.get('http://api.scraperapi.com/', params=urlencode({'api_key': proxyapi, 'url': url}))\n",
    "\n",
    "response = TextResponse(r.url, body = r.text, encoding = 'utf-8')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3a4dc57",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:45.113311Z",
     "start_time": "2021-09-02T02:14:45.068986Z"
    }
   },
   "outputs": [],
   "source": [
    "response.text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce63a50f",
   "metadata": {},
   "source": [
    "# Json Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f872860",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:46.247415Z",
     "start_time": "2021-09-02T02:14:46.206480Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from pprint import pprint\n",
    "xpath = './/script//text()'\n",
    "\n",
    "json_data = response.xpath(xpath).extract()\n",
    "len(json_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfdc1b8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:48.835928Z",
     "start_time": "2021-09-02T02:14:46.567325Z"
    }
   },
   "outputs": [],
   "source": [
    "json_string = [i for i in json_data if 'window.__REDUX_STATE__ = JSON.parse(' in i][0]\n",
    "json_info = [i for i in json_string.split(';\\n') if 'window.__REDUX_STATE__ = JSON.parse(' in i][0]\n",
    "\n",
    "json_info = json_info.split('JSON.parse(')[1][:-1] # remove prefix\n",
    "# x = x[:-1] # remove suffix\n",
    "# json_info\n",
    "\n",
    "\n",
    "\n",
    "x = json_info\n",
    "x = json.loads(x)\n",
    "x = json.loads(x)\n",
    "pprint(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a79fa329",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:48.841696Z",
     "start_time": "2021-09-02T02:14:48.839132Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_info = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ec7b708",
   "metadata": {},
   "source": [
    "## Root Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60cef6d9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:48.849769Z",
     "start_time": "2021-09-02T02:14:48.843804Z"
    }
   },
   "outputs": [],
   "source": [
    "[# 'routing',\n",
    " # 'theme',\n",
    " # 'absystem',\n",
    " # '@location',\n",
    " # 'pageload',\n",
    " # '@@viewport',\n",
    " # 'prefetch',\n",
    " # 'user',\n",
    "#  'signInForm',\n",
    " # '@@profile/in_network_insurance',\n",
    " # 'insurance-picker',\n",
    " # 'insurance',\n",
    " # '@@locationsCarousel',\n",
    " # 'NewExistingRadio',\n",
    " # '@@pictureGallery',\n",
    " 'profile',\n",
    "#  'seo',\n",
    " # 'patientPoweredSearch',\n",
    " # 'search',\n",
    " # '@@quickLinks',\n",
    " # 'timesgrid',\n",
    " # '@@videoVisitExplainerModal',\n",
    " # '@@profile/availability',\n",
    " # '@@mobileProfileBookingModal',\n",
    " # 'cookies'\n",
    "]\n",
    "\n",
    "\n",
    "# doc_info['timesgrid'] = x['timesgrid']\n",
    "doc_info"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02a81b16",
   "metadata": {},
   "source": [
    "## Profile Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1212cb5b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:48.859345Z",
     "start_time": "2021-09-02T02:14:48.854361Z"
    }
   },
   "outputs": [],
   "source": [
    "col = 'profile'\n",
    "\n",
    "profile = x[col]\n",
    "[i for i in profile]\n",
    "# x[col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e8aa5d2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:48.866435Z",
     "start_time": "2021-09-02T02:14:48.861486Z"
    }
   },
   "outputs": [],
   "source": [
    "cols = [# 'bookingSpotlightIsOn',\n",
    " # 'carrierId',\n",
    " # 'expandedLocations',\n",
    " # 'initParameters',\n",
    " # 'mapIsActive',\n",
    " # 'planId',\n",
    " # 'data',    # 'data',\n",
    " # 'dataVariables',\n",
    " 'providerId',\n",
    " # 'deviceType',\n",
    "#  'provider', # 'provider'\n",
    " # 'isNewPatient ',\n",
    " 'isMentalHealthProvider',\n",
    " 'doesProviderHaveInNetworkInsurances',\n",
    " # 'selectedLocationId',\n",
    "#  'procedureId',\n",
    " 'commonVirtualVisitReasons',\n",
    " # 'videoVisitExplainerModalType',\n",
    " 'customDimensions',\n",
    " # 'defaultProcedureId',\n",
    "#  'eventData',\n",
    " # 'host',\n",
    " # 'pageId',\n",
    " # 'pageCategory',\n",
    " # 'prefix',\n",
    " # 'seo',\n",
    "#  'trustedInsuranceRating',\n",
    " 'brandingVanityUrl']\n",
    "\n",
    "\n",
    "for col in cols: doc_info[col] = profile[col]\n",
    "    \n",
    "pprint(doc_info)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c5aa2a1",
   "metadata": {},
   "source": [
    "### Profile Info - DataProvider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e5e26c8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:48.871661Z",
     "start_time": "2021-09-02T02:14:48.868827Z"
    }
   },
   "outputs": [],
   "source": [
    "profile_data_provider = profile['data']['provider']\n",
    "# [i for i in profile_data_provider]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd38e537",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:50.151718Z",
     "start_time": "2021-09-02T02:14:48.873840Z"
    }
   },
   "outputs": [],
   "source": [
    "cols = ['isOnlyInNetworkBookable','acceptsNewPatients','errorCode','id','monolithId','status',\n",
    " 'profileUrl','isApproved','prenominal','postnominal','firstName',\n",
    " 'lastName','directoryIds', 'approvedFullName', 'nameInSentence','newPatientAppointmentsAsIntroCalls',\n",
    " \n",
    " 'genderIdentity', 'frontEndCirclePictureUrl', 'frontEndSquarePictureUrl', 'showPhotoWithCircleCss', 'smallPictures',\n",
    " 'averageRating', 'averageBedsideRating','averageWaitTimeRating',\n",
    " 'isPreview','personIdentifierSource', 'dataProvidedBy', 'dataProvidedByUrl',\n",
    " \n",
    " 'mayChargeAdditionalCopay','onlySeesChildren', 'optOutOfRatings', 'previewSummary',\n",
    " 'highlyRecommendPercentage', 'lowWaitTimedPercentage', 'hasNewPatientAvailability',\n",
    " 'isTopRebookingProvider', 'hasVirtualLocations',\n",
    " 'selfPaySettings', 'sellingPoints', 'approvedNonVirtualLocations', 'practices',\n",
    " 'providerLocations', 'relevantSpecialty', 'mainSpecialtyId','specialties','approvedLocations',\n",
    " \n",
    "\n",
    " 'statement', 'defaultProcedureId', 'hospitalAffiliations', 'certifications', 'education',\n",
    " 'memberships', 'typedMemberships','awards','badges', 'languages','npi',\n",
    " 'inNetworkInsurance','procedures','popularProcedures','isBookable','seo','reviewCount',\n",
    " 'reviews', 'properties', 'trustedInsuranceRating', 'providerBadges','locations',\n",
    " 'sortedLocations', 'numVirtualLocations',\n",
    " #  '__typename',\n",
    " # 'recentReviews',\n",
    "]\n",
    "\n",
    "\n",
    "# l = profile_data_provider['isOnlyInNetworkBookable']\n",
    "# pprint(l)\n",
    "\n",
    "\n",
    "\n",
    "for col in cols: doc_info[col] = profile_data_provider[col]\n",
    "    \n",
    "    \n",
    "pprint(doc_info)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d53d63b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T02:14:50.164502Z",
     "start_time": "2021-09-02T02:14:50.159219Z"
    }
   },
   "outputs": [],
   "source": [
    "print([i for i in doc_info])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da065247",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:26:39.872216Z",
     "start_time": "2021-08-24T23:26:39.868883Z"
    }
   },
   "outputs": [],
   "source": [
    "len(doc_info['reviews'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d87e68fd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:26:42.068301Z",
     "start_time": "2021-08-24T23:26:42.064351Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_info['complete_reviews'] = False if doc_info['reviewCount'] > len(doc_info['reviews']) else True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "448eeeb3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:27:04.854111Z",
     "start_time": "2021-08-24T23:27:04.825312Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_info['reviews']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1a37e11",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T01:20:15.096469Z",
     "start_time": "2021-08-24T01:20:15.088634Z"
    }
   },
   "source": [
    "# POST: URL (Failed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd55bf2a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T21:36:53.398374Z",
     "start_time": "2021-08-24T21:36:53.393991Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# import requests\n",
    "\n",
    "\n",
    "# review_api_url = 'https://api.zocdoc.com/directory/v2/gql'\n",
    "# providerID = 'pr_mw0z_x5HqE-YgjiprYcWax' # TODO\n",
    "# limit = 100 # TODO\n",
    "\n",
    "       \n",
    "# forms = {\n",
    "#     'operationName': 'getProviderReviews',\n",
    "#     'query': \"query getProviderReviews($providerId: String, $directoryId: String, $limit: Int, $sortReviewsBy: ReviewsSortBy, $sortReviewsDescending: Boolean) {\\n  provider(id: $providerId, directoryId: $directoryId) {\\n    id\\n    ...providerReviews\\n    __typename\\n  }\\n}\\n\\nfragment providerReviews on Provider {\\n  reviews(limit: $limit, sortBy: $sortReviewsBy, sortDescending: $sortReviewsDescending) {\\n    reviewId\\n    bedsideRating\\n    comment\\n    date\\n    dateRange\\n    helpfulCount\\n    isMailedInReview\\n    isPartnerReview\\n    isVirtualVisit\\n    overallRating\\n    patientName\\n    waitTimeRating\\n    __typename\\n  }\\n  __typename\\n}\\n\", \n",
    "#     'variables':{\n",
    "#         'directoryId': \"-1\",\n",
    "#         \"limit\": str(limit),\n",
    "#         \"providerId\": providerID,\n",
    "#         \"sortReviewsBy\":\"date\",\n",
    "#         \"sortReviewsDescending\": 'true',\n",
    "#     }\n",
    "# }\n",
    "\n",
    "\n",
    "# r = requests.post('http://localhost:8050/render.html',\n",
    "#               json={'url': review_api_url,\n",
    "#                     'http_method': 'POST',\n",
    "#                     'body': forms,\n",
    "#                     'wait': 2})\n",
    "\n",
    "# print(r.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fbd6a65",
   "metadata": {},
   "source": [
    "# Selenium (when reviews > 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb10669e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:39:45.379647Z",
     "start_time": "2021-08-24T23:39:31.478175Z"
    }
   },
   "outputs": [],
   "source": [
    "# select your webdriver\n",
    "from scrapy.http import HtmlResponse\n",
    "from selenium import webdriver\n",
    "import time\n",
    "\n",
    "REVIEW_PER_PAGE = 50\n",
    "webdriver_path = '_source/chromedriver_macm1'\n",
    "reviewCount = doc_info['reviewCount']\n",
    "    \n",
    "    \n",
    "    \n",
    "def get_review(review_selector):\n",
    "    d = {}\n",
    "    d['comment'] = review_selector.xpath('.//*[@itemprop=\"reviewBody\"]//span/text()').extract_first()\n",
    "    d['date'] = review_selector.xpath('.//*[@itemprop=\"datePublished\"]/text()').extract_first()\n",
    "    d['patientName'] = review_selector.xpath('.//*[@itemprop=\"author\"]/text()').extract_first()\n",
    "    d['patient_type'] = review_selector.xpath('.//*[@data-test=\"review-source\"]/text()').extract_first()\n",
    "    d['overallRating'] = review_selector.xpath('.//*[@data-test=\"stars-svg-wrapper\"]/svg/@data-rating').extract_first()\n",
    "    return d\n",
    "\n",
    "\n",
    "def get_complete_reviews_from_selenium(url, reviewCount, REVIEW_PER_PAGE, webdriver_path):\n",
    "    options = webdriver.ChromeOptions()\n",
    "    # options.add_argument(\"start-maximized\")\n",
    "    options.add_argument(\"disable-infobars\")\n",
    "    options.add_argument(\"--disable-extensions\")\n",
    "    options.add_argument('--ignore-certificate-errors')\n",
    "    options.add_argument('--incognito')\n",
    "    # options.add_argument('--headless')\n",
    "\n",
    "    driver = webdriver.Chrome(webdriver_path, options=options)\n",
    "    driver.get(url)\n",
    "    driver.title\n",
    "    time.sleep(2)\n",
    "    \n",
    "    xpath = './/button[@data-test=\"reviews-read-more-button\"]'\n",
    "    \n",
    "    for _ in range(int(reviewCount / REVIEW_PER_PAGE )):\n",
    "        more_result_button = driver.find_element_by_xpath(xpath)\n",
    "        print(more_result_button)\n",
    "        driver.execute_script(\"arguments[0].click();\", more_result_button)\n",
    "        time.sleep(1)\n",
    "        \n",
    "    response = HtmlResponse(driver.current_url, body=driver.page_source, encoding='utf-8')\n",
    "    reviews = response.xpath('//div[@itemprop=\"reviews\"]')\n",
    "    reviews = [get_review(i) for i in reviews]\n",
    "    # len(reviews)\n",
    "    driver.quit()\n",
    "    return reviews\n",
    "\n",
    "\n",
    "\n",
    "reviews = get_complete_reviews_from_selenium(url, reviewCount, REVIEW_PER_PAGE, webdriver_path)\n",
    "\n",
    "print('\\nReviews: reported {} vs collected {}'.format(reviewCount, len(reviews)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d30c9d1c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T21:44:43.682786Z",
     "start_time": "2021-08-24T21:44:32.964704Z"
    }
   },
   "source": [
    "\n",
    "# All In One"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc1d0d47",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:40:58.934721Z",
     "start_time": "2021-08-24T23:40:58.906889Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "import pandas as pd\n",
    "import time\n",
    "import requests\n",
    "from scrapy.http import TextResponse\n",
    "import json\n",
    "import html\n",
    "import argparse\n",
    "import random\n",
    "from datetime import datetime\n",
    "from scrapy.http import HtmlResponse\n",
    "from selenium import webdriver\n",
    "import time\n",
    "from datetime import datetime\n",
    "\n",
    "\n",
    "\n",
    "headers = {\n",
    "    \"Connection\": \"keep-alive\",\n",
    "    \"Cache-Control\": \"max-age=0\",\n",
    "    \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\",\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36\",\n",
    "}\n",
    "        \n",
    "def get_review(review_selector):\n",
    "    d = {}\n",
    "    d['comment'] = review_selector.xpath('.//*[@itemprop=\"reviewBody\"]//span/text()').extract_first()\n",
    "    d['date'] = review_selector.xpath('.//*[@itemprop=\"datePublished\"]/text()').extract_first()\n",
    "    d['patientName'] = review_selector.xpath('.//*[@itemprop=\"author\"]/text()').extract_first()\n",
    "    d['patient_type'] = review_selector.xpath('.//*[@data-test=\"review-source\"]/text()').extract_first()\n",
    "    d['overallRating'] = review_selector.xpath('.//*[@data-test=\"stars-svg-wrapper\"]/svg/@data-rating').extract_first()\n",
    "    return d\n",
    "\n",
    "def get_complete_reviews_from_selenium(url, reviewCount, REVIEW_PER_PAGE, webdriver_path):\n",
    "    options = webdriver.ChromeOptions()\n",
    "    # options.add_argument(\"start-maximized\")\n",
    "    options.add_argument(\"disable-infobars\")\n",
    "    options.add_argument(\"--disable-extensions\")\n",
    "    options.add_argument('--ignore-certificate-errors')\n",
    "    options.add_argument('--incognito')\n",
    "    # options.add_argument('--headless')\n",
    "\n",
    "    driver = webdriver.Chrome(webdriver_path, options=options)\n",
    "    driver.get(url)\n",
    "    driver.title\n",
    "    time.sleep(2)\n",
    "    \n",
    "    xpath = './/button[@data-test=\"reviews-read-more-button\"]'\n",
    "    \n",
    "    for _ in range(int(reviewCount / REVIEW_PER_PAGE )):\n",
    "        more_result_button = driver.find_element_by_xpath(xpath)\n",
    "        print(more_result_button)\n",
    "        driver.execute_script(\"arguments[0].click();\", more_result_button)\n",
    "        time.sleep(1)\n",
    "        \n",
    "    response = HtmlResponse(driver.current_url, body=driver.page_source, encoding='utf-8')\n",
    "    reviews = response.xpath('//div[@itemprop=\"reviews\"]')\n",
    "    reviews = [get_review(i) for i in reviews]\n",
    "    # len(reviews)\n",
    "    driver.quit()\n",
    "    return reviews\n",
    "\n",
    "\n",
    "def get_physician_info_from_zocdoc_url(ph_url, REVIEW_PER_PAGE, webdriver_path, use_webdriver):\n",
    "    doc_info = {}\n",
    "    \n",
    "    r = requests.get('http://localhost:8050/render.html', params={'url': ph_url, 'wait':0})\n",
    "    response = TextResponse(r.url, body = r.text, encoding = 'utf-8')\n",
    "    \n",
    "    xpath = './/script//text()'\n",
    "\n",
    "    json_data = response.xpath(xpath).extract()\n",
    "    # len(json_data)\n",
    "\n",
    "    json_string = [i for i in json_data if 'window.__REDUX_STATE__ = JSON.parse(' in i][0]\n",
    "    json_info = [i for i in json_string.split(';\\n') if 'window.__REDUX_STATE__ = JSON.parse(' in i][0]\n",
    "\n",
    "    json_info = json_info.split('JSON.parse(')[1][:-1] # remove prefix\n",
    "    # x = x[:-1] # remove suffix\n",
    "    # json_info\n",
    "\n",
    "    x = json_info\n",
    "    x = json.loads(x)\n",
    "    x = json.loads(x) # yes, double json.load operations\n",
    "\n",
    "\n",
    "    # (1) basic profile\n",
    "    col = 'profile'; profile = x[col]\n",
    "\n",
    "    cols = [ 'providerId', 'isMentalHealthProvider',\n",
    "        'doesProviderHaveInNetworkInsurances', \n",
    "        'commonVirtualVisitReasons', 'customDimensions', 'brandingVanityUrl']\n",
    "\n",
    "    for col in cols: doc_info[col] = profile[col]\n",
    "\n",
    "    # (2) provider\n",
    "    profile_data_provider = profile['data']['provider']\n",
    "\n",
    "    cols = ['isOnlyInNetworkBookable','acceptsNewPatients','errorCode','id','monolithId','status',\n",
    "        'profileUrl','isApproved','prenominal','postnominal','firstName',\n",
    "        'lastName','directoryIds', 'approvedFullName', 'nameInSentence','newPatientAppointmentsAsIntroCalls',\n",
    "        'genderIdentity', 'frontEndCirclePictureUrl', 'frontEndSquarePictureUrl', 'showPhotoWithCircleCss', 'smallPictures',\n",
    "        'averageRating', 'averageBedsideRating','averageWaitTimeRating',\n",
    "        'isPreview','personIdentifierSource', 'dataProvidedBy', 'dataProvidedByUrl',\n",
    "        'mayChargeAdditionalCopay','onlySeesChildren', 'optOutOfRatings', 'previewSummary',\n",
    "        'highlyRecommendPercentage', 'lowWaitTimedPercentage', 'hasNewPatientAvailability',\n",
    "        'isTopRebookingProvider', 'hasVirtualLocations',\n",
    "        'selfPaySettings', 'sellingPoints', 'approvedNonVirtualLocations', 'practices',\n",
    "        'providerLocations', 'relevantSpecialty', 'mainSpecialtyId','specialties','approvedLocations',\n",
    "        'statement', 'defaultProcedureId', 'hospitalAffiliations', 'certifications', 'education',\n",
    "        'memberships', 'typedMemberships','awards','badges', 'languages','npi',\n",
    "        'inNetworkInsurance','procedures','popularProcedures','isBookable','seo','reviewCount',\n",
    "        'reviews', 'properties', 'trustedInsuranceRating', 'providerBadges','locations',\n",
    "        'sortedLocations', 'numVirtualLocations',]\n",
    "\n",
    "    for col in cols: doc_info[col] = profile_data_provider[col]\n",
    "\n",
    "    doc_info['complete_reviews'] = False if doc_info['reviewCount'] > len(doc_info['reviews']) else True\n",
    "\n",
    "\n",
    "    reviewCount = doc_info['reviewCount']\n",
    "\n",
    "    if doc_info['complete_reviews'] == False and use_webdriver == True:\n",
    "        reviews = get_complete_reviews_from_selenium(ph_url, reviewCount, \n",
    "                                                    REVIEW_PER_PAGE, webdriver_path)\n",
    "\n",
    "        print('\\nReviews: reported {} vs collected {}'.format(reviewCount, len(reviews)))\n",
    "        if len(reviews) == reviewCount: doc_info['complete_reviews'] = True \n",
    "        \n",
    "        doc_info['reviews'] = reviews\n",
    "    \n",
    "    return doc_info\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1174a397",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:41:14.908688Z",
     "start_time": "2021-08-24T23:40:59.634567Z"
    }
   },
   "outputs": [],
   "source": [
    "ph_url = url\n",
    "REVIEW_PER_PAGE = 50\n",
    "webdriver_path = '_source/chromedriver_macm1'\n",
    "\n",
    "doc_info = get_physician_info_from_zocdoc_url(ph_url, REVIEW_PER_PAGE, webdriver_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c7dcf1b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:41:14.913726Z",
     "start_time": "2021-08-24T23:41:14.910488Z"
    }
   },
   "outputs": [],
   "source": [
    "len(doc_info['reviews'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9fa79f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-24T23:41:14.924238Z",
     "start_time": "2021-08-24T23:41:14.914793Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_info['reviews']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}