Skip to content

Commit

Permalink
Refactor how session and crumb are obtained
Browse files Browse the repository at this point in the history
  • Loading branch information
@dpguthrie
dpguthrie committed Dec 6, 2023
1 parent 6fb1151 commit 0dac661
Show file tree
Hide file tree
Showing 9 changed files with 2,025 additions and 1,973 deletions.
129 changes: 84 additions & 45 deletions tests/test_ticker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ def test_adj_ohlc(ticker):
assert ticker.history(period="max", adj_ohlc=True) is not None


class TestHistoryDataframe():
class TestHistoryDataframe:
"""Tests for `utils.__init__._history_dataframe` and dependencies."""

@pytest.fixture
Expand Down Expand Up @@ -250,13 +250,20 @@ def timestamps_daily(self, utc, tz_oz, tz_us, tz_hk):

expected_utc = pd.DatetimeIndex(
[
'2022-11-04 13:30:00', '2022-11-07 14:30:00',
'2022-11-08 14:30:00', '2022-11-09 14:30:00',
'2022-11-10 14:30:00', '2022-11-11 14:30:00',
'2022-11-14 14:30:00', '2022-11-15 14:30:00',
'2022-11-16 14:30:00', '2022-11-17 14:30:00',
'2022-11-18 14:30:00', '2022-11-21 14:30:00',
'2022-11-22 14:30:00', '2022-11-23 14:30:00',
'2022-11-04 13:30:00',
'2022-11-07 14:30:00',
'2022-11-08 14:30:00',
'2022-11-09 14:30:00',
'2022-11-10 14:30:00',
'2022-11-11 14:30:00',
'2022-11-14 14:30:00',
'2022-11-15 14:30:00',
'2022-11-16 14:30:00',
'2022-11-17 14:30:00',
'2022-11-18 14:30:00',
'2022-11-21 14:30:00',
'2022-11-22 14:30:00',
'2022-11-23 14:30:00',
],
tz=utc,
)
Expand Down Expand Up @@ -284,13 +291,20 @@ def timestamps_daily(self, utc, tz_oz, tz_us, tz_hk):

expected_oz = pd.DatetimeIndex(
[
'2022-11-05 00:30:00', '2022-11-08 01:30:00',
'2022-11-09 01:30:00', '2022-11-10 01:30:00',
'2022-11-11 01:30:00', '2022-11-12 01:30:00',
'2022-11-15 01:30:00', '2022-11-16 01:30:00',
'2022-11-17 01:30:00', '2022-11-18 01:30:00',
'2022-11-19 01:30:00', '2022-11-22 01:30:00',
'2022-11-23 01:30:00', '2022-11-24 01:30:00',
'2022-11-05 00:30:00',
'2022-11-08 01:30:00',
'2022-11-09 01:30:00',
'2022-11-10 01:30:00',
'2022-11-11 01:30:00',
'2022-11-12 01:30:00',
'2022-11-15 01:30:00',
'2022-11-16 01:30:00',
'2022-11-17 01:30:00',
'2022-11-18 01:30:00',
'2022-11-19 01:30:00',
'2022-11-22 01:30:00',
'2022-11-23 01:30:00',
'2022-11-24 01:30:00',
],
tz=tz_oz,
)
Expand Down Expand Up @@ -318,13 +332,20 @@ def timestamps_daily(self, utc, tz_oz, tz_us, tz_hk):

expected_us = pd.DatetimeIndex(
[
'2022-11-04 09:30:00', '2022-11-07 09:30:00',
'2022-11-08 09:30:00', '2022-11-09 09:30:00',
'2022-11-10 09:30:00', '2022-11-11 09:30:00',
'2022-11-14 09:30:00', '2022-11-15 09:30:00',
'2022-11-16 09:30:00', '2022-11-17 09:30:00',
'2022-11-18 09:30:00', '2022-11-21 09:30:00',
'2022-11-22 09:30:00', '2022-11-23 09:30:00',
'2022-11-04 09:30:00',
'2022-11-07 09:30:00',
'2022-11-08 09:30:00',
'2022-11-09 09:30:00',
'2022-11-10 09:30:00',
'2022-11-11 09:30:00',
'2022-11-14 09:30:00',
'2022-11-15 09:30:00',
'2022-11-16 09:30:00',
'2022-11-17 09:30:00',
'2022-11-18 09:30:00',
'2022-11-21 09:30:00',
'2022-11-22 09:30:00',
'2022-11-23 09:30:00',
],
tz=tz_us,
)
Expand Down Expand Up @@ -352,13 +373,20 @@ def timestamps_daily(self, utc, tz_oz, tz_us, tz_hk):

expected_hk = pd.DatetimeIndex(
[
'2022-11-04 21:30', '2022-11-07 22:30',
'2022-11-08 22:30', '2022-11-09 22:30',
'2022-11-10 22:30', '2022-11-11 22:30',
'2022-11-14 22:30', '2022-11-15 22:30',
'2022-11-16 22:30', '2022-11-17 22:30',
'2022-11-18 22:30', '2022-11-21 22:30',
'2022-11-22 22:30', '2022-11-23 22:30',
'2022-11-04 21:30',
'2022-11-07 22:30',
'2022-11-08 22:30',
'2022-11-09 22:30',
'2022-11-10 22:30',
'2022-11-11 22:30',
'2022-11-14 22:30',
'2022-11-15 22:30',
'2022-11-16 22:30',
'2022-11-17 22:30',
'2022-11-18 22:30',
'2022-11-21 22:30',
'2022-11-22 22:30',
'2022-11-23 22:30',
],
tz=tz_hk,
)
Expand All @@ -370,7 +398,11 @@ def timestamps_daily(self, utc, tz_oz, tz_us, tz_hk):
)

yield (
tss, expected_us_days, expected_oz_days, expected_hk_days, expected_utc_days
tss,
expected_us_days,
expected_oz_days,
expected_hk_days,
expected_utc_days,
)

@pytest.fixture
Expand Down Expand Up @@ -451,12 +483,7 @@ def get_splits(tss):
indice = 11
ts = tss[indice]
d = {
str(ts): {
'data': ts,
'numerator': 3,
'denominator': 1,
'splitRatio': '3:1'
}
str(ts): {'data': ts, 'numerator': 3, 'denominator': 1, 'splitRatio': '3:1'}
}
expected = [3 if i == indice else float('nan') for i in range(14)]
return d, expected
Expand Down Expand Up @@ -606,7 +633,10 @@ def f(data, adj_timezone):
dividends, _ = dividends_daily

expecteds = (
expected_daily_utc, expected_daily_us, expected_daily_oz, expected_daily_hk
expected_daily_utc,
expected_daily_us,
expected_daily_oz,
expected_daily_hk,
)
tzs = (utc, tz_us, tz_oz, tz_hk)
for expected, tz in zip(expecteds, tzs):
Expand Down Expand Up @@ -663,7 +693,9 @@ def test_live_indice(
# verify live indice has local timezone when adj_timezone True
rtrn = _history_dataframe(data, daily=True, adj_timezone=True)
expected_li = pd.Timestamp("2022-11-23 14:31", tz=tz_us).to_pydatetime()
expected_index = expected_index[:-1].insert(len(expected_index)-1, expected_li)
expected_index = expected_index[:-1].insert(
len(expected_index) - 1, expected_li
)
expected_df.index = expected_index
assert_frame_equal(rtrn, expected_df)

Expand Down Expand Up @@ -728,13 +760,20 @@ def timestamps_intraday(self, utc):

expected_index_utc = pd.DatetimeIndex(
[
'2022-11-04 13:30:00', '2022-11-04 14:30:00',
'2022-11-04 15:30:00', '2022-11-04 16:30:00',
'2022-11-04 17:30:00', '2022-11-04 18:30:00',
'2022-11-04 19:30:00', '2022-11-07 14:30:00',
'2022-11-07 15:30:00', '2022-11-07 16:30:00',
'2022-11-07 17:30:00', '2022-11-07 18:30:00',
'2022-11-07 19:30:00', '2022-11-07 20:30:00'
'2022-11-04 13:30:00',
'2022-11-04 14:30:00',
'2022-11-04 15:30:00',
'2022-11-04 16:30:00',
'2022-11-04 17:30:00',
'2022-11-04 18:30:00',
'2022-11-04 19:30:00',
'2022-11-07 14:30:00',
'2022-11-07 15:30:00',
'2022-11-07 16:30:00',
'2022-11-07 17:30:00',
'2022-11-07 18:30:00',
'2022-11-07 19:30:00',
'2022-11-07 20:30:00',
],
tz=utc,
)
Expand Down
11 changes: 8 additions & 3 deletions yahooquery/base.py
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,28 @@
# stdlib
import os
import time
from concurrent.futures import as_completed
from datetime import datetime

# third party
from requests_futures.sessions import FuturesSession

from tqdm import tqdm
from yahooquery.headless import _has_selenium, YahooFinanceHeadless

# first party
from yahooquery.headless import YahooFinanceHeadless, _has_selenium
from yahooquery.utils import (
convert_to_list,
initialize_session,
get_crumb,
initialize_session,
setup_session,
)
from yahooquery.utils.countries import COUNTRIES

try:
# stdlib
from urllib import parse
except ImportError:
# third party
import urlparse as parse


Expand Down
3 changes: 2 additions & 1 deletion yahooquery/headless.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
from requests.cookies import RequestsCookieJar

try:
# third party
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
Expand Down
5 changes: 3 additions & 2 deletions yahooquery/misc.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# stdlib
from datetime import datetime
from typing import List

# third party
import pandas as pd
import requests

from .utils.countries import COUNTRIES
from .utils import initialize_session

from .utils.countries import COUNTRIES

BASE_URL = "https://query2.finance.yahoo.com"

Expand Down
2 changes: 2 additions & 0 deletions yahooquery/research.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# stdlib
import json
from datetime import datetime, timedelta

# third party
import pandas as pd

from .base import _YahooFinance
Expand Down
3 changes: 3 additions & 0 deletions yahooquery/screener.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
# stdlib
import re

from .base import _YahooFinance
from .utils.screeners import SCREENERS

try:
# stdlib
from urllib import parse
except ImportError:
# third party
import urlparse as parse


Expand Down
7 changes: 5 additions & 2 deletions yahooquery/ticker.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
from datetime import datetime, timedelta
# stdlib
import re
from datetime import datetime, timedelta

# third party
import pandas as pd

# first party
from yahooquery.base import _YahooFinance
from yahooquery.utils import convert_to_timestamp, flatten_list, _history_dataframe
from yahooquery.utils import _history_dataframe, convert_to_timestamp, flatten_list


class Ticker(_YahooFinance):
Expand Down
6 changes: 2 additions & 4 deletions yahooquery/utils/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# stdlib
import datetime
import json
import logging
import random
import re
Expand All @@ -14,7 +13,6 @@
from requests_futures.sessions import FuturesSession
from urllib3.exceptions import MaxRetryError


logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -1387,12 +1385,12 @@ def setup_session(host: str):
return session.cookies, crumb


def get_crumb(session) -> str:
def get_crumb(session):
try:
response = session.get("https://query2.finance.yahoo.com/v1/test/getcrumb")
return response.text

except (ConnectionError, RetryError) as e:
except (ConnectionError, RetryError):
logger.critical(
"Failed to obtain crumb. Ability to retrieve data will be significantly "
"limited."
Expand Down
Loading

0 comments on commit 0dac661

Please sign in to comment.