Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Drop parfive less than 2 #6942

Merged
merged 4 commits into from
Apr 21, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/6742.feature.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ The minimum required versions of several core dependencies have been updated:
- Python 3.9
- astropy 5.0.1
- numpy 1.21.0
- parfive 2.0.0
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should go in a new changelog entry, so it's linked with this PR and not the one corresponding to this file.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed.


The minimum required versions of these optional dependencies has also been updated:

Expand Down
1 change: 1 addition & 0 deletions docs/whatsnew/5.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ The minimum required versions of several core dependencies have been updated:
- Python 3.9
- astropy 5.0.1
- numpy 1.21.0
- parfive 2.0
nabobalis marked this conversation as resolved.
Show resolved Hide resolved

The minimum required versions of these optional dependencies has also been updated:

Expand Down
2 changes: 1 addition & 1 deletion setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ install_requires =
astropy>=5.0.1
numpy>=1.21.0
packaging>=19.0
parfive[ftp]>=1.2.0
parfive[ftp]>=2.0.0

[options.extras_require]
# The list of available extras is also in the installation docs, if you add or remove one please edit it there as well.
Expand Down
10 changes: 10 additions & 0 deletions sunpy/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,16 @@ def undo_config_dir_patch():
os.environ["SUNPY_CONFIGDIR"] = oridir


@pytest.fixture(scope='session', autouse=True)
def parfive_header_test(request):
"""
Add a keyword to tell parfive this is a test run.
"""
os.environ["PARFIVE_SUNPY_TESTS"] = "True"
yield
del os.environ["PARFIVE_SUNPY_TESTS"]
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was not sure how to do this, so I did this as it was quick.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems legit to me 👍

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this seems good, but why don't we make this a generic environment var for being in our test suite? We might have other non-parfive uses for it in the future?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can do that.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was not sure what to call it, so I made one up.



@pytest.fixture(scope='session', autouse=True)
def hide_parfive_progress(request):
"""
Expand Down
10 changes: 1 addition & 9 deletions sunpy/data/data_manager/tests/test_downloader.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
from unittest.mock import patch

import parfive
import pytest
from packaging.version import Version
from parfive import Results
from parfive.results import Error

from sunpy.data.data_manager.downloader import DownloaderError, ParfiveDownloader

Expand All @@ -12,15 +11,8 @@ def test_ParfiveDownloader_errors():
"""
Test that ParfiveDownloader raises an error when the download fails.
"""

downloader = ParfiveDownloader()
results = Results()

# TODO: Remove this when we support parfive 2.0.
if Version(parfive.__version__) >= Version("2.0.0"):
from parfive.results import Error
else:
Error = results._error
results.errors.append(Error("", "FAKE_URL", ValueError("TEST_ERROR")))
with patch('parfive.Downloader.download') as download:
download.return_value = results
Expand Down
5 changes: 1 addition & 4 deletions sunpy/net/fido_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,10 +398,7 @@ def fetch(self, *query_results, path=None, max_conn=5, progress=True,
if is_jsoc_only:
max_conn = 1
max_splits = 1
if parfive_version >= Version("2.0.0"):
downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
else:
downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite)
downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
elif not isinstance(downloader, parfive.Downloader):
raise TypeError("The downloader argument must be a parfive.Downloader instance.")

Expand Down
6 changes: 1 addition & 5 deletions sunpy/net/jsoc/jsoc.py
Original file line number Diff line number Diff line change
Expand Up @@ -534,11 +534,7 @@ def get_request(self, requests, path=None, overwrite=False, progress=True,
# Private communication from JSOC say we should not use more than one connection.
if max_conn != self.default_max_conn:
log.info(f"Setting max parallel downloads to 1 for the JSOC client.")
if parfive_version >= Version("2.0.0"):
downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
else:
downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite)

downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
urls = []
for request in requests:
if request.status == 0:
Expand Down
42 changes: 11 additions & 31 deletions sunpy/util/parfive_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import aiohttp
import parfive
from packaging.version import Version
from parfive import Results
from parfive import Results, SessionConfig

import sunpy

Expand All @@ -14,36 +14,16 @@
parfive_version = Version(parfive.__version__)
sunpy_headers = {
"User-Agent": f"sunpy/{sunpy.__version__} parfive/{parfive.__version__} "
f"aiohttp/{aiohttp.__version__} python/{sys.version[:5]}"
f"aiohttp/{aiohttp.__version__} python/{sys.version[:5]} "
}
if os.environ.get("PARFIVE_SUNPY_TESTS"):
dstansby marked this conversation as resolved.
Show resolved Hide resolved
sunpy_headers["User-Agent"] = f"sunpy-CI-tests/{sunpy.__version__} parfive/{parfive.__version__} "
config = SessionConfig(headers=sunpy_headers)


if parfive_version < Version("2.0.0"):
# Overload the parfive downloader class to set the User-Agent string
class Downloader(parfive.Downloader):
@wraps(parfive.Downloader.__init__)
def __init__(self, *args, **kwargs):
headers = kwargs.pop("headers", {})
kwargs["headers"] = {**sunpy_headers, **headers}
# Works with conftest to hide the progress bar.
progress = os.environ.get("PARFIVE_HIDE_PROGESS", None)
if progress:
kwargs["progress"] = False
super().__init__(*args, **kwargs)

else:
from parfive import SessionConfig

config = SessionConfig(headers=sunpy_headers)

class Downloader(parfive.Downloader):
@wraps(parfive.Downloader.__init__)
def __init__(self, *args, **kwargs):
if "config" not in kwargs:
kwargs["config"] = config

# This is to make our sample data download code work on 1.x and 2.x
# when we depend on 2+ we should remove this.
headers = kwargs.pop("headers", {})
kwargs["config"].headers = {**kwargs["config"].headers, **headers}
super().__init__(*args, **kwargs)
class Downloader(parfive.Downloader):
@wraps(parfive.Downloader.__init__)
def __init__(self, *args, **kwargs):
if "config" not in kwargs:
kwargs["config"] = config
super().__init__(*args, **kwargs)