From 3d8667b99aaf72a9d3daa21c9d20e66b97f7bc47 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 26 Jan 2022 13:45:49 +0000 Subject: [PATCH 01/13] fix tests and uprev test dependencies --- Makefile | 2 +- arq/connections.py | 5 ++- arq/worker.py | 12 +++++-- pyproject.toml | 57 ++++++++++++++++++++++++++++++++++ setup.cfg | 55 -------------------------------- tests/requirements-linting.txt | 15 +++++---- tests/requirements-testing.txt | 2 +- tests/test_cron.py | 3 +- tests/test_worker.py | 6 +++- 9 files changed, 87 insertions(+), 70 deletions(-) create mode 100644 pyproject.toml diff --git a/Makefile b/Makefile index 35fd24c5..f46bf9d5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := all isort = isort arq tests -black = black -S -l 120 --target-version py37 arq tests +black = black arq tests .PHONY: install install: diff --git a/arq/connections.py b/arq/connections.py index 770d86ba..59b6b47f 100644 --- a/arq/connections.py +++ b/arq/connections.py @@ -266,7 +266,10 @@ async def pool_factory(*args: Any, **kwargs: Any) -> Redis: async def log_redis_info(redis: Redis, log_func: Callable[[str], Any]) -> None: with await redis as r: info_server, info_memory, info_clients, key_count = await asyncio.gather( - r.info(section='Server'), r.info(section='Memory'), r.info(section='Clients'), r.dbsize(), + r.info(section='Server'), + r.info(section='Memory'), + r.info(section='Clients'), + r.dbsize(), ) redis_version = info_server.get('server', {}).get('redis_version', '?') diff --git a/arq/worker.py b/arq/worker.py index 4a32450e..00ddb0b3 100644 --- a/arq/worker.py +++ b/arq/worker.py @@ -600,7 +600,13 @@ async def job_failed(exc: BaseException) -> None: await asyncio.shield( self.finish_job( - job_id, finish, result_data, result_timeout_s, keep_result_forever, incr_score, keep_in_progress, + job_id, + finish, + result_data, + result_timeout_s, + keep_result_forever, + incr_score, + keep_in_progress, ) ) @@ -642,7 +648,9 @@ async def finish_failed_job(self, job_id: str, result_data: Optional[bytes]) -> await conn.unwatch() tr = conn.multi_exec() tr.delete( - retry_key_prefix + job_id, in_progress_key_prefix + job_id, job_key_prefix + job_id, + retry_key_prefix + job_id, + in_progress_key_prefix + job_id, + job_key_prefix + job_id, ) tr.zrem(abort_jobs_ss, job_id) tr.zrem(self.queue_name, job_id) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6db514f2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[tool.pytest.ini_options] +testpaths = 'tests' +filterwarnings = ['error', 'ignore::DeprecationWarning:aioredis'] + +[tool.coverage.run] +source = ['src'] +branch = true + +[tool.coverage.report] +precision = 2 +exclude_lines = [ + 'pragma: no cover', + 'raise NotImplementedError', + 'raise NotImplemented', + 'if TYPE_CHECKING:', + '@overload', +] + +[tool.black] +color = true +line-length = 120 +target-version = ['py39'] +skip-string-normalization = true + +[tool.isort] +line_length = 120 +known_third_party = 'foxglove' +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +color_output = true + +[tool.mypy] +follow_imports = 'silent' +strict_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +disallow_any_generics = true +check_untyped_defs = true +no_implicit_reexport = true +warn_unused_configs = true +disallow_subclassing_any = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true +disallow_untyped_calls = true + +# for strict mypy: (this is the tricky one :-)) +disallow_untyped_defs = true + +# remaining arguments from `mypy --strict` which cause errors +#no_implicit_optional = true +#warn_return_any = true + +[[tool.mypy.overrides]] +module = ['aioredis', 'watchgod'] +ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg index 12a7f168..8d42cac7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,59 +1,4 @@ -[tool:pytest] -testpaths = tests -timeout = 5 -filterwarnings = - error - ignore::DeprecationWarning:aioredis - [flake8] max-complexity = 10 max-line-length = 120 ignore = E203, W503 - -[coverage:run] -source = arq -branch = True - -[coverage:report] -precision = 2 -exclude_lines = - pragma: no cover - raise NotImplementedError - raise NotImplemented - if TYPE_CHECKING: - @overload - -[isort] -line_length=120 -known_third_party=pytest -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True - -[mypy] -follow_imports = silent -strict_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -disallow_any_generics = True -check_untyped_defs = True -no_implicit_reexport = True -warn_unused_configs = True -disallow_subclassing_any = True -disallow_incomplete_defs = True -disallow_untyped_decorators = True -disallow_untyped_calls = True - -# for strict mypy: (this is the tricky one :-)) -disallow_untyped_defs = True - -# remaining arguments from `mypy --strict` which cause errors -;no_implicit_optional = True -;warn_return_any = True - -[mypy-aioredis] -ignore_missing_imports = true - -[mypy-watchgod] -ignore_missing_imports = true diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt index 92e22d95..0940e8c7 100644 --- a/tests/requirements-linting.txt +++ b/tests/requirements-linting.txt @@ -1,8 +1,7 @@ -black==19.10b0 -flake8==3.7.9 -flake8-quotes==3 -isort==5.8.0 -mypy==0.812 -pycodestyle==2.5.0 -pyflakes==2.1.1 -twine==3.1.1 +black==21.12b0 +flake8==4.0.1 +flake8-quotes==3.3.1 +isort==5.10.1 +mypy==0.931 +pycodestyle==2.8.0 +pyflakes==2.4.0 diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt index 420e112c..2d9b6fa8 100644 --- a/tests/requirements-testing.txt +++ b/tests/requirements-testing.txt @@ -7,4 +7,4 @@ pytest-mock==3 pytest-sugar==0.9.2 pytest-timeout==1.3.3 pytest-toolbox==0.4 -twine==3.1.1 +twine==3.7.1 diff --git a/tests/test_cron.py b/tests/test_cron.py index 8df3da15..2b7555de 100644 --- a/tests/test_cron.py +++ b/tests/test_cron.py @@ -154,7 +154,8 @@ async def try_sleep(ctx): raise asyncio.CancelledError worker: Worker = worker( - cron_jobs=[cron(try_sleep, microsecond=20, run_at_startup=True, max_tries=2)], poll_delay=0.01, + cron_jobs=[cron(try_sleep, microsecond=20, run_at_startup=True, max_tries=2)], + poll_delay=0.01, ) await worker.main() assert worker.jobs_complete == 1 diff --git a/tests/test_worker.py b/tests/test_worker.py index 1822fdf9..e9c8c1c2 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -853,7 +853,11 @@ async def test(ctx): await arq_redis.enqueue_job('func', _job_id='testing') worker: Worker = worker( - functions=[func(test, name='func')], on_job_start=on_start, on_job_end=on_end, job_timeout=0.2, poll_delay=0.1, + functions=[func(test, name='func')], + on_job_start=on_start, + on_job_end=on_end, + job_timeout=0.2, + poll_delay=0.1, ) assert worker.jobs_complete == 0 assert worker.jobs_failed == 0 From 67be7b25b0473700da5be6a371dce0cfac99069c Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 26 Jan 2022 13:50:21 +0000 Subject: [PATCH 02/13] fixing build --- Makefile | 2 +- arq/version.py | 2 +- setup.cfg | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) delete mode 100644 setup.cfg diff --git a/Makefile b/Makefile index f46bf9d5..0155352b 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ format: .PHONY: lint lint: - flake8 arq/ tests/ + flake8 --max-complexity 10 --max-line-length 120 --ignore E203,W503 arq/ tests/ $(isort) --check-only --df $(black) --check diff --git a/arq/version.py b/arq/version.py index 83522fc0..841cb6c5 100644 --- a/arq/version.py +++ b/arq/version.py @@ -1,3 +1,3 @@ __all__ = ['VERSION'] -VERSION = 'dev' +VERSION = '0.0.dev0' diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8d42cac7..00000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-complexity = 10 -max-line-length = 120 -ignore = E203, W503 From 4f58f4588820f85965717ce222a538648c81ffae Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 26 Jan 2022 14:01:41 +0000 Subject: [PATCH 03/13] isort colours --- tests/requirements-linting.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt index 0940e8c7..405bd395 100644 --- a/tests/requirements-linting.txt +++ b/tests/requirements-linting.txt @@ -1,7 +1,7 @@ black==21.12b0 flake8==4.0.1 flake8-quotes==3.3.1 -isort==5.10.1 +isort[colors]==5.10.1 mypy==0.931 pycodestyle==2.8.0 pyflakes==2.4.0 From 2ae252772f2fd03e2f99a8dc750d47956187f026 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 26 Jan 2022 14:08:25 +0000 Subject: [PATCH 04/13] uprev test deps --- pyproject.toml | 1 + tests/conftest.py | 12 +++++++++--- tests/requirements-testing.txt | 16 ++++++++-------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6db514f2..c38645e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.pytest.ini_options] testpaths = 'tests' filterwarnings = ['error', 'ignore::DeprecationWarning:aioredis'] +asyncio_mode = 'auto' [tool.coverage.run] source = ['src'] diff --git a/tests/conftest.py b/tests/conftest.py index e071f857..aa7c9704 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,13 @@ from arq.worker import Worker -@pytest.yield_fixture +@pytest.fixture(name='loop') +def _fix_loop(event_loop): + asyncio.set_event_loop(event_loop) + return event_loop + + +@pytest.fixture async def arq_redis(loop): redis_ = await create_redis_pool( ('localhost', 6379), encoding='utf8', loop=loop, commands_factory=ArqRedis, minsize=5 @@ -20,7 +26,7 @@ async def arq_redis(loop): await redis_.wait_closed() -@pytest.yield_fixture +@pytest.fixture async def arq_redis_msgpack(loop): redis_ = await create_redis_pool( ('localhost', 6379), @@ -36,7 +42,7 @@ async def arq_redis_msgpack(loop): await redis_.wait_closed() -@pytest.yield_fixture +@pytest.fixture async def worker(arq_redis): worker_: Worker = None diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt index 2d9b6fa8..169e0b3d 100644 --- a/tests/requirements-testing.txt +++ b/tests/requirements-testing.txt @@ -1,10 +1,10 @@ -coverage==5.1 -msgpack==0.6.1 -pytest==5.3.5 -pytest-aiohttp==0.3.0 -pytest-cov==2.8.1 -pytest-mock==3 -pytest-sugar==0.9.2 -pytest-timeout==1.3.3 +coverage==6.3 +msgpack==1.0.3 +pytest==6.2.5 +pytest-asyncio==0.17.2 +pytest-cov==3.0.0 +pytest-mock==3.6.1 +pytest-sugar==0.9.4 +pytest-timeout==2.1.0 pytest-toolbox==0.4 twine==3.7.1 From 83c11b278e0d92defe93e59bb684fc33faf14d64 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 26 Jan 2022 14:09:49 +0000 Subject: [PATCH 05/13] drop python 3.6 support --- .github/workflows/ci.yml | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b7f0460..fb0f71b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: fail-fast: false matrix: os: [ubuntu] - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ['3.7', '3.8', '3.9'] env: PYTHON: ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 4bb91b72..d3bc9120 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', From 67e543929c6f8be987fe964f38d87fac0b14aaa2 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 26 Jan 2022 14:46:58 +0000 Subject: [PATCH 06/13] remove broken test --- tests/test_main.py | 7 ++++--- tests/test_utils.py | 14 -------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 263fe1ac..6ffb9383 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -172,9 +172,10 @@ async def count(ctx, v): tasks = [] for i in range(50): - tasks.extend( - [arq_redis.enqueue_job('count', i, _job_id=f'v-{i}'), arq_redis.enqueue_job('count', i, _job_id=f'v-{i}')] - ) + tasks += [ + arq_redis.enqueue_job('count', i, _job_id=f'v-{i}'), + arq_redis.enqueue_job('count', i, _job_id=f'v-{i}'), + ] shuffle(tasks) await asyncio.gather(*tasks) diff --git a/tests/test_utils.py b/tests/test_utils.py index 9f30fb09..f07be313 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -26,20 +26,6 @@ async def test_redis_timeout(mocker, create_pool): assert arq.utils.asyncio.sleep.call_count == 5 -async def test_redis_sentinel_failure(create_pool): - """ - FIXME: this is currently causing 3 "Task was destroyed but it is pending!" warnings - """ - settings = RedisSettings() - settings.host = [('localhost', 6379), ('localhost', 6379)] - settings.sentinel = True - try: - pool = await create_pool(settings) - await pool.ping('ping') - except Exception as e: - assert 'unknown command `SENTINEL`' in str(e) - - async def test_redis_success_log(caplog, create_pool): caplog.set_level(logging.INFO) settings = RedisSettings() From 511c805e612e6784dbc0b7bf493a8b9eefd98d83 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Thu, 27 Jan 2022 23:31:32 +0000 Subject: [PATCH 07/13] trying to fix tests :poop: --- pyproject.toml | 1 + tests/requirements-testing.txt | 2 +- tests/test_cli.py | 4 ++++ tests/test_jobs.py | 14 +++++++------- tests/test_main.py | 16 ++++++++-------- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c38645e6..de7d5f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ testpaths = 'tests' filterwarnings = ['error', 'ignore::DeprecationWarning:aioredis'] asyncio_mode = 'auto' +timeout = 10 [tool.coverage.run] source = ['src'] diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt index 169e0b3d..35c070c8 100644 --- a/tests/requirements-testing.txt +++ b/tests/requirements-testing.txt @@ -1,4 +1,5 @@ coverage==6.3 +dirty-equals==0.0.1 msgpack==1.0.3 pytest==6.2.5 pytest-asyncio==0.17.2 @@ -6,5 +7,4 @@ pytest-cov==3.0.0 pytest-mock==3.6.1 pytest-sugar==0.9.4 pytest-timeout==2.1.0 -pytest-toolbox==0.4 twine==3.7.1 diff --git a/tests/test_cli.py b/tests/test_cli.py index d44529c3..57fb88a8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import pytest from click.testing import CliRunner from arq.cli import cli @@ -19,6 +20,7 @@ def test_help(): assert result.output.startswith('Usage: arq [OPTIONS] WORKER_SETTINGS\n') +@pytest.mark.skip(reason='this is breaking the event loop for other tests') def test_run(): runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings']) @@ -26,6 +28,7 @@ def test_run(): assert 'Starting worker for 1 functions: foobar' in result.output +@pytest.mark.skip(reason='this is breaking the event loop for other tests') def test_check(): runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--check']) @@ -37,6 +40,7 @@ async def mock_awatch(): yield [1] +@pytest.mark.skip(reason='this is breaking the event loop for other tests') def test_run_watch(mocker): mocker.patch('watchgod.awatch', return_value=mock_awatch()) runner = CliRunner() diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 83df2cb0..a03da2f0 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -2,7 +2,7 @@ import pickle import pytest -from pytest_toolbox.comparison import CloseToNow +from dirty_equals import IsNow from arq import Worker, func from arq.connections import ArqRedis, RedisSettings, create_pool @@ -49,11 +49,11 @@ async def foobar(ctx, *args, **kwargs): function='foobar', args=(1, 2), kwargs={'c': 3}, - enqueue_time=CloseToNow(), + enqueue_time=IsNow(), success=True, result=42, - start_time=CloseToNow(), - finish_time=CloseToNow(), + start_time=IsNow(), + finish_time=IsNow(), score=None, queue_name=expected_queue_name, ) @@ -64,11 +64,11 @@ async def foobar(ctx, *args, **kwargs): args=(1, 2), kwargs={'c': 3}, job_try=1, - enqueue_time=CloseToNow(), + enqueue_time=IsNow(), success=True, result=42, - start_time=CloseToNow(), - finish_time=CloseToNow(), + start_time=IsNow(), + finish_time=IsNow(), score=None, queue_name=expected_queue_name, job_id=j.job_id, diff --git a/tests/test_main.py b/tests/test_main.py index 6ffb9383..381f1767 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -9,7 +9,7 @@ import msgpack import pytest -from pytest_toolbox.comparison import AnyInt, CloseToNow +from dirty_equals import IsInt, IsNow from arq.connections import ArqRedis from arq.constants import default_queue_name @@ -130,7 +130,7 @@ async def test_job_info(arq_redis: ArqRedis): t_before = time() j = await arq_redis.enqueue_job('foobar', 123, a=456) info = await j.info() - assert info.enqueue_time == CloseToNow() + assert info.enqueue_time == IsNow assert info.job_try is None assert info.function == 'foobar' assert info.args == (123,) @@ -250,24 +250,24 @@ async def test_get_jobs(arq_redis: ArqRedis): 'args': (), 'kwargs': {'a': 1, 'b': 2, 'c': 3}, 'job_try': None, - 'enqueue_time': CloseToNow(), - 'score': AnyInt(), + 'enqueue_time': IsNow, + 'score': IsInt, }, { 'function': 'second', 'args': (4,), 'kwargs': {'b': 5, 'c': 6}, 'job_try': None, - 'enqueue_time': CloseToNow(), - 'score': AnyInt(), + 'enqueue_time': IsNow, + 'score': IsInt, }, { 'function': 'third', 'args': (7,), 'kwargs': {'b': 8}, 'job_try': None, - 'enqueue_time': CloseToNow(), - 'score': AnyInt(), + 'enqueue_time': IsNow, + 'score': IsInt, }, ] assert jobs[0].score < jobs[1].score < jobs[2].score From c941a502bdca375d9872a0afa79d3e1b53411c6f Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 28 Jan 2022 12:06:02 +0000 Subject: [PATCH 08/13] uprev dirty-equals --- tests/requirements-testing.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt index 35c070c8..7aab0df3 100644 --- a/tests/requirements-testing.txt +++ b/tests/requirements-testing.txt @@ -1,10 +1,10 @@ coverage==6.3 -dirty-equals==0.0.1 +dirty-equals==0.0.2 msgpack==1.0.3 pytest==6.2.5 pytest-asyncio==0.17.2 pytest-cov==3.0.0 -pytest-mock==3.6.1 +pytest-mock==3.7.0 pytest-sugar==0.9.4 pytest-timeout==2.1.0 twine==3.7.1 From 11bac754ad438b5f57642fcb7fd79de8b65374fd Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 28 Jan 2022 12:08:10 +0000 Subject: [PATCH 09/13] revent codecov breaking builds --- .codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index c9526802..0ea43a02 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,6 +1,9 @@ coverage: precision: 2 range: [95, 100] + status: + patch: false + project: false comment: layout: 'header, diff, flags, files, footer' From 4d42c9b70a6d2508b500d09e8fab660181695b6b Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 4 Mar 2022 13:06:13 +0000 Subject: [PATCH 10/13] uprev dirty-equals --- tests/requirements-testing.txt | 2 +- tests/test_jobs.py | 12 ++++++------ tests/test_main.py | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt index 7aab0df3..d23e7c4e 100644 --- a/tests/requirements-testing.txt +++ b/tests/requirements-testing.txt @@ -1,5 +1,5 @@ coverage==6.3 -dirty-equals==0.0.2 +dirty-equals==0.1 msgpack==1.0.3 pytest==6.2.5 pytest-asyncio==0.17.2 diff --git a/tests/test_jobs.py b/tests/test_jobs.py index a03da2f0..e8f23216 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -49,11 +49,11 @@ async def foobar(ctx, *args, **kwargs): function='foobar', args=(1, 2), kwargs={'c': 3}, - enqueue_time=IsNow(), + enqueue_time=IsNow(tz='utc'), success=True, result=42, - start_time=IsNow(), - finish_time=IsNow(), + start_time=IsNow(tz='utc'), + finish_time=IsNow(tz='utc'), score=None, queue_name=expected_queue_name, ) @@ -64,11 +64,11 @@ async def foobar(ctx, *args, **kwargs): args=(1, 2), kwargs={'c': 3}, job_try=1, - enqueue_time=IsNow(), + enqueue_time=IsNow(tz='utc'), success=True, result=42, - start_time=IsNow(), - finish_time=IsNow(), + start_time=IsNow(tz='utc'), + finish_time=IsNow(tz='utc'), score=None, queue_name=expected_queue_name, job_id=j.job_id, diff --git a/tests/test_main.py b/tests/test_main.py index 381f1767..69e340db 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -130,7 +130,7 @@ async def test_job_info(arq_redis: ArqRedis): t_before = time() j = await arq_redis.enqueue_job('foobar', 123, a=456) info = await j.info() - assert info.enqueue_time == IsNow + assert info.enqueue_time == IsNow(tz='utc') assert info.job_try is None assert info.function == 'foobar' assert info.args == (123,) @@ -250,24 +250,24 @@ async def test_get_jobs(arq_redis: ArqRedis): 'args': (), 'kwargs': {'a': 1, 'b': 2, 'c': 3}, 'job_try': None, - 'enqueue_time': IsNow, - 'score': IsInt, + 'enqueue_time': IsNow(tz='utc'), + 'score': IsInt(), }, { 'function': 'second', 'args': (4,), 'kwargs': {'b': 5, 'c': 6}, 'job_try': None, - 'enqueue_time': IsNow, - 'score': IsInt, + 'enqueue_time': IsNow(tz='utc'), + 'score': IsInt(), }, { 'function': 'third', 'args': (7,), 'kwargs': {'b': 8}, 'job_try': None, - 'enqueue_time': IsNow, - 'score': IsInt, + 'enqueue_time': IsNow(tz='utc'), + 'score': IsInt(), }, ] assert jobs[0].score < jobs[1].score < jobs[2].score From 2012a6c9c043ac3c2640884b0dbdb5181a738378 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 4 Mar 2022 13:34:40 +0000 Subject: [PATCH 11/13] fix cli tests --- arq/cli.py | 2 +- arq/worker.py | 3 +-- tests/test_cli.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/arq/cli.py b/arq/cli.py index 8cc03227..97f62992 100644 --- a/arq/cli.py +++ b/arq/cli.py @@ -43,7 +43,7 @@ def cli(*, worker_settings: str, burst: bool, check: bool, watch: str, verbose: else: kwargs = {} if burst is None else {'burst': burst} if watch: - asyncio.get_event_loop().run_until_complete(watch_reload(watch, worker_settings_)) + asyncio.run(watch_reload(watch, worker_settings_)) else: run_worker(worker_settings_, **kwargs) diff --git a/arq/worker.py b/arq/worker.py index 00ddb0b3..bb299ab1 100644 --- a/arq/worker.py +++ b/arq/worker.py @@ -806,5 +806,4 @@ def check_health(settings_cls: 'WorkerSettingsType') -> int: redis_settings = cast(Optional[RedisSettings], cls_kwargs.get('redis_settings')) health_check_key = cast(Optional[str], cls_kwargs.get('health_check_key')) queue_name = cast(Optional[str], cls_kwargs.get('queue_name')) - loop = asyncio.get_event_loop() - return loop.run_until_complete(async_check_health(redis_settings, health_check_key, queue_name)) + return asyncio.run(async_check_health(redis_settings, health_check_key, queue_name)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 57fb88a8..c2291be9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,5 @@ +import asyncio + import pytest from click.testing import CliRunner @@ -20,16 +22,16 @@ def test_help(): assert result.output.startswith('Usage: arq [OPTIONS] WORKER_SETTINGS\n') -@pytest.mark.skip(reason='this is breaking the event loop for other tests') -def test_run(): +def test_run(event_loop): runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings']) assert result.exit_code == 0 assert 'Starting worker for 1 functions: foobar' in result.output + tasks = asyncio.all_tasks(event_loop) + assert not tasks -@pytest.mark.skip(reason='this is breaking the event loop for other tests') -def test_check(): +def test_check(event_loop): runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--check']) assert result.exit_code == 1 @@ -40,8 +42,8 @@ async def mock_awatch(): yield [1] -@pytest.mark.skip(reason='this is breaking the event loop for other tests') -def test_run_watch(mocker): +@pytest.mark.filterwarnings('ignore::DeprecationWarning') +def test_run_watch(mocker, event_loop): mocker.patch('watchgod.awatch', return_value=mock_awatch()) runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--watch', 'tests']) From 101d8b561e5f79a9370d0cc52c5e88bdb98714a6 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 4 Mar 2022 14:09:34 +0000 Subject: [PATCH 12/13] comments and reinstate test_redis_sentinel_failure test --- arq/version.py | 2 ++ tests/conftest.py | 16 ++++++++++++++++ tests/test_cli.py | 8 ++++---- tests/test_utils.py | 11 +++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/arq/version.py b/arq/version.py index 841cb6c5..b47d1126 100644 --- a/arq/version.py +++ b/arq/version.py @@ -1,3 +1,5 @@ __all__ = ['VERSION'] +# version is set automatically in CI before release, +# see https://gist.github.com/samuelcolvin/da2f521da5d2195fbfd65da3b8f58589 VERSION = '0.0.dev0' diff --git a/tests/conftest.py b/tests/conftest.py index aa7c9704..0b6a9950 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -74,3 +74,19 @@ async def create_pool_(settings, *args, **kwargs): p.close() await asyncio.gather(*[p.wait_closed() for p in pools]) + + +@pytest.fixture(name='cancel_remaining_task') +def fix_cancel_remaining_task(loop): + async def cancel_remaining_task(): + tasks = asyncio.all_tasks(loop) + cancelled = [] + for task in tasks: + if task.get_coro().__name__ != 'cancel_remaining_task': + cancelled.append(task) + task.cancel() + await asyncio.gather(*cancelled, return_exceptions=True) + + yield + + loop.run_until_complete(cancel_remaining_task()) diff --git a/tests/test_cli.py b/tests/test_cli.py index c2291be9..6db2f405 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,16 +22,16 @@ def test_help(): assert result.output.startswith('Usage: arq [OPTIONS] WORKER_SETTINGS\n') -def test_run(event_loop): +def test_run(loop): runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings']) assert result.exit_code == 0 assert 'Starting worker for 1 functions: foobar' in result.output - tasks = asyncio.all_tasks(event_loop) + tasks = asyncio.all_tasks(loop) assert not tasks -def test_check(event_loop): +def test_check(loop): runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--check']) assert result.exit_code == 1 @@ -43,7 +43,7 @@ async def mock_awatch(): @pytest.mark.filterwarnings('ignore::DeprecationWarning') -def test_run_watch(mocker, event_loop): +def test_run_watch(mocker, loop): mocker.patch('watchgod.awatch', return_value=mock_awatch()) runner = CliRunner() result = runner.invoke(cli, ['tests.test_cli.WorkerSettings', '--watch', 'tests']) diff --git a/tests/test_utils.py b/tests/test_utils.py index f07be313..c2fcfca4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,11 @@ +import asyncio import logging import re from datetime import timedelta import pytest from pydantic import BaseModel, validator +from aioredis.errors import MasterReplyError import arq.typing import arq.utils @@ -26,6 +28,15 @@ async def test_redis_timeout(mocker, create_pool): assert arq.utils.asyncio.sleep.call_count == 5 +async def test_redis_sentinel_failure(create_pool, cancel_remaining_task): + settings = RedisSettings() + settings.host = [('localhost', 6379), ('localhost', 6379)] + settings.sentinel = True + with pytest.raises(MasterReplyError, match='unknown command `SENTINEL`'): + pool = await create_pool(settings) + await pool.ping('ping') + + async def test_redis_success_log(caplog, create_pool): caplog.set_level(logging.INFO) settings = RedisSettings() From ac1e1d9ef6a3b7674fff864d6b61e4704f6ddb6e Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Fri, 4 Mar 2022 14:17:32 +0000 Subject: [PATCH 13/13] fix for 3.7, formatting --- tests/conftest.py | 3 ++- tests/test_utils.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0b6a9950..5492abe9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -82,7 +82,8 @@ async def cancel_remaining_task(): tasks = asyncio.all_tasks(loop) cancelled = [] for task in tasks: - if task.get_coro().__name__ != 'cancel_remaining_task': + # in repr works in 3.7 where get_coro() is not available + if 'cancel_remaining_task()' not in repr(task): cancelled.append(task) task.cancel() await asyncio.gather(*cancelled, return_exceptions=True) diff --git a/tests/test_utils.py b/tests/test_utils.py index c2fcfca4..4d7c8120 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,11 +1,10 @@ -import asyncio import logging import re from datetime import timedelta import pytest -from pydantic import BaseModel, validator from aioredis.errors import MasterReplyError +from pydantic import BaseModel, validator import arq.typing import arq.utils