adding python 3.8 (#178)

* adding python 3.8 * move to github actions * fix tests * fixing python 3.8 tests * fix formatting * fix readme
python-arq · Apr 22, 2020 · 3fec03c · 3fec03c
1 parent 532ed21
commit 3fec03c
Show file tree

Hide file tree

Showing 16 changed files with 196 additions and 103 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,90 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '**'
+  pull_request: {}
+
+jobs:
+  test:
+    name: test py${{ matrix.python-version }} on ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        python-version: ['3.6', '3.7', '3.8']
+
+    env:
+      PYTHON: ${{ matrix.python-version }}
+      OS: ${{ matrix.os }}
+
+    runs-on: ${{ format('{0}-latest', matrix.os) }}
+
+    services:
+      redis:
+        image: redis:5
+        ports:
+        - 6379:6379
+        options: --entrypoint redis-server
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: install dependencies
+      run: |
+        make install
+        pip freeze
+
+    - name: lint
+      run: make lint
+
+    - name: test
+      run: |
+        make test
+        coverage xml
+
+    - uses: samuelcolvin/codecov-action@env-vars
+      with:
+        file: ./coverage.xml
+        env_vars: PYTHON,OS
+
+  deploy:
+    name: Deploy
+    needs: test
+    if: "success() && startsWith(github.ref, 'refs/tags/')"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: set up python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.8'
+
+      - name: install
+        run: |
+          make install
+          pip install -U wheel twine
+
+      - name: build
+        run: python setup.py sdist bdist_wheel
+
+      - run: twine check dist/*
+
+      - name: check tag
+        run: PACKAGE=arq python <(curl -Ls https://git.io/JvQsH)
+
+      - name: upload to pypi
+        run: twine upload dist/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.pypi_token }}
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,4 @@
-/env/
-/env36/
+/env*/
 /.idea
 __pycache__/
 *.py[cod]

diff --git a/.pyup.yml b/.pyup.yml
diff --git a/.travis.yml b/.travis.yml
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,10 @@
 History
 -------
 
+v0.18.5 (unreleased)
+....................
+* Python 3.8 support
+
 v0.18.4 (2019-12-19)
 ....................
 * Add ``py.typed`` file to tell mypy the package has type hints, #163

diff --git a/Makefile b/Makefile
@@ -15,10 +15,9 @@ format:
 
 .PHONY: lint
 lint:
-	python setup.py check -rms
 	flake8 arq/ tests/
-	$(isort) --check-only
-	$(black) --check arq
+	$(isort) --check-only -df
+	$(black) --check
 
 .PHONY: test
 test:
@@ -29,7 +28,7 @@ testcov:
 	pytest --cov=arq && (echo "building coverage html"; coverage html)
 
 .PHONY: all
-all: testcov lint
+all: lint testcov
 
 .PHONY: clean
 clean:
@@ -38,6 +37,7 @@ clean:
 	rm -f `find . -type f -name '*~' `
 	rm -f `find . -type f -name '.*~' `
 	rm -rf .cache
+	rm -rf .pytest_cache
 	rm -rf htmlcov
 	rm -rf *.egg-info
 	rm -f .coverage

diff --git a/README.md b/README.md
@@ -0,0 +1,11 @@
+# arq
+
+[![CI](https://github.com/samuelcolvin/arq/workflows/CI/badge.svg?event=push)](https://github.com/samuelcolvin/arq/actions?query=event%3Apush+branch%3Amaster+workflow%3ACI)
+[![Coverage](https://codecov.io/gh/samuelcolvin/arq/branch/master/graph/badge.svg)](https://codecov.io/gh/samuelcolvin/arq)
+[![pypi](https://img.shields.io/pypi/v/arq.svg)](https://pypi.python.org/pypi/arq)
+[![versions](https://img.shields.io/pypi/pyversions/arq.svg)](https://github.com/samuelcolvin/arq)
+[![license](https://img.shields.io/github/license/samuelcolvin/arq.svg)](https://github.com/samuelcolvin/arq/blob/master/LICENSE)
+
+Job queues in python with asyncio and redis.
+
+See [documentation](https://arq-docs.helpmanual.io/) for more details.
diff --git a/README.rst b/README.rst
diff --git a/arq/version.py b/arq/version.py
@@ -1,5 +1,3 @@
-from distutils.version import StrictVersion
+__all__ = ('VERSION',)
 
-__all__ = ['VERSION']
-
-VERSION = StrictVersion('0.18.4')
+VERSION = '0.18.5a1'
diff --git a/arq/worker.py b/arq/worker.py
@@ -121,6 +121,10 @@ def __repr__(self):
         return f'<{str(self)}>'
 
 
+class RetryJob(RuntimeError):
+    pass
+
+
 class Worker:
     """
     Main class for running jobs.
@@ -450,7 +454,7 @@ async def job_failed(exc: Exception):
                 if e.defer_score:
                     incr_score = e.defer_score + (timestamp_ms() - score)
                 self.jobs_retried += 1
-            elif self.retry_jobs and isinstance(e, asyncio.CancelledError):
+            elif self.retry_jobs and isinstance(e, (asyncio.CancelledError, RetryJob)):
                 logger.info('%6.2fs ↻ %s cancelled, will be run again', t, ref)
                 self.jobs_retried += 1
             else:

diff --git a/setup.cfg b/setup.cfg
@@ -1,7 +1,9 @@
 [tool:pytest]
 testpaths = tests
 timeout = 5
-filterwarnings = error
+filterwarnings =
+    error
+    ignore::DeprecationWarning:aioredis
 
 [flake8]
 max-complexity = 10

diff --git a/setup.py b/setup.py
@@ -3,20 +3,22 @@
 from importlib.machinery import SourceFileLoader
 from setuptools import setup
 
-readme = Path(__file__).parent.joinpath('README.rst')
+description = 'Job queues in python with asyncio and redis'
+readme = Path(__file__).parent.joinpath('README.md')
 if readme.exists():
     with readme.open() as f:
         long_description = f.read()
 else:
-    long_description = '-'
+    long_description = description + '.\n\nSee https://arq-docs.helpmanual.io/ for documentation.'
 # avoid loading the package before requirements are installed:
 version = SourceFileLoader('version', 'arq/version.py').load_module()
 
 setup(
     name='arq',
-    version=str(version.VERSION),
-    description='Job queues in python with asyncio and redis.',
+    version=version.VERSION,
+    description=description,
     long_description=long_description,
+    long_description_content_type='text/markdown',
     classifiers=[
         'Development Status :: 4 - Beta',
         'Environment :: Console',
@@ -32,6 +34,7 @@
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Internet',
         'Topic :: Software Development :: Libraries :: Python Modules',
         'Topic :: System :: Clustering',
@@ -55,7 +58,7 @@
         'async-timeout>=3.0.0',
         'aioredis>=1.1.0',
         'click>=6.7',
-        'pydantic>=0.20',
+        'pydantic>=1',
         'dataclasses>=0.6;python_version == "3.6"'
     ],
     extras_require={

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,10 +1,11 @@
+import asyncio
 import functools
 
 import msgpack
 import pytest
 from aioredis import create_redis_pool
 
-from arq.connections import ArqRedis
+from arq.connections import ArqRedis, create_pool
 from arq.worker import Worker
 
 
@@ -50,3 +51,20 @@ def create(functions=[], burst=True, poll_delay=0, max_jobs=10, **kwargs):
 
     if worker_:
         await worker_.close()
+
+
+@pytest.fixture(name='create_pool')
+async def fix_create_pool(loop):
+    pools = []
+
+    async def create_pool_(settings, *args, **kwargs):
+        pool = await create_pool(settings, *args, **kwargs)
+        pools.append(pool)
+        return pool
+
+    yield create_pool_
+
+    for p in pools:
+        p.close()
+
+    await asyncio.gather(*[p.wait_closed() for p in pools])
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -1,15 +1,16 @@
-black==19.3b0
-coverage==4.5.3
-docutils==0.14
-flake8==3.7.7
-isort==4.3.17
+black==19.10b0
+coverage==5.1
+flake8==3.7.9
+flake8-quotes==3
+isort==4.3.21
+msgpack==0.6.1
 pycodestyle==2.5.0
 pyflakes==2.1.1
-pytest==4.4.1
+pytest==5.3.5
 pytest-aiohttp==0.3.0
-pytest-cov==2.6.1
-pytest-mock==1.10.4
+pytest-cov==2.8.1
+pytest-mock==3
 pytest-sugar==0.9.2
 pytest-timeout==1.3.3
 pytest-toolbox==0.4
-msgpack==0.6.1
+twine==3.1.1