From f37a43ab95262a936f55f902a105cc3732b6183e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 16 Nov 2020 15:31:42 +0000 Subject: [PATCH 01/11] drop py2.6/3.2/3.3 and distutils (no longer tested) - drop py2.6 (replaces/closes #502 <- fixes #620, invalidates/fixes #127) - drop distutils in favour of setup.cfg (fixes #723, fixes #721) - related setuptools_scm (#722) --- MANIFEST.in | 2 - examples/7zx.py | 6 +- pyproject.toml | 3 + setup.cfg | 91 ++++++++++++++++++++++++++++++ setup.py | 107 +---------------------------------- tox.ini | 21 +------ tqdm/_utils.py | 2 +- tqdm/std.py | 17 +++--- tqdm/utils.py | 144 +++++++++++------------------------------------- 9 files changed, 145 insertions(+), 248 deletions(-) create mode 100644 pyproject.toml diff --git a/MANIFEST.in b/MANIFEST.in index b6e6483d1..6fbd80b15 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ # Misc include .coveragerc include CONTRIBUTING.md -include LICENCE include logo.png # include images/logo.gif include Makefile @@ -16,7 +15,6 @@ include requirements-dev.txt # Examples/Documentation recursive-include examples *.py -include README.rst include tqdm/tqdm.1 include tqdm/completion.sh include DEMO.ipynb diff --git a/examples/7zx.py b/examples/7zx.py index 5f0a5e8b9..8db7105c8 100644 --- a/examples/7zx.py +++ b/examples/7zx.py @@ -29,7 +29,7 @@ import io __author__ = "Casper da Costa-Luis " __licence__ = "MPLv2.0" -__version__ = "0.2.1" +__version__ = "0.2.2" __license__ = __licence__ RE_SCN = re.compile(r"([0-9]+)\s+([0-9]+)\s+(.*)$", flags=re.M) @@ -59,8 +59,8 @@ def main(): if totals_s != totals[s]: log.warn("%s: individual total %d != 7z total %d" % ( fn, totals_s, totals[s])) - fcomp = dict((n, int(c if args.compressed else u)) - for (u, c, n) in finfo[:-1]) + fcomp = {n: int(c if args.compressed else u) + for (u, c, n) in finfo[:-1]} # log.debug(fcomp) # zips : {'zipname' : {'filename' : int(size)}} zips[fn] = fcomp diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..abcd7baf7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=38.6.0"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 46d3a553b..6fe318fb1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,94 @@ python_files = tests_*.py testpaths = tests addopts = -v --tb=short usefixtures = pretest_posttest + +[metadata] +name = tqdm +url = https://github.com/tqdm/tqdm +project_urls = + Changelog = https://tqdm.github.io/releases + Documentation = https://github.com/tqdm/tqdm#tqdm + Documentation (dev) = https://tqdm.github.io/docs/tqdm + Wiki = https://github.com/tqdm/tqdm/wiki +maintainer = tqdm developers +maintainer_email = python.tqdm@gmail.com +license = MPLv2.0, MIT Licences +license_file = LICENCE +description = Fast, Extensible Progress Meter +long_description = file: README.rst +long_description_content_type = text/x-rst +keywords = progressbar, progressmeter, progress, bar, meter, rate, eta, console, terminal, time +platforms = any +provides = tqdm +# Trove classifiers (https://pypi.org/pypi?%3Aaction=list_classifiers) +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: MacOS X + Environment :: Other Environment + Environment :: Win32 (MS Windows) + Environment :: X11 Applications + Framework :: IPython + Framework :: Jupyter + Intended Audience :: Developers + Intended Audience :: Education + Intended Audience :: End Users/Desktop + Intended Audience :: Other Audience + Intended Audience :: System Administrators + License :: OSI Approved :: MIT License + License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) + Operating System :: MacOS + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft + Operating System :: Microsoft :: MS-DOS + Operating System :: Microsoft :: Windows + Operating System :: POSIX + Operating System :: POSIX :: BSD + Operating System :: POSIX :: BSD :: FreeBSD + Operating System :: POSIX :: Linux + Operating System :: POSIX :: SunOS/Solaris + Operating System :: Unix + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: Implementation + Programming Language :: Python :: Implementation :: IronPython + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Unix Shell + Topic :: Desktop Environment + Topic :: Education :: Computer Aided Instruction (CAI) + Topic :: Education :: Testing + Topic :: Office/Business + Topic :: Other/Nonlisted Topic + Topic :: Software Development :: Build Tools + Topic :: Software Development :: Libraries + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Software Development :: Pre-processors + Topic :: Software Development :: User Interfaces + Topic :: System :: Installation/Setup + Topic :: System :: Logging + Topic :: System :: Monitoring + Topic :: System :: Shells + Topic :: Terminals + Topic :: Utilities + +[options] +setup_requires = setuptools>=38.6.0 +python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +tests_require = pytest; flake8; coverage +include_package_data = True +packages = find: +[options.entry_points] +console_scripts = + tqdm = tqdm.cli:main +[options.package_data] +tqdm = CONTRIBUTING.md, examples/*.py, tqdm.1, completion.sh, requirements-dev.txt +[options.packages.find] +exclude = benchmarks, tests diff --git a/setup.py b/setup.py index 4d06a7a96..dc81a96ac 100755 --- a/setup.py +++ b/setup.py @@ -1,18 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from io import open as io_open +from setuptools import setup import os -try: - from setuptools import setup, find_packages -except ImportError: - from distutils.core import setup - - def find_packages(where='.'): - # os.walk -> list[(dirname, list[subdirs], list[files])] - return [folder.replace("/", ".").lstrip(".") - for (folder, _, fils) in os.walk(where) - if "__init__.py" in fils] import sys -from io import open as io_open # Get version from tqdm/_version.py __version__ = None @@ -36,96 +27,4 @@ def find_packages(where='.'): extras_require['dev'] = [i.strip().split('#', 1)[0].strip() for i in fd.read().strip().split('\n')] -README_rst = '' -fndoc = os.path.join(src_dir, 'README.rst') -with io_open(fndoc, mode='r', encoding='utf-8') as fd: - README_rst = fd.read() -setup( - name='tqdm', - version=__version__, - description='Fast, Extensible Progress Meter', - long_description=README_rst, - license='MPLv2.0, MIT Licences', - url='https://github.com/tqdm/tqdm', - project_urls={'Changelog': 'https://tqdm.github.io/releases', - 'Documentation': 'https://github.com/tqdm/tqdm#tqdm', - 'Documentation (dev)': 'https://tqdm.github.io/docs/tqdm', - 'Wiki': 'https://github.com/tqdm/tqdm/wiki'}, - maintainer='tqdm developers', - maintainer_email='python.tqdm@gmail.com', - platforms=['any'], - packages=['tqdm'] + ['tqdm.' + i for i in find_packages('tqdm')], - provides=['tqdm'], - extras_require=extras_require, - entry_points={'console_scripts': ['tqdm=tqdm.cli:main'], }, - package_data={'tqdm': ['CONTRIBUTING.md', 'LICENCE', 'examples/*.py', - 'tqdm.1', 'completion.sh', 'requirements-dev.txt']}, - python_requires='>=2.6, !=3.0.*, !=3.1.*', - classifiers=[ - # Trove classifiers - # (https://pypi.org/pypi?%3Aaction=list_classifiers) - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Environment :: MacOS X', - 'Environment :: Other Environment', - 'Environment :: Win32 (MS Windows)', - 'Environment :: X11 Applications', - 'Framework :: IPython', - 'Framework :: Jupyter', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Other Audience', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: MIT License', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', - 'Operating System :: MacOS', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft', - 'Operating System :: Microsoft :: MS-DOS', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: POSIX :: BSD', - 'Operating System :: POSIX :: BSD :: FreeBSD', - 'Operating System :: POSIX :: Linux', - 'Operating System :: POSIX :: SunOS/Solaris', - 'Operating System :: Unix', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation', - 'Programming Language :: Python :: Implementation :: IronPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Programming Language :: Unix Shell', - 'Topic :: Desktop Environment', - 'Topic :: Education :: Computer Aided Instruction (CAI)', - 'Topic :: Education :: Testing', - 'Topic :: Office/Business', - 'Topic :: Other/Nonlisted Topic', - 'Topic :: Software Development :: Build Tools', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Software Development :: Pre-processors', - 'Topic :: Software Development :: User Interfaces', - 'Topic :: System :: Installation/Setup', - 'Topic :: System :: Logging', - 'Topic :: System :: Monitoring', - 'Topic :: System :: Shells', - 'Topic :: Terminals', - 'Topic :: Utilities' - ], - keywords='progressbar progressmeter progress bar meter' - ' rate eta console terminal time', - test_suite='pytest', - tests_require=['pytest', 'flake8', 'coverage'], -) +setup(version=__version__, extras_require=extras_require) diff --git a/tox.ini b/tox.ini index a9eebd60c..bd8bc0657 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,9 @@ # and then run "tox" from this directory. [tox] -# deprecation warning: py{26,32,33,34} -envlist = py{26,27,33,34,35,36,37,38,39,py2,py3}{,-tf}{,-keras}, perf, flake8, setup.py +# deprecation warning: py{34} +envlist = py{27,34,35,36,37,38,39,py2,py3}{,-tf}{,-keras}, perf, flake8, setup.py +isolated_build = True [coverage] deps = @@ -48,28 +49,12 @@ allowlist_externals = {[extra]allowlist_externals} # no cython/numpy/pandas for py{py2,py3,26,33,34} -[testenv:py26] -deps = - pytest - pytest-cov - coverage - coveralls==1.2.0 - codecov - pycparser==2.18 - idna==2.7 -commands = - {[coverage]commands} - codecov - [testenv:pypy] deps = {[extra]deps} [testenv:pypy3] deps = {[extra]deps} -[testenv:py33] -deps = {[extra]deps} - [testenv:py34] # py34-compatible pandas deps = {[extra]deps} diff --git a/tqdm/_utils.py b/tqdm/_utils.py index 084327f47..915b5106b 100644 --- a/tqdm/_utils.py +++ b/tqdm/_utils.py @@ -1,4 +1,4 @@ -from .utils import CUR_OS, IS_WIN, IS_NIX, RE_ANSI, _range, _unich, _unicode, colorama, WeakSet, _basestring, _OrderedDict, FormatReplace, Comparable, SimpleTextIOWrapper, _is_utf, _supports_unicode, _is_ascii, _screen_shape_wrapper, _screen_shape_windows, _screen_shape_tput, _screen_shape_linux, _environ_cols_wrapper, _term_move_up # NOQA +from .utils import CUR_OS, IS_WIN, IS_NIX, RE_ANSI, _range, _unich, _unicode, colorama, _basestring, FormatReplace, Comparable, SimpleTextIOWrapper, _is_utf, _supports_unicode, _is_ascii, _screen_shape_wrapper, _screen_shape_windows, _screen_shape_tput, _screen_shape_linux, _environ_cols_wrapper, _term_move_up # NOQA from .std import TqdmDeprecationWarning from warnings import warn warn("This function will be removed in tqdm==5.0.0\n" diff --git a/tqdm/std.py b/tqdm/std.py index 804dfe1d4..f525dea84 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -8,20 +8,21 @@ ... ... """ from __future__ import absolute_import, division -# compatibility functions and utilities -from .utils import _supports_unicode, _screen_shape_wrapper, _range, _unich, \ - _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \ - Comparable, _is_ascii, FormatReplace, disp_len, disp_trim, \ - SimpleTextIOWrapper, DisableOnWriteError, CallbackIOWrapper -from ._monitor import TMonitor -# native libraries +from collections import OrderedDict from contextlib import contextmanager from datetime import datetime, timedelta from numbers import Number from time import time from warnings import warn +from weakref import WeakSet import sys +from .utils import _supports_unicode, _screen_shape_wrapper, _range, _unich, \ + _term_move_up, _unicode, _basestring, \ + Comparable, _is_ascii, FormatReplace, disp_len, disp_trim, \ + SimpleTextIOWrapper, DisableOnWriteError, CallbackIOWrapper +from ._monitor import TMonitor + __author__ = "https://github.com/tqdm/tqdm#contributions" __all__ = ['tqdm', 'trange', 'TqdmTypeError', 'TqdmKeyError', 'TqdmWarning', @@ -1443,7 +1444,7 @@ def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): kwargs : dict, optional """ # Sort in alphabetical order to be more deterministic - postfix = _OrderedDict([] if ordered_dict is None else ordered_dict) + postfix = OrderedDict([] if ordered_dict is None else ordered_dict) for key in sorted(kwargs.keys()): postfix[key] = kwargs[key] # Preprocess stats according to datatype diff --git a/tqdm/utils.py b/tqdm/utils.py index 5108750d4..af03abade 100644 --- a/tqdm/utils.py +++ b/tqdm/utils.py @@ -5,127 +5,47 @@ from warnings import warn import os import re -import sys import subprocess +import sys -CUR_OS = sys.platform -IS_WIN = any(CUR_OS.startswith(i) for i in ['win32', 'cygwin']) -IS_NIX = any(CUR_OS.startswith(i) for i in ['aix', 'linux', 'darwin']) -RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]") +# py2/3 compat +try: + _range = xrange +except NameError: + _range = range +try: + _unich = unichr +except NameError: + _unich = chr -# Py2/3 compat. Empty conditional to avoid coverage -if True: # pragma: no cover - try: - _range = xrange - except NameError: - _range = range +try: + _unicode = unicode +except NameError: + _unicode = str - try: - _unich = unichr - except NameError: - _unich = chr +try: + _basestring = basestring +except NameError: + _basestring = str - try: - _unicode = unicode - except NameError: - _unicode = str +CUR_OS = sys.platform +IS_WIN = any(CUR_OS.startswith(i) for i in ['win32', 'cygwin']) +IS_NIX = any(CUR_OS.startswith(i) for i in ['aix', 'linux', 'darwin']) +RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]") - try: - if IS_WIN: - import colorama - else: - raise ImportError - except ImportError: - colorama = None +try: + if IS_WIN: + import colorama else: - try: - colorama.init(strip=False) - except TypeError: - colorama.init() - - try: - from weakref import WeakSet - except ImportError: - WeakSet = set - + raise ImportError +except ImportError: + colorama = None +else: try: - _basestring = basestring - except NameError: - _basestring = str - - try: # py>=2.7,>=3.1 - from collections import OrderedDict as _OrderedDict - except ImportError: - try: # older Python versions with backported ordereddict lib - from ordereddict import OrderedDict as _OrderedDict - except ImportError: # older Python versions without ordereddict lib - # Py2.6,3.0 compat, from PEP 372 - from collections import MutableMapping - - class _OrderedDict(dict, MutableMapping): - # Methods with direct access to underlying attributes - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at 1 argument, got %d', - len(args)) - if not hasattr(self, '_keys'): - self._keys = [] - self.update(*args, **kwds) - - def clear(self): - del self._keys[:] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - self._keys.append(key) - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - self._keys.remove(key) - - def __iter__(self): - return iter(self._keys) - - def __reversed__(self): - return reversed(self._keys) - - def popitem(self): - if not self: - raise KeyError - key = self._keys.pop() - value = dict.pop(self, key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - inst_dict.pop('_keys', None) - return self.__class__, (items,), inst_dict - - # Methods with indirect access via the above methods - setdefault = MutableMapping.setdefault - update = MutableMapping.update - pop = MutableMapping.pop - keys = MutableMapping.keys - values = MutableMapping.values - items = MutableMapping.items - - def __repr__(self): - pairs = ', '.join(map('%r: %r'.__mod__, self.items())) - return '%s({%s})' % (self.__class__.__name__, pairs) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d + colorama.init(strip=False) + except TypeError: + colorama.init() class FormatReplace(object): From dadb349248e14f63cdaa8c7605b14218ee354b93 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 16 Nov 2020 16:58:55 +0000 Subject: [PATCH 02/11] drop appveyor - win already tested by GHA --- appveyor.yml | 17 ----------------- tox.ini | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 33f4f105a..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,17 +0,0 @@ -build: false -environment: - global: - TOXENV: py - matrix: - - PYVER: 27 - - PYVER: 36 - - PYVER: 37 - #- PYVER: 38 -install: - - git fetch --tags - - C:\Python%PYVER%\scripts\pip install -U tox -test_script: - - C:\Python%PYVER%\scripts\tox -cache: - - '%LOCALAPPDATA%\pip\cache' - - '%USERPROFILE%\.cache\pre-commit' diff --git a/tox.ini b/tox.ini index bd8bc0657..eacee6d24 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ commands = allowlist_externals = {[coverage]allowlist_externals} [testenv] -passenv = CI TOXENV CODECOV_* COVERALLS_* CODACY_* HOME DISTUTILS_USE_SDK MSSdk INCLUDE LIB +passenv = CI TOXENV CODECOV_* COVERALLS_* CODACY_* HOME deps = {[extra]deps} cython From 69a179a075283a486eed151d4f6b1ebfd34f7def Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 16 Nov 2020 20:05:11 +0000 Subject: [PATCH 03/11] use setuptools_scm, upgrade snap build - fixes #722 --- .github/workflows/check.yml | 8 ++-- .github/workflows/test.yml | 10 ++++- .gitignore | 14 ++++--- .meta/{.snapcraft.yml => mksnap.py} | 41 ++++++++++++++++--- CONTRIBUTING.md | 50 +++++++++-------------- Makefile | 15 ++++--- pyproject.toml | 6 ++- requirements-dev.txt | 1 + setup.cfg | 2 +- setup.py | 12 +----- tests/tests_version.py | 12 +++--- tqdm/__init__.py | 3 +- tqdm/_version.py | 62 ----------------------------- tqdm/cli.py | 5 ++- tqdm/version.py | 9 +++++ 15 files changed, 111 insertions(+), 139 deletions(-) rename .meta/{.snapcraft.yml => mksnap.py} (59%) delete mode 100644 tqdm/_version.py create mode 100644 tqdm/version.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 7048ededd..6cc0ec205 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -15,13 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: '3.x' - - name: Install - run: pip install -U tox - - name: Test - run: tox + - run: pip install -U tox + - run: tox env: TOXENV: ${{ matrix.TOXENV }} asvfull: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c972256ad..2c3bfbe8b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: '3.x' @@ -23,6 +25,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} @@ -40,6 +44,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} @@ -80,14 +86,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: '3.x' - name: Install run: | sudo apt-get install -yqq pandoc - pip install setuptools_scm - git fetch --unshallow --tags pip install .[dev] make build .dockerignore snapcraft.yaml - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') diff --git a/.gitignore b/.gitignore index 0462ce335..b45f8695b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,12 +4,14 @@ *.so # Packages -tqdm.egg-info -build/ -dist/ -snapcraft.yaml -tqdm_*_amd64.snap -.dockerignore +/tqdm/_dist_ver.py +/.eggs/ +/tqdm.egg-info +/build/ +/dist/ +/snapcraft.yaml +/tqdm_*_amd64.snap +/.dockerignore # Unit test / coverage reports .tox/ diff --git a/.meta/.snapcraft.yml b/.meta/mksnap.py similarity index 59% rename from .meta/.snapcraft.yml rename to .meta/mksnap.py index bb1738049..5feef0b9f 100644 --- a/.meta/.snapcraft.yml +++ b/.meta/mksnap.py @@ -1,4 +1,24 @@ -name: tqdm +# -*- encoding: utf-8 -*- +""" +Auto-generate snapcraft.yaml. +""" +from __future__ import print_function +from configparser import ConfigParser +from io import open as io_open +from os import path +from subprocess import check_output +import re +import sys + +sys.path.insert(1, path.dirname(path.dirname(__file__))) +import tqdm # NOQA + +src_dir = path.abspath(path.dirname(__file__)) +cfg = ConfigParser() +cfg.read(path.join(path.dirname(src_dir), 'setup.cfg')) +setup_requires = re.split( + r"[;\s]+", str(cfg["options"].get("setup_requires")), flags=re.M) +snap_yml = r"""name: tqdm summary: A fast, extensible CLI progress bar description: | https://tqdm.github.io @@ -31,19 +51,20 @@ `tqdm` does not require any dependencies, just an environment supporting `carriage return \r` and `line feed \n` control characters. -adopt-info: tqdm grade: stable confinement: strict base: core18 -icon: {icon} +icon: logo.png +version: '{version}' license: MPL-2.0 parts: tqdm: plugin: python + python-packages: {setup_requires} python-version: python3 - source: {source} + source: . source-commit: '{commit}' - parse-info: [setup.py] + build-packages: [git] override-build: | snapcraftctl build cp $SNAPCRAFT_PART_BUILD/tqdm/completion.sh $SNAPCRAFT_PART_INSTALL/ @@ -51,3 +72,13 @@ tqdm: command: bin/tqdm completer: completion.sh +""".format( + version=tqdm.__version__, + setup_requires=repr(setup_requires), + commit=check_output(['git', 'describe', '--always']).decode('U8').strip()) +fname = path.join(path.dirname(src_dir), 'snapcraft.yaml') + +if __name__ == "__main__": + with io_open(fname, mode='w', encoding='utf-8') as fd: + fd.write(snap_yml.decode('U8') if hasattr(snap_yml, 'decode') + else snap_yml) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c11fb5a4..75eefaad5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -132,8 +132,6 @@ It's probably a good idea to add `[python setup.py] make pre-commit` to The tqdm repository managers should: -- regularly bump the version number in the file -[_version.py](https://raw.githubusercontent.com/tqdm/tqdm/master/tqdm/_version.py) - follow the [Semantic Versioning](https://semver.org/) convention - take care of this (instead of users) to avoid PR conflicts solely due to the version file bumping @@ -208,16 +206,7 @@ git merge --no-ff pr-branch-name [python setup.py] make alltests ``` -### 5 Version - -Modify `tqdm/_version.py` and amend the last (merge) commit: - -``` -git add tqdm/_version.py -git commit --amend # Add "+ bump version" in the commit message -``` - -### 6 Push to master +### 5 Push to master ``` git push origin master @@ -349,35 +338,34 @@ to assist with maintenance. ## QUICK DEV SUMMARY For experienced devs, once happy with local master, follow the steps below. -Much is automated so really it's steps 1-6, then 12(a). - -1. bump version in `tqdm/_version.py` -2. test (`[python setup.py] make alltests`) -3. `git commit [--amend] # -m "bump version"` -4. `git push` -5. wait for tests to pass - a) in case of failure, fix and go back to (2) -6. `git tag vM.m.p && git push --tags` or comment `/tag vM.m.p commit_hash` -7. **`[AUTO:GHA]`** `[python setup.py] make distclean` -8. **`[AUTO:GHA]`** `[python setup.py] make build` -9. **`[AUTO:GHA]`** upload to PyPI. either: +Much is automated so really it's steps 1-5, then 11(a). + +1. test (`[python setup.py] make alltests`) +2. `git commit [--amend] # -m "bump version"` +3. `git push` +4. wait for tests to pass + a) in case of failure, fix and go back to (1) +5. `git tag vM.m.p && git push --tags` or comment `/tag vM.m.p commit_hash` +6. **`[AUTO:GHA]`** `[python setup.py] make distclean` +7. **`[AUTO:GHA]`** `[python setup.py] make build` +8. **`[AUTO:GHA]`** upload to PyPI. either: a) `[python setup.py] make pypi`, or b) `twine upload -s -i $(git config user.signingkey) dist/tqdm-*` -10. **`[AUTO:GHA]`** upload to docker hub: +9. **`[AUTO:GHA]`** upload to docker hub: a) `make -B docker` b) `docker push tqdm/tqdm:latest` c) `docker push tqdm/tqdm:$(docker run -i --rm tqdm/tqdm -v)` -11. **`[AUTO:GHA]`** upload to snapcraft: +10. **`[AUTO:GHA]`** upload to snapcraft: a) `make snap`, and b) `snapcraft push tqdm*.snap --release stable` -12. Wait for GHA to draft a new release on +11. Wait for GHA to draft a new release on a) replace the commit history with helpful release notes, and click publish b) **`[AUTO:GHA]`** attach `dist/tqdm-*` binaries (usually only `*.whl*`) -13. **`[SUB][AUTO:GHA-rel]`** run `make` in the `wiki` submodule to update release notes -14. **`[SUB][AUTO:GHA-rel]`** run `make deploy` in the `docs` submodule to update website -15. **`[SUB][AUTO:GHA-rel]`** accept the automated PR in the `feedstock` submodule to update conda -16. **`[AUTO:GHA-rel]`** update the [gh-pages project] benchmarks +12. **`[SUB][AUTO:GHA-rel]`** run `make` in the `wiki` submodule to update release notes +13. **`[SUB][AUTO:GHA-rel]`** run `make deploy` in the `docs` submodule to update website +14. **`[SUB][AUTO:GHA-rel]`** accept the automated PR in the `feedstock` submodule to update conda +15. **`[AUTO:GHA-rel]`** update the [gh-pages project] benchmarks a) `[python setup.py] make testasvfull` b) `asv gh-pages` diff --git a/Makefile b/Makefile index 0e479d234..ff9d2e0e8 100644 --- a/Makefile +++ b/Makefile @@ -102,15 +102,11 @@ tqdm/completion.sh: .meta/mkcompletion.py tqdm/std.py tqdm/cli.py README.rst: .meta/.readme.rst tqdm/std.py tqdm/cli.py @python .meta/mkdocs.py -snapcraft.yaml: .meta/.snapcraft.yml - cat "$<" | sed -e 's/{version}/'"`python -m tqdm --version`"'/g' \ - -e 's/{commit}/'"`git describe --always`"'/g' \ - -e 's/{source}/./g' -e 's/{icon}/logo.png/g' \ - -e 's/{description}/https:\/\/tqdm.github.io/g' > "$@" +snapcraft.yaml: .meta/mksnap.py + @python .meta/mksnap.py -.dockerignore: .gitignore - echo '*' > $@ - echo '!dist/*.whl' >> $@ +.dockerignore: + @+python -c "fd=open('.dockerignore', 'w'); fd.write('*\n!dist/*.whl\n')" distclean: @+make coverclean @@ -125,16 +121,19 @@ prebuildclean: @+python -c "import shutil; shutil.rmtree('build', True)" @+python -c "import shutil; shutil.rmtree('dist', True)" @+python -c "import shutil; shutil.rmtree('tqdm.egg-info', True)" + @+python -c "import shutil; shutil.rmtree('.eggs', True)" coverclean: @+python -c "import os; os.remove('.coverage') if os.path.exists('.coverage') else None" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('.coverage.*')]" @+python -c "import shutil; shutil.rmtree('tests/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('benchmarks/__pycache__', True)" @+python -c "import shutil; shutil.rmtree('tqdm/__pycache__', True)" @+python -c "import shutil; shutil.rmtree('tqdm/contrib/__pycache__', True)" @+python -c "import shutil; shutil.rmtree('tqdm/examples/__pycache__', True)" clean: @+python -c "import os, glob; [os.remove(i) for i in glob.glob('*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tests/*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('benchmarks/*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/contrib/*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/examples/*.py[co]')]" diff --git a/pyproject.toml b/pyproject.toml index abcd7baf7..c2f3f1b96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ [build-system] -requires = ["setuptools>=38.6.0"] +requires = ["setuptools>=38.6.0", "setuptools_scm"] build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "tqdm/_dist_ver.py" +write_to_template = "__version__ = '{version}'\n" diff --git a/requirements-dev.txt b/requirements-dev.txt index f7882f50e..f19dbe0ce 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ twine # pymake pypi argopt # cd wiki && pymake pydoc-markdown # cd docs && pymake wheel # setup.py bdist_wheel +setuptools_scm # version diff --git a/setup.cfg b/setup.cfg index 6fe318fb1..be184f1cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -90,7 +90,7 @@ classifiers = Topic :: Utilities [options] -setup_requires = setuptools>=38.6.0 +setup_requires = setuptools>=38.6.0; setuptools_scm python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* tests_require = pytest; flake8; coverage include_package_data = True diff --git a/setup.py b/setup.py index dc81a96ac..a30f79632 100755 --- a/setup.py +++ b/setup.py @@ -5,17 +5,9 @@ import os import sys -# Get version from tqdm/_version.py -__version__ = None src_dir = os.path.abspath(os.path.dirname(__file__)) -version_file = os.path.join(src_dir, 'tqdm', '_version.py') -with io_open(version_file, mode='r') as fd: - exec(fd.read()) - -# Executing makefile commands if specified -if sys.argv[1].lower().strip() == 'make': +if sys.argv[1].lower().strip() == 'make': # exec Makefile commands import pymake - # Filename of the makefile fpath = os.path.join(src_dir, 'Makefile') pymake.main(['-f', fpath] + sys.argv[2:]) # Stop to avoid setup.py raising non-standard command error @@ -27,4 +19,4 @@ extras_require['dev'] = [i.strip().split('#', 1)[0].strip() for i in fd.read().strip().split('\n')] -setup(version=__version__, extras_require=extras_require) +setup(use_scm_version=True, extras_require=extras_require) diff --git a/tests/tests_version.py b/tests/tests_version.py index 2232b1dad..ff93965b0 100644 --- a/tests/tests_version.py +++ b/tests/tests_version.py @@ -1,3 +1,5 @@ +"""Test `tqdm.__version__`.""" +from ast import literal_eval import re from .tests_tqdm import pretest_posttest # NOQA, pylint: disable=unused-import @@ -7,8 +9,8 @@ def test_version(): """Test version string""" from tqdm import __version__ version_parts = re.split('[.-]', __version__) - assert 3 <= len(version_parts) # must have at least Major.minor.patch - try: - map(int, version_parts[:3]) - except ValueError: - raise TypeError('Version Major.minor.patch must be 3 integers') + if __version__ != "UNKNOWN": + assert 3 <= len(version_parts), "must have at least Major.minor.patch" + assert all([isinstance(literal_eval(i), int) + for i in version_parts[:3]]), ( + "Version Major.minor.patch must be 3 integers") diff --git a/tqdm/__init__.py b/tqdm/__init__.py index 670d6457e..70601f417 100644 --- a/tqdm/__init__.py +++ b/tqdm/__init__.py @@ -4,11 +4,10 @@ from ._tqdm_pandas import tqdm_pandas from .cli import main # TODO: remove in v5.0.0 from ._monitor import TMonitor, TqdmSynchronisationWarning -from ._version import __version__ # NOQA +from .version import __version__ from .std import TqdmTypeError, TqdmKeyError, TqdmWarning, \ TqdmDeprecationWarning, TqdmExperimentalWarning, \ TqdmMonitorWarning - __all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas', 'tqdm_notebook', 'tnrange', 'main', 'TMonitor', 'TqdmTypeError', 'TqdmKeyError', diff --git a/tqdm/_version.py b/tqdm/_version.py deleted file mode 100644 index 50b7c800f..000000000 --- a/tqdm/_version.py +++ /dev/null @@ -1,62 +0,0 @@ -# Definition of the version number -import os -from io import open as io_open - -__all__ = ["__version__"] - -# major, minor, patch, -extra -version_info = 4, 52, 0 - -# Nice string for the version -__version__ = '.'.join(map(str, version_info)) - - -# auto -extra based on commit hash (if not tagged as release) -scriptdir = os.path.dirname(__file__) -gitdir = os.path.abspath(os.path.join(scriptdir, "..", ".git")) -if os.path.isdir(gitdir): # pragma: nocover - try: - extra = None - # Open config file to check if we are in tqdm project - with io_open(os.path.join(gitdir, "config"), 'r') as fh_config: - if 'tqdm' in fh_config.read(): - # Open the HEAD file - with io_open(os.path.join(gitdir, "HEAD"), 'r') as fh_head: - extra = fh_head.readline().strip() - # in a branch => HEAD points to file containing last commit - if 'ref:' in extra: - # reference file path - ref_file = extra[5:] - branch_name = ref_file.rsplit('/', 1)[-1] - - ref_file_path = os.path.abspath(os.path.join( - gitdir, ref_file)) - # check that we are in git folder - # (by stripping the git folder from the ref file path) - if os.path.relpath(ref_file_path, gitdir).replace( - '\\', '/') != ref_file: - # out of git folder - extra = None - else: - # open the ref file - with io_open(ref_file_path, 'r') as fh_branch: - commit_hash = fh_branch.readline().strip() - extra = commit_hash[:8] - if branch_name != "master": - extra += '.' + branch_name - - # detached HEAD mode, already have commit hash - else: - extra = extra[:8] - - # Append commit hash (and branch) to version string if not tagged - if extra is not None: - with io_open(os.path.join(gitdir, "refs", "tags", - 'v' + __version__)) as fdv: - if fdv.readline().strip()[:8] != extra[:8]: - __version__ += '-' + extra - except Exception as e: - if "No such file" in str(e): - __version__ += "-git.UNKNOWN" - else: - raise diff --git a/tqdm/cli.py b/tqdm/cli.py index 956e924c9..7f79b5f63 100644 --- a/tqdm/cli.py +++ b/tqdm/cli.py @@ -1,12 +1,13 @@ """ Module version for monitoring CLI pipes (`... | python -m tqdm | ...`). """ -from .std import tqdm, TqdmTypeError, TqdmKeyError -from ._version import __version__ # NOQA from ast import literal_eval as numeric import logging import re import sys + +from .std import tqdm, TqdmTypeError, TqdmKeyError +from .version import __version__ __all__ = ["main"] log = logging.getLogger(__name__) diff --git a/tqdm/version.py b/tqdm/version.py new file mode 100644 index 000000000..11cbaea79 --- /dev/null +++ b/tqdm/version.py @@ -0,0 +1,9 @@ +"""`tqdm` version detector. Precedence: installed dist, git, 'UNKNOWN'.""" +try: + from ._dist_ver import __version__ +except ImportError: + try: + from setuptools_scm import get_version + __version__ = get_version(root='..', relative_to=__file__) + except (ImportError, LookupError): + __version__ = "UNKNOWN" From e20d873a0338d0e8d6d3d5277107d291262c6223 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 17 Nov 2020 13:57:42 +0000 Subject: [PATCH 04/11] CI: add reviewdog linting --- .github/workflows/check.yml | 20 ++++++++++++++++++-- Makefile | 2 +- tox.ini | 6 ++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6cc0ec205..fa9a0313a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -6,11 +6,11 @@ on: - cron: '36 1 * * SUN' # M H d m w (Sundays at 01:36) jobs: check: - name: '${{ matrix.TOXENV }}' + name: ${{ matrix.TOXENV }} strategy: matrix: TOXENV: - - flake8,setup.py + - setup.py - perf runs-on: ubuntu-latest steps: @@ -24,6 +24,22 @@ jobs: - run: tox env: TOXENV: ${{ matrix.TOXENV }} + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - uses: reviewdog/action-setup@v1 + - run: pip install -U flake8 + - name: flake8 + run: | + set -o pipefail + flake8 -j8 --count --statistics . | \ + reviewdog -f=pep8 -name=flake8 -tee -reporter=github-check -filter-mode nofilter + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} asvfull: if: (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')) || github.event_name == 'schedule' name: Benchmark (Full) diff --git a/Makefile b/Makefile index ff9d2e0e8..dd1960864 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ distclean: pre-commit: # quick sanity checks @make --no-print-directory testsetup - flake8 -j 8 --count --statistics tqdm/ tests/ examples/ + flake8 -j 8 --count --statistics setup.py tqdm/ tests/ examples/ pytest -qq -k "basic_overhead or not (perf or keras or pandas or monitoring)" prebuildclean: @+python -c "import shutil; shutil.rmtree('build', True)" diff --git a/tox.ini b/tox.ini index eacee6d24..01cfb9571 100644 --- a/tox.ini +++ b/tox.ini @@ -60,10 +60,8 @@ deps = {[extra]deps} deps = {[extra]deps} [testenv:perf] -deps = - pytest -commands = - pytest --durations=0 -k tests_perf +deps = pytest +commands = pytest --durations=0 -k tests_perf [testenv:flake8] deps = flake8 From 8788ad2545c589a973062b8962a0dc63c1bed6f1 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 17 Nov 2020 14:22:57 +0000 Subject: [PATCH 05/11] tests: misc tidy --- .github/workflows/test.yml | 21 ++++++++++++++++----- tox.ini | 13 ++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c3bfbe8b..b43b46e3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,8 +30,16 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - - run: pip install -U tox + - name: install + shell: bash + run: | + pip install -U tox + mkdir -p "$HOME/bin" + curl -sfL https://coverage.codacy.com/get.sh > "$HOME/bin/codacy" + chmod +x "$HOME/bin/codacy" + echo "$HOME/bin" >> $GITHUB_PATH - run: tox -e py${PYVER/./} + shell: bash env: PYVER: ${{ matrix.python }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -49,7 +57,13 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - - run: pip install -U tox + - name: install + run: | + pip install -U tox + mkdir -p "$HOME/bin" + curl -sfL https://coverage.codacy.com/get.sh > "$HOME/bin/codacy" + chmod +x "$HOME/bin/codacy" + echo "$HOME/bin" >> $GITHUB_PATH - name: tox run: | if [[ "$PYVER" == py* ]]; then @@ -104,7 +118,6 @@ jobs: skip_existing: true - id: collect_assets name: Collect assets - shell: bash run: | echo "::set-output name=asset_path::$(ls dist/*.whl)" echo "::set-output name=asset_name::$(basename dist/*.whl)" @@ -154,7 +167,6 @@ jobs: with: use_lxd: true - name: Snap build - shell: bash run: | export SNAPCRAFT_IMAGE_INFO='{"build_url": "https://github.com/tqdm/tqdm/actions/runs/'$GITHUB_RUN_ID'"}' sg lxd -c 'snapcraft --use-lxd' @@ -168,7 +180,6 @@ jobs: snapcraft_token: ${{ secrets.SNAP_TOKEN }} - if: github.event_name == 'push' && steps.collect_assets.outputs.snap_channel name: Snap deploy - shell: bash run: | if [ -n "$(ls tqdm*.snap 2>/dev/null)" ]; then sudo snapcraft upload tqdm*.snap --release $CHANNEL diff --git a/tox.ini b/tox.ini index 01cfb9571..f65c1ee77 100644 --- a/tox.ini +++ b/tox.ini @@ -17,11 +17,8 @@ deps = commands = pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not tests_perf" - coveralls - - curl -OL https://coverage.codacy.com/get.sh - - bash get.sh report -r coverage.xml -allowlist_externals = - curl - bash + - codacy report -r coverage.xml +allowlist_externals = codacy [extra] deps = @@ -31,8 +28,7 @@ commands = pytest --durations=10 --cov=tqdm --cov-report=xml --cov-report=term -k "not tests_perf" - coveralls codecov - - curl -OL https://coverage.codacy.com/get.sh - - bash get.sh report -r coverage.xml + - codacy report -r coverage.xml allowlist_externals = {[coverage]allowlist_externals} [testenv] @@ -65,8 +61,7 @@ commands = pytest --durations=0 -k tests_perf [testenv:flake8] deps = flake8 -commands = - flake8 -j 8 --count --statistics . +commands = flake8 -j 8 --count --statistics . [testenv:setup.py] deps = From abffb34e473dafc6d6947555a676aed8dbd0c7f9 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 17 Nov 2020 14:57:54 +0000 Subject: [PATCH 06/11] CI: don't test devel PR - vis https://github.community/t/how-to-ignore-branch-names-on-pull-requests/16627 --- .github/workflows/check.yml | 2 ++ .github/workflows/test.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fa9a0313a..d3d0a0a61 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -6,6 +6,7 @@ on: - cron: '36 1 * * SUN' # M H d m w (Sundays at 01:36) jobs: check: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' name: ${{ matrix.TOXENV }} strategy: matrix: @@ -25,6 +26,7 @@ jobs: env: TOXENV: ${{ matrix.TOXENV }} lint: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b43b46e3d..99f7d1e23 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: python-version: '3.x' - run: pip install . test-os: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' strategy: matrix: python: [2.7, 3.7] @@ -45,6 +46,7 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} test: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' strategy: matrix: python: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy3] @@ -86,6 +88,7 @@ jobs: with: parallel: true finish: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' name: pytest cov continue-on-error: ${{ github.event_name != 'push' }} needs: [test, test-os] @@ -96,6 +99,7 @@ jobs: with: parallel-finished: true deploy: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' needs: [check, test, test-os] runs-on: ubuntu-latest steps: From b80490e12cc97bf3cbf7735bbb6d883f68eccda5 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 18 Nov 2020 22:22:32 +0000 Subject: [PATCH 07/11] more sanitising --- .github/workflows/test.yml | 1 + .meta/mksnap.py | 11 +---------- Makefile | 6 +++--- pyproject.toml | 2 +- requirements-dev.txt | 1 - setup.cfg | 4 ++-- 6 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99f7d1e23..4cd02aacb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,7 @@ on: - cron: '2 1 * * 6' # M H d m w (Saturdays at 1:02) jobs: check: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' name: install runs-on: ubuntu-latest steps: diff --git a/.meta/mksnap.py b/.meta/mksnap.py index 5feef0b9f..337e46b8c 100644 --- a/.meta/mksnap.py +++ b/.meta/mksnap.py @@ -2,22 +2,15 @@ """ Auto-generate snapcraft.yaml. """ -from __future__ import print_function -from configparser import ConfigParser from io import open as io_open from os import path from subprocess import check_output -import re import sys sys.path.insert(1, path.dirname(path.dirname(__file__))) import tqdm # NOQA src_dir = path.abspath(path.dirname(__file__)) -cfg = ConfigParser() -cfg.read(path.join(path.dirname(src_dir), 'setup.cfg')) -setup_requires = re.split( - r"[;\s]+", str(cfg["options"].get("setup_requires")), flags=re.M) snap_yml = r"""name: tqdm summary: A fast, extensible CLI progress bar description: | @@ -60,8 +53,7 @@ parts: tqdm: plugin: python - python-packages: {setup_requires} - python-version: python3 + python-packages: [disco-py] source: . source-commit: '{commit}' build-packages: [git] @@ -74,7 +66,6 @@ completer: completion.sh """.format( version=tqdm.__version__, - setup_requires=repr(setup_requires), commit=check_output(['git', 'describe', '--always']).decode('U8').strip()) fname = path.join(path.dirname(src_dir), 'snapcraft.yaml') diff --git a/Makefile b/Makefile index dd1960864..82f239da2 100644 --- a/Makefile +++ b/Makefile @@ -115,13 +115,14 @@ distclean: pre-commit: # quick sanity checks @make --no-print-directory testsetup - flake8 -j 8 --count --statistics setup.py tqdm/ tests/ examples/ + flake8 -j 8 --count --statistics setup.py .meta/ tqdm/ tests/ examples/ pytest -qq -k "basic_overhead or not (perf or keras or pandas or monitoring)" prebuildclean: @+python -c "import shutil; shutil.rmtree('build', True)" @+python -c "import shutil; shutil.rmtree('dist', True)" @+python -c "import shutil; shutil.rmtree('tqdm.egg-info', True)" @+python -c "import shutil; shutil.rmtree('.eggs', True)" + @+python -c "import os; os.remove('tqdm/_dist_ver.py') if os.path.exists('tqdm/_dist_ver.py') else None" coverclean: @+python -c "import os; os.remove('.coverage') if os.path.exists('.coverage') else None" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('.coverage.*')]" @@ -170,9 +171,8 @@ snap: @make -B snapcraft.yaml snapcraft docker: + @make build @make .dockerignore - @make coverclean - @make clean docker build . -t tqdm/tqdm docker tag tqdm/tqdm:latest tqdm/tqdm:$(shell docker run -i --rm tqdm/tqdm -v) none: diff --git a/pyproject.toml b/pyproject.toml index c2f3f1b96..3eb7bbcf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=38.6.0", "setuptools_scm"] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] diff --git a/requirements-dev.txt b/requirements-dev.txt index f19dbe0ce..f7882f50e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,3 @@ twine # pymake pypi argopt # cd wiki && pymake pydoc-markdown # cd docs && pymake wheel # setup.py bdist_wheel -setuptools_scm # version diff --git a/setup.cfg b/setup.cfg index be184f1cb..79f44adfd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ universal = 1 [flake8] ignore = W503,W504,E722 max_line_length = 80 -exclude = .asv,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ +exclude = .asv,.eggs,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ [tool:pytest] python_files = tests_*.py @@ -90,7 +90,7 @@ classifiers = Topic :: Utilities [options] -setup_requires = setuptools>=38.6.0; setuptools_scm +setup_requires = setuptools>=42; setuptools_scm[toml]>=3.4 python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* tests_require = pytest; flake8; coverage include_package_data = True From ee5f8d21e88b59fe89abfcbc8762f66b2d648ade Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 20 Nov 2020 23:27:22 +0000 Subject: [PATCH 08/11] fix nested `asyncio`, document async `break` hazard - fixes #1074 --- .meta/.readme.rst | 25 +++++++++++++++++++++++++ README.rst | 25 +++++++++++++++++++++++++ tests/py37_asyncio.py | 21 +++++++++++---------- tqdm/asyncio.py | 20 ++++++++++++++++++++ 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/.meta/.readme.rst b/.meta/.readme.rst index b9852c430..5f9e0f4fd 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -744,6 +744,31 @@ custom callback take advantage of this, simply use the return value of external_callback(**self.format_dict) return displayed +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + Pandas Integration ~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index f54b42f5c..0284e60bf 100644 --- a/README.rst +++ b/README.rst @@ -961,6 +961,31 @@ custom callback take advantage of this, simply use the return value of external_callback(**self.format_dict) return displayed +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + Pandas Integration ~~~~~~~~~~~~~~~~~~ diff --git a/tests/py37_asyncio.py b/tests/py37_asyncio.py index 1f079645b..cd9389d7f 100644 --- a/tests/py37_asyncio.py +++ b/tests/py37_asyncio.py @@ -14,8 +14,8 @@ def with_setup_sync(func): @wraps(func) - def inner(): - return asyncio.run(func()) + def inner(*args, **kwargs): + return asyncio.run(func(*args, **kwargs)) return inner @@ -35,20 +35,21 @@ async def acount(*args, **kwargs): @with_setup_sync -async def test_generators(): +async def test_generators(capsys): """Test asyncio generators""" - with closing(StringIO()) as our_file: - async for i in tqdm(count(), desc="counter", file=our_file): + with tqdm(count(), desc="counter") as pbar: + async for i in pbar: if i >= 8: break - assert '9it' in our_file.getvalue() - our_file.seek(0) - our_file.truncate() + _, err = capsys.readouterr() + assert '9it' in err - async for i in tqdm(acount(), desc="async_counter", file=our_file): + with tqdm(acount(), desc="async_counter") as pbar: + async for i in pbar: if i >= 8: break - assert '9it' in our_file.getvalue() + _, err = capsys.readouterr() + assert '9it' in err @with_setup_sync diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index d7f08a991..e3e2d2760 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -62,6 +62,26 @@ def as_completed(cls, fs, *, loop=None, timeout=None, total=None, yield from cls(asyncio.as_completed(fs, loop=loop, timeout=timeout), total=total, **tqdm_kwargs) + def __new__(cls, *args, **kwargs): + """ + Workaround for mixed-class same-stream nested progressbars. + See [#509](https://github.com/tqdm/tqdm/issues/509) + """ + with cls.get_lock(): + try: + cls._instances = std_tqdm._instances + except AttributeError: + pass + instance = super(tqdm_asyncio, cls).__new__(cls, *args, **kwargs) + with cls.get_lock(): + try: + # `std_tqdm` may have been changed so update + cls._instances.update(std_tqdm._instances) + except AttributeError: + pass + std_tqdm._instances = cls._instances + return instance + def tarange(*args, **kwargs): """ From a35529157dea73566742a14985b339527e5373bc Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 21 Nov 2020 00:49:07 +0000 Subject: [PATCH 09/11] use pytest-asyncio --- tests/py37_asyncio.py | 47 +++++++++++++++++++++++++------------------ tox.ini | 2 +- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/tests/py37_asyncio.py b/tests/py37_asyncio.py index cd9389d7f..603b63832 100644 --- a/tests/py37_asyncio.py +++ b/tests/py37_asyncio.py @@ -1,24 +1,18 @@ -from functools import partial, wraps +from functools import partial from time import time import asyncio +from pytest import mark + from tqdm.asyncio import tqdm_asyncio, tarange from .tests_tqdm import pretest_posttest # NOQA, pylint: disable=unused-import from .tests_tqdm import StringIO, closing -from .tests_perf import retry_on_except tqdm = partial(tqdm_asyncio, miniters=0, mininterval=0) trange = partial(tarange, miniters=0, mininterval=0) as_completed = partial(tqdm_asyncio.as_completed, miniters=0, mininterval=0) -def with_setup_sync(func): - @wraps(func) - def inner(*args, **kwargs): - return asyncio.run(func(*args, **kwargs)) - return inner - - def count(start=0, step=1): i = start while True: @@ -34,7 +28,16 @@ async def acount(*args, **kwargs): yield i -@with_setup_sync +@mark.asyncio +async def test_break(): + """Test asyncio break""" + pbar = tqdm(count()) + async for _ in pbar: + break + pbar.close() + + +@mark.asyncio async def test_generators(capsys): """Test asyncio generators""" with tqdm(count(), desc="counter") as pbar: @@ -52,7 +55,7 @@ async def test_generators(capsys): assert '9it' in err -@with_setup_sync +@mark.asyncio async def test_range(): """Test asyncio range""" with closing(StringIO()) as our_file: @@ -67,7 +70,7 @@ async def test_range(): assert '9/9' in our_file.getvalue() -@with_setup_sync +@mark.asyncio async def test_nested(): """Test asyncio nested""" with closing(StringIO()) as our_file: @@ -78,7 +81,7 @@ async def test_nested(): assert 'outer: 100%' in our_file.getvalue() -@with_setup_sync +@mark.asyncio async def test_coroutines(): """Test asyncio coroutine.send""" with closing(StringIO()) as our_file: @@ -92,16 +95,20 @@ async def test_coroutines(): assert '10it' in our_file.getvalue() -@retry_on_except(check_cpu_time=False) -@with_setup_sync -async def test_as_completed(): +@mark.asyncio +async def test_as_completed(capsys): """Test asyncio as_completed""" - with closing(StringIO()) as our_file: + for retry in range(3): t = time() skew = time() - t for i in as_completed([asyncio.sleep(0.01 * i) - for i in range(30, 0, -1)], file=our_file): + for i in range(30, 0, -1)]): await i t = time() - t - 2 * skew - assert 0.27 < t < 0.33, t - assert '30/30' in our_file.getvalue() + try: + assert 0.27 < t < 0.33, t + _, err = capsys.readouterr() + assert '30/30' in err + except AssertionError: + if retry == 2: + raise diff --git a/tox.ini b/tox.ini index f65c1ee77..7cc43c56c 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ isolated_build = True deps = pytest pytest-cov + py{37,38,39}: pytest-asyncio coverage coveralls commands = @@ -52,7 +53,6 @@ deps = {[extra]deps} deps = {[extra]deps} [testenv:py34] -# py34-compatible pandas deps = {[extra]deps} [testenv:perf] From 2ddae6b23130614e5d76c1886f1be7935700a7f1 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 21 Nov 2020 01:52:47 +0000 Subject: [PATCH 10/11] abstract and document `get_new` - fixes #509 --- .meta/.readme.rst | 21 ++++++++++++++++++++- README.rst | 21 ++++++++++++++++++++- tqdm/asyncio.py | 19 +------------------ tqdm/contrib/discord.py | 19 +------------------ tqdm/contrib/telegram.py | 20 ++------------------ tqdm/std.py | 21 +++++++++++++++++++++ 6 files changed, 65 insertions(+), 56 deletions(-) diff --git a/.meta/.readme.rst b/.meta/.readme.rst index 5f9e0f4fd..12e46a61c 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -918,7 +918,9 @@ For further customisation, (e.g. GUIs such as notebook or plotting packages). In the latter case: 1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable - terminal ``status_printer`` creation. + terminal ``status_printer`` creation. Otherwise (if terminal is required), + ``def __new__()`` to call ``cls.get_new()`` (see below) to ensure correct + nested positioning. 2. Redefine: ``close()``, ``clear()``, ``display()``. Consider overloading ``display()`` to use e.g. @@ -932,6 +934,23 @@ above recommendation: - `tqdm/contrib/telegram.py `__ - `tqdm/contrib/discord.py `__ +Note that multiple different ``tqdm`` subclasses which all write to the terminal +(``gui=False``) can cause positioning issues when used simultaneously (in nested +mode). To fix this, custom subclasses which expect to write to the terminal +should define a ``__new__()`` method as follows: + +.. code:: python + + from tqdm import tqdm as std_tqdm + + class TqdmExt(std_tqdm): + def __new__(cls, *args, **kwargs): + return cls.get_new(super(TqdmExt, cls), std_tqdm, *args, **kwargs) + +This approach is used ``tqdm.asyncio`` and ``tqdm.contrib.telegram/discord``. +However it is not necessary for ``tqdm.notebook/gui`` since they don't use the +terminal. + Dynamic Monitor/Meter ~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index 0284e60bf..07eab6566 100644 --- a/README.rst +++ b/README.rst @@ -1135,7 +1135,9 @@ For further customisation, (e.g. GUIs such as notebook or plotting packages). In the latter case: 1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable - terminal ``status_printer`` creation. + terminal ``status_printer`` creation. Otherwise (if terminal is required), + ``def __new__()`` to call ``cls.get_new()`` (see below) to ensure correct + nested positioning. 2. Redefine: ``close()``, ``clear()``, ``display()``. Consider overloading ``display()`` to use e.g. @@ -1149,6 +1151,23 @@ above recommendation: - `tqdm/contrib/telegram.py `__ - `tqdm/contrib/discord.py `__ +Note that multiple different ``tqdm`` subclasses which all write to the terminal +(``gui=False``) can cause positioning issues when used simultaneously (in nested +mode). To fix this, custom subclasses which expect to write to the terminal +should define a ``__new__()`` method as follows: + +.. code:: python + + from tqdm import tqdm as std_tqdm + + class TqdmExt(std_tqdm): + def __new__(cls, *args, **kwargs): + return cls.get_new(super(TqdmExt, cls), std_tqdm, *args, **kwargs) + +This approach is used ``tqdm.asyncio`` and ``tqdm.contrib.telegram/discord``. +However it is not necessary for ``tqdm.notebook/gui`` since they don't use the +terminal. + Dynamic Monitor/Meter ~~~~~~~~~~~~~~~~~~~~~ diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py index e3e2d2760..568fe0c82 100644 --- a/tqdm/asyncio.py +++ b/tqdm/asyncio.py @@ -63,24 +63,7 @@ def as_completed(cls, fs, *, loop=None, timeout=None, total=None, total=total, **tqdm_kwargs) def __new__(cls, *args, **kwargs): - """ - Workaround for mixed-class same-stream nested progressbars. - See [#509](https://github.com/tqdm/tqdm/issues/509) - """ - with cls.get_lock(): - try: - cls._instances = std_tqdm._instances - except AttributeError: - pass - instance = super(tqdm_asyncio, cls).__new__(cls, *args, **kwargs) - with cls.get_lock(): - try: - # `std_tqdm` may have been changed so update - cls._instances.update(std_tqdm._instances) - except AttributeError: - pass - std_tqdm._instances = cls._instances - return instance + return cls.get_new(super(tqdm_asyncio, cls), std_tqdm, *args, **kwargs) def tarange(*args, **kwargs): diff --git a/tqdm/contrib/discord.py b/tqdm/contrib/discord.py index 9c0763aa4..9212a595b 100644 --- a/tqdm/contrib/discord.py +++ b/tqdm/contrib/discord.py @@ -103,24 +103,7 @@ def display(self, **kwargs): self.dio.write(self.format_meter(**fmt)) def __new__(cls, *args, **kwargs): - """ - Workaround for mixed-class same-stream nested progressbars. - See [#509](https://github.com/tqdm/tqdm/issues/509) - """ - with cls.get_lock(): - try: - cls._instances = tqdm_auto._instances - except AttributeError: - pass - instance = super(tqdm_discord, cls).__new__(cls, *args, **kwargs) - with cls.get_lock(): - try: - # `tqdm_auto` may have been changed so update - cls._instances.update(tqdm_auto._instances) - except AttributeError: - pass - tqdm_auto._instances = cls._instances - return instance + return cls.get_new(super(tqdm_discord, cls), tqdm_auto, *args, **kwargs) def tdrange(*args, **kwargs): diff --git a/tqdm/contrib/telegram.py b/tqdm/contrib/telegram.py index 1e1da6db5..ccdbe5a50 100644 --- a/tqdm/contrib/telegram.py +++ b/tqdm/contrib/telegram.py @@ -107,24 +107,8 @@ def display(self, **kwargs): self.tgio.write(self.format_meter(**fmt)) def __new__(cls, *args, **kwargs): - """ - Workaround for mixed-class same-stream nested progressbars. - See [#509](https://github.com/tqdm/tqdm/issues/509) - """ - with cls.get_lock(): - try: - cls._instances = tqdm_auto._instances - except AttributeError: - pass - instance = super(tqdm_telegram, cls).__new__(cls, *args, **kwargs) - with cls.get_lock(): - try: - # `tqdm_auto` may have been changed so update - cls._instances.update(tqdm_auto._instances) - except AttributeError: - pass - tqdm_auto._instances = cls._instances - return instance + return cls.get_new( + super(tqdm_telegram, cls), tqdm_auto, *args, **kwargs) def ttgrange(*args, **kwargs): diff --git a/tqdm/std.py b/tqdm/std.py index f525dea84..407e263a2 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -575,6 +575,27 @@ def __new__(cls, *_, **__): # Return the instance return instance + @classmethod + def get_new(cls, super_cls, base_cls, *args, **kwargs): + """ + Workaround for mixed-class same-stream nested progressbars. + See [#509](https://github.com/tqdm/tqdm/issues/509) + """ + with cls.get_lock(): + try: + cls._instances = base_cls._instances + except AttributeError: + pass + instance = super_cls.__new__(cls, *args, **kwargs) + with cls.get_lock(): + try: + # `base_cls` may have been changed so update + cls._instances.update(base_cls._instances) + except AttributeError: + pass + base_cls._instances = cls._instances + return instance + @classmethod def _get_free_pos(cls, instance=None): """Skips specified instance.""" From 04c567dec1b5a82a887ce1e6d66593363d5c08e7 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 21 Nov 2020 15:58:44 +0000 Subject: [PATCH 11/11] fix typos --- .meta/.readme.rst | 5 +++-- README.rst | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.meta/.readme.rst b/.meta/.readme.rst index 12e46a61c..23c4ec368 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -111,7 +111,7 @@ There are 3 channels to choose from: snap install tqdm --candidate # master branch snap install tqdm --edge # devel branch -Note than ``snap`` binaries are purely for CLI use (not ``import``-able), and +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and automatically set up ``bash`` tab-completion. Latest Docker release @@ -577,7 +577,8 @@ available to keep nested bars on their respective lines. For manual control over positioning (e.g. for multi-processing use), you may specify ``position=n`` where ``n=0`` for the outermost bar, ``n=1`` for the next, and so on. -However, it's best to check if `tqdm` can work without manual `position` first. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. .. code:: python diff --git a/README.rst b/README.rst index 07eab6566..9c87bade4 100644 --- a/README.rst +++ b/README.rst @@ -111,7 +111,7 @@ There are 3 channels to choose from: snap install tqdm --candidate # master branch snap install tqdm --edge # devel branch -Note than ``snap`` binaries are purely for CLI use (not ``import``-able), and +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and automatically set up ``bash`` tab-completion. Latest Docker release @@ -794,7 +794,8 @@ available to keep nested bars on their respective lines. For manual control over positioning (e.g. for multi-processing use), you may specify ``position=n`` where ``n=0`` for the outermost bar, ``n=1`` for the next, and so on. -However, it's best to check if `tqdm` can work without manual `position` first. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. .. code:: python