diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 17afdd3a..660db2ba 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [ '3.7', '3.8', '3.9', '3.10' ] + python: [ '3.9', '3.10', '3.11'] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it diff --git a/.github/workflows/pypi_packaging.yml b/.github/workflows/pypi_packaging.yml index 8e40f725..e30e1cc8 100644 --- a/.github/workflows/pypi_packaging.yml +++ b/.github/workflows/pypi_packaging.yml @@ -80,9 +80,6 @@ jobs: - name: Setup Env run: python3 -m pip install --upgrade pip build - - name: Validate setup.py - run: python3 setup.py check --metadata - - name: Build run: python3 -m build diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7f92b04f..11d1591c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [ '3.7', '3.8', '3.9', '3.10' ] + python: [ '3.9', '3.10', '3.11'] # Steps represent a sequence of tasks that will be executed as part of the job diff --git a/README.md b/README.md index 72f5d42a..810b7b49 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,19 @@ Cromshell is a CLI for submitting workflows to a Cromwell server and monitoring/ * Requires the `bq_cost_table` key to exist in the cromshell configuration file and have a value equal to the BigQuery cost table for your GCP billing project. + * For example, your `~/.cromshell/cromshell_config.json` should contain: + ```json + { + "cromwell_server": "", + "requests_timeout": 5, + "bq_cost_table": "" + } + ``` + where `` can be found by navigating to [BigQuery](https://console.cloud.google.com/bigquery), selecting the appropriate google project, and locating the table containing cost information. + + BigQuery example image + + Clicking on the table and opening the "DETAILS" tab, you'll find the exact path to the table in the "Table ID" section. Everything after the google project name (after the first `.`) should be included in ``. * `-c/--color` Color outliers in task level cost results. * `-d/--detailed` Get the cost for a workflow at the task level. diff --git a/dev-requirements.txt b/dev-requirements.txt index d11262ad..547f7ab1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -isort==4.3.21 +isort==5.13.2 tox wheel bumpversion @@ -6,8 +6,8 @@ twine black flake8 pylint +tomli==2.0.1 cython sphinx sphinx-rtd-theme -mypy==0.770 --r requirements.txt \ No newline at end of file +mypy==1.8.0 diff --git a/developer_docs/img/bq_cost_table.png b/developer_docs/img/bq_cost_table.png new file mode 100644 index 00000000..ef92490a Binary files /dev/null and b/developer_docs/img/bq_cost_table.png differ diff --git a/developer_docs/runtests.md b/developer_docs/runtests.md index 4a6c8a39..c1fa4a70 100644 --- a/developer_docs/runtests.md +++ b/developer_docs/runtests.md @@ -14,13 +14,13 @@ as seen below tox -e lint # reformat all project files -black src tests setup.py +black src tests # sort imports in project files -isort -rc src tests setup.py +isort -rc src tests # check pep8 against all project files -flake8 src tests setup.py +flake8 src tests # lint python code for common errors and codestyle issues pylint src diff --git a/pyproject.toml b/pyproject.toml index 2f2d20b8..b958c51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,4 @@ + [build-system] requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" @@ -7,8 +8,53 @@ name = "cromshell" # Version number is automatically set via bumpversion. DO NOT MODIFY: version = "2.1.1" readme = "README.md" - +description="Command Line Interface (CLI) for Cromwell servers" +authors=[ + {name = "Jonn Smith", email = "jonn@broadinstitute.org"}, + {name = "Louis Bergelson", email = "louisb@broadinstitute.org"}, + {name = "Beri Shifaw", email = "bshifaw@broadinstitute.org"}, +] +license={text = "BSD 3-Clause"} +requires-python=">=3.9" +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + ] +keywords = [ + "cromwell", "cromshell", "cromwell-cli", "cromwell-client", "cromwell-api", + "cromwell-rest", "cromwell-utilities", "cromwell-tools", +] +dynamic = ["dependencies", "optional-dependencies"] [project.urls] "Homepage" = "https://github.com/broadinstitute/cromshell" "Bug Tracker" = "https://github.com/broadinstitute/cromshell/issues" + +[project.scripts] +cromshell = "cromshell.__main__:main_entry" + +# Configuration for the `setuptools` package +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +optional-dependencies = {dev = { file = ["dev-requirements.txt"]}, tests = { file = ["test-requirements.txt"]}} + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cromshell*"] + +# former setup.cfg configuration for mypy +[tool.mypy."numpy.*"] +ignore_missing_imports = true + +[tool.mypy."pysam.*"] +ignore_missing_imports = true + +[tool.mypy."pytest.*"] +ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 987a17fc..00000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[mypy-numpy.*] -ignore_missing_imports = True - -[mypy-pysam.*] -ignore_missing_imports = True - -[mypy-pytest.*] -ignore_missing_imports = True diff --git a/setup.py b/setup.py deleted file mode 100644 index c4407e65..00000000 --- a/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -from setuptools import find_packages, setup - -with open("requirements.txt") as fh: - install_requires = fh.readlines() - -# PIPY needs a list of required packages from the test-requirements list. One of the -# lines in this file is a reference to the requirements.txt file, because PYPI will -# assume it's a package and fail; the line below will remove the requirements.txt line -# and add the extracted packages from that file to the list of required test packages. -with open("test-requirements.txt") as fh: - test_install_requires = fh.readlines() - test_install_requires.remove("-r requirements.txt") - test_install_requires.extend(install_requires) - -with open("README.md") as fh: - long_description = fh.read() - -# following src dir layout according to -# https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure - -# Version number is automatically set via bumpversion. DO NOT MODIFY: -version = "2.1.1" -setup( - name="cromshell", - version=version, - description="Command Line Interface (CLI) for Cromwell servers", - author="Jonn Smith, Louis Bergelson, Beri Shifaw", - author_email="jonn@broadinstitute.org, louisb@broadinstitute.org, " - "bshifaw@broadinstitute.org", - url="https://github.com/broadinstitute/cromshell", - license="BSD 3-Clause", - long_description=long_description, - install_requires=install_requires, - tests_require=test_install_requires, - python_requires=">=3.7", - packages=find_packages("src"), - package_dir={"": "src"}, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - ], - entry_points={"console_scripts": ["cromshell=cromshell.__main__:main_entry"]}, - include_package_data=True, -) diff --git a/src/cromshell/cost/command.py b/src/cromshell/cost/command.py index 2931fe05..3d459ac8 100644 --- a/src/cromshell/cost/command.py +++ b/src/cromshell/cost/command.py @@ -182,7 +182,7 @@ def create_bq_query(detailed: bool, bq_cost_table: str) -> str: AND task.key LIKE "wdl-task-name" AND wfid.key LIKE "cromwell-workflow-id" AND wfid.value like @workflow_id - AND partition_time BETWEEN @start_date AND @end_date + AND export_time BETWEEN @start_date AND @end_date GROUP BY 1,2,3 ORDER BY 4 DESC """ @@ -190,7 +190,7 @@ def create_bq_query(detailed: bool, bq_cost_table: str) -> str: return f""" SELECT sum(cost) as cost FROM {bq_cost_table}, UNNEST(labels) - WHERE value LIKE @workflow_id AND partition_time BETWEEN @start_date AND @end_date + WHERE value LIKE @workflow_id AND export_time BETWEEN @start_date AND @end_date """ diff --git a/test-requirements.txt b/test-requirements.txt index c769a376..f533e277 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,4 +2,3 @@ pytest pytest-cov pytest-dependency coverage>=4.5 --r requirements.txt \ No newline at end of file diff --git a/tests/unit/test_cost.py b/tests/unit/test_cost.py index 6fdb787e..349d0ee5 100644 --- a/tests/unit/test_cost.py +++ b/tests/unit/test_cost.py @@ -23,7 +23,7 @@ class TestCost: AND task.key LIKE "wdl-task-name" AND wfid.key LIKE "cromwell-workflow-id" AND wfid.value like @workflow_id - AND partition_time BETWEEN @start_date AND @end_date + AND export_time BETWEEN @start_date AND @end_date GROUP BY 1,2,3 ORDER BY 4 DESC """, @@ -34,7 +34,7 @@ class TestCost: """ SELECT sum(cost) as cost FROM cost:table1, UNNEST(labels) - WHERE value LIKE @workflow_id AND partition_time BETWEEN @start_date AND @end_date + WHERE value LIKE @workflow_id AND export_time BETWEEN @start_date AND @end_date """, ], ], diff --git a/tox.ini b/tox.ini index e3085d67..064c8797 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = lint,py37,py38,py39,py310 +envlist = lint,py39,py310,py311 skip_missing_interpreters = true @@ -15,6 +15,7 @@ ignore = E203, E501, W503 [testenv] deps = -rtest-requirements.txt + -rrequirements.txt commands = pytest tests/unit tests/integration tests/acceptance \ --cov cromshell \ @@ -28,16 +29,19 @@ setenv = [testenv:unit] deps = -rtest-requirements.txt + -rrequirements.txt commands = pytest tests/unit {posargs} [testenv:integration] deps = -rtest-requirements.txt + -rrequirements.txt commands = pytest tests/integration {posargs} [testenv:acceptance] deps = -rtest-requirements.txt + -rrequirements.txt commands = pytest tests/acceptance {posargs} @@ -49,9 +53,9 @@ deps = flake8 commands = - black --check --diff --target-version py38 src tests setup.py - isort --check-only --profile black --diff -rc tests src setup.py - flake8 src tests setup.py + black --check --diff --target-version py38 src tests + isort --check-only --profile black --diff tests src + flake8 src tests pylint --exit-zero src [testenv:lint-edit] @@ -61,14 +65,7 @@ deps = flake8 commands = - black --target-version py38 src tests setup.py - isort --profile black -rc tests src setup.py - flake8 src tests setup.py + black --target-version py38 src tests + isort --profile black tests src + flake8 src tests pylint --exit-zero src - -[gh-actions] -python = - 3.7: py37 - 3.8: py38 - 3.9: py39 - 3.10: py310