Merge pull request twisted#1543 from twisted/10121-pypy-no-coverage

Author: adiroiban Reviewer: wiml, altendky Fixes: ticket:10121 Fix random PyPy CI failures. Improve GHA CI speed. Improve coverage reporting.
debugmm · Apr 5, 2021 · b3cf4b5 · b3cf4b5
2 parents 4478372 + a371b9c
commit b3cf4b5
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 26 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -21,7 +21,9 @@ jobs:
     runs-on: ubuntu-20.04
     env:
       TOXENV: "${{ matrix.tox-env }}"
-      TRIAL_ARGS: "${{ matrix.trial-args }}"
+      # As of April 2021 GHA VM have 2 CPUs - Azure Standard_DS2_v2
+      # Trial distributed jobs enabled to speed up the CI jobs.
+      TRIAL_ARGS: "-j 4"
     name: ${{ matrix.python-version }}${{ matrix.noipv6 }}-${{ matrix.tox-env }}
     strategy:
       fail-fast: false
@@ -32,10 +34,10 @@ jobs:
         # Run on latest minor release of each major python version.
         python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-alpha.5]
         tox-env: ['alldeps-withcov-posix']
+        # By default, tests are executed without disabling IPv6.
         noipv6: ['']
-        # As of 9 March 2021 GHA VM have 2 CPUs - Azure Standard_DS2_v2
-        # Trial distributed jobs enabled to speed up the CI jobs.
-        trial-args: ['-j 4']
+        # Tests are executed with the default target which is the full test suite.
+        trial-target: ['']
 
         include:
 
@@ -47,22 +49,24 @@ jobs:
           # end to end functional test usage for non-distributed trial runs.
           - python-version: 3.6.7
             tox-env: nodeps-withcov-posix
-            trial-args: '-j 4'
           # `noipv6` is created to make sure all is OK on an OS which doesn't
           # have IPv6 available.
           # Any supported Python version is OK for this job.
           - python-version: 3.6
             tox-env: alldeps-withcov-posix
             noipv6: -noipv6
-            trial-args: '-j 4'
-          # On PYPY concurrent test jobs result in random failures so for now
-          # run non-distributed tests.
+          # On PYPY coverage is very slow (5min vs 30min) as there is no C
+          # extension.
+          # There is very little PYPY specific code so there is not much to
+          # gain from reporting coverage.
           - python-version: pypy-3.6
-            tox-env: alldeps-withcov-posix
-            trial-args:
+            tox-env: alldeps-nocov-posix
+          - python-version: pypy-3.7
+            tox-env: alldeps-nocov-posix
+          # We still run some tests with coverage
           - python-version: pypy-3.7
             tox-env: alldeps-withcov-posix
-            trial-args:
+            trial-target: twisted.test.test_compat twisted.test.test_defer twisted.internet.test.test_socket twisted.trial.test.test_tests
 
 
     steps:
@@ -99,36 +103,32 @@ jobs:
     - uses: twisted/python-info-action@v1
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip tox coveralls
-        tox --notest
+        python -m pip install --upgrade pip tox coverage coveralls
 
     - name: Test
       run: |
-        python --version
-        tox -q
+        tox ${{ matrix.trial-target }}
 
     - name: Prepare coverage
-      if: always()
-      continue-on-error: true
+      if: always() && contains(matrix['tox-env'], 'withcov')
       run: |
         # sub-process coverage are generated in separate files so we combine them
         # to get an unified coverage for the local run.
         # The XML is generate to be used with 3rd party tools like diff-cover.
         python -m coverage combine
         python -m coverage xml -o coverage.xml -i
-        python -m coverage report
+        python -m coverage report --skip-covered
 
     - uses: codecov/codecov-action@v1
-      if: always()
-      continue-on-error: true
+      if: always() && contains(matrix['tox-env'], 'withcov')
       with:
         files: coverage.xml
         name: lnx-${{ matrix.python-version }}-${{ matrix.tox-env }}${{ matrix.noipv6 }}
         fail_ci_if_error: true
         functionalities: gcov,search
 
     - name: Publish to Coveralls
-      if: always()
+      if: always() && contains(matrix['tox-env'], 'withcov')
       continue-on-error: true
       run: |
         python -m coveralls -v

diff --git a/azure-pipelines/run_test_steps.yml b/azure-pipelines/run_test_steps.yml
@@ -89,9 +89,15 @@ steps:
 # https://github.com/TheKevJames/coveralls-python/blob/04b6a2876e4e7ab2e8cf0778f88ce23f94679931/coveralls/api.py#L147
 # https://github.com/TheKevJames/coveralls-python/blob/04b6a2876e4e7ab2e8cf0778f88ce23f94679931/coveralls/git.py#L31
 - bash: |
-    export CI_BRANCH=$SYSTEM_PULLREQUEST_SOURCEBRANCH
-    export CI_PULL_REQUEST=$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER
-    export CI_BUILD_URL=$SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI/pull/$CI_PULL_REQUEST
+    # Start by trying to get the branch for a push.
+    export CI_BRANCH=$BUILD_SOURCEBRANCHNAME
+    if ["$CI_BRANCH" == "merge"]; then
+      # Looks like we have a PR request.
+      export CI_BRANCH=$SYSTEM_PULLREQUEST_SOURCEBRANCH
+      export CI_PULL_REQUEST=$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER
+      export CI_BUILD_URL=$SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI/pull/$CI_PULL_REQUEST
+    fi
+
     python -m coveralls -v
   displayName: 'Report to Coveralls'
   condition: always()

diff --git a/codecov.yml b/codecov.yml
@@ -1,15 +1,43 @@
 #
-# For documentation see https://codecov.io/gh/twisted/twisted/settings/yaml
+# For documentation: https://docs.codecov.io/docs/codecovyml-reference
+# Twisted settings: https://codecov.io/gh/twisted/twisted/settings/yaml
 #
 # We want 100% coverage for new patches to make sure we are always increasing
 # the coverage.
 #
+codecov:
+  require_ci_to_pass: yes
+  notify:
+    # We have at least 10 builds in GitHub Actions and 12 in Azure
+    # and lint + mypy + docs + ReadTheDocs
+    after_n_builds: 15
+    wait_for_ci: yes
+
 coverage:
   precision: 2
   round: down
   status:
     patch:
       default:
+        # New code should have 100% CI coverage as the minimum
+        # quality assurance measurement.
+        # If there is a good reason for new code not to have coverage,
+        # add inline pragma comments.
         target: 100%
+    project:
+      default:
+        # Temporary allow for a bit of slack in overall code coverage due to
+        # swinging coverage that is not triggered by changes in a PR.
+        # See: https://twistedmatrix.com/trac/ticket/10170
+        threshold: 0.02%
+
+
+# We don't want to receive general PR comments about coverage.
+# We have the commit status checks and that should be enough.
+# See https://docs.codecov.io/docs/pull-request-comments
+comment: false
 
-comment: off
+# See https://docs.codecov.io/docs/github-checks
+github_checks:
+  # We want codecov to send inline PR comments for missing coverage.
+  annotations: true
diff --git a/src/twisted/newsfragments/10121.misc b/src/twisted/newsfragments/10121.misc
diff --git a/src/twisted/trial/test/test_tests.py b/src/twisted/trial/test/test_tests.py
@@ -1166,6 +1166,7 @@ def test_decorateTestSuiteReferences(self):
         """
         getrefcount = getattr(sys, "getrefcount", None)
         if getrefcount is None:
+            # For example non CPython like _PYPY.
             raise unittest.SkipTest("getrefcount not supported on this platform")
         test = self.TestCase()
         suite = unittest.TestSuite([test])