Merge cov-subprocess-8266-3: Twisted's coverage reports should includ…

…e the scripts that are run in subprocesses Author: hawkowl Reviewer: adiroiban Fixes: twisted#8266 git-svn-id: svn://svn.twistedmatrix.com/svn/Twisted/trunk@47362 bbbe8e31-12d6-0310-92fd-ac37d47ddeeb
cwaldbieser · May 8, 2016 · 5a00a15 · 5a00a15
1 parent 6223e68
commit 5a00a15
Show file tree

Hide file tree

Showing 23 changed files with 227 additions and 166 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,6 +1,13 @@
 [run]
-source = twisted
 branch = True
+parallel = True
+source = twisted
+
+[paths]
+source=
+   twisted
+   build/*/lib/python*/site-packages/twisted
+   build/pypy*/site-packages/twisted
 
 [report]
 precision = 2

diff --git a/.gitignore b/.gitignore
@@ -10,7 +10,7 @@ docs/_build/
 dist/
 venv/
 htmlcov/
-.coverage
+.coverage*
 *~
 *.lock
 apidocs/
diff --git a/admin/_copy.py b/admin/_copy.py
@@ -0,0 +1,10 @@
+"""
+A cross-platform copying tool that supports globbing.
+"""
+
+import sys
+import glob
+import shutil
+
+for f in glob.glob(sys.argv[1]):
+    shutil.copy(f, sys.argv[2])
diff --git a/admin/zz_coverage.pth b/admin/zz_coverage.pth
@@ -0,0 +1 @@
+import coverage; coverage.process_startup() # A .pth hack to load coverage process startup in subprocesses spawned by Python, see http://nedbatchelder.com/blog/201001/running_code_at_python_startup.html
diff --git a/tox.ini b/tox.ini
@@ -36,6 +36,8 @@ deps =
      ; Documentation
      apidocs: pydoctor
      narrativedocs: sphinx
+setenv =
+   COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
 
 commands =
     {tests,nomodules}: {envbindir}/trial --reactor={env:TWISTED_REACTOR:default} {posargs:twisted}
@@ -46,9 +48,11 @@ commands =
     apidocs: {toxinidir}/bin/admin/build-apidocs {toxinidir} apidocs
     narrativedocs: sphinx-build -aW -b html -d {toxinidir}/docs/_build {toxinidir}/docs {toxinidir}/docs/_build/
 
+    coverage: python {toxinidir}/admin/_copy.py {toxinidir}/admin/zz_coverage.pth {envsitepackagesdir}/zz_coverage.pth
     coverage: coverage erase
-    coverage: coverage run --rcfile={toxinidir}/.coveragerc {envbindir}/trial --reactor={env:TWISTED_REACTOR:default} {posargs:twisted}
-    coverage: coverage report --rcfile={toxinidir}/.coveragerc
+    coverage: coverage run -p --rcfile={toxinidir}/.coveragerc {envbindir}/trial --reactor={env:TWISTED_REACTOR:default} {posargs:twisted}
+    coverage: python {toxinidir}/admin/_copy.py "_trial_temp/.coverage*" {toxinidir}
+    coverage: python {toxinidir}/admin/_copy.py ".coverage*" {toxinidir}
 
 [testenv:twistedchecker]
 basepython=python2.7

diff --git a/twisted/internet/test/test_process.py b/twisted/internet/test/test_process.py
@@ -3,6 +3,9 @@
 
 """
 Tests for implementations of L{IReactorProcess}.
+
+@var properEnv: A copy of L{os.environ} which has L{bytes} keys/values on POSIX
+    platforms and native L{str} keys/values on Windows.
 """
 
 from __future__ import division, absolute_import, print_function
@@ -20,7 +23,8 @@
 from twisted.python.log import msg, err
 from twisted.python.runtime import platform
 from twisted.python.filepath import FilePath, _asFilesystemBytes
-from twisted.python.compat import networkString, _PY3, xrange, items
+from twisted.python.compat import (networkString, _PY3, xrange, items,
+                                   bytesEnviron)
 from twisted.internet import utils
 from twisted.internet.interfaces import IReactorProcess, IProcessTransport
 from twisted.internet.defer import Deferred, succeed
@@ -37,12 +41,19 @@
     resource = None
     process = None
     _uidgidSkip = "Cannot change UID/GID on Windows"
+
+    properEnv = dict(os.environ)
+    properEnv["PYTHONPATH"] = os.pathsep.join(sys.path)
 else:
     import resource
     from twisted.internet import process
     if os.getuid() != 0:
         _uidgidSkip = "Cannot change UID/GID except as root"
 
+    properEnv = bytesEnviron()
+    properEnv[b"PYTHONPATH"] = os.pathsep.join(sys.path).encode(
+        sys.getfilesystemencoding())
+
 
 
 def onlyOnPOSIX(testMethod):
@@ -597,7 +608,7 @@ class ProcessTestsBuilder(ProcessTestsBuilderBase):
     """
     usePTY = False
 
-    keepStdioOpenProgram = FilePath(__file__).sibling(b'process_helper.py').path
+    keepStdioOpenProgram = b'twisted.internet.test.process_helper'
     if platform.isWindows():
         keepStdioOpenArg = b"windows"
     else:
@@ -622,12 +633,12 @@ def makeConnection(self, transport):
             def childConnectionLost(self, childFD):
                 lost[childFD].callback(None)
 
-        target = FilePath(__file__).sibling(b"process_loseconnection.py")
+        target = b"twisted.internet.test.process_loseconnection"
 
         reactor = self.buildReactor()
         reactor.callWhenRunning(
             reactor.spawnProcess, Closer(), pyExe,
-            [pyExe, target.path], usePTY=self.usePTY)
+            [pyExe, b"-m", target], env=properEnv, usePTY=self.usePTY)
 
         def cbConnected(transport):
             transport.write(b'2\n')
@@ -680,9 +691,9 @@ def processEnded(self, reason):
         reactor = self.buildReactor()
         reactor.callWhenRunning(
             reactor.spawnProcess, Ender(), pyExe,
-            [pyExe, self.keepStdioOpenProgram, b"child",
+            [pyExe, b"-m", self.keepStdioOpenProgram, b"child",
              self.keepStdioOpenArg],
-            usePTY=self.usePTY)
+            env=properEnv, usePTY=self.usePTY)
 
         def cbEnded(args):
             failure, = args
@@ -728,9 +739,9 @@ def processExited(self, reason):
         reactor = self.buildReactor()
         reactor.callWhenRunning(
             reactor.spawnProcess, Waiter(), pyExe,
-            [pyExe, self.keepStdioOpenProgram, b"child",
+            [pyExe, b"-u", b"-m", self.keepStdioOpenProgram, b"child",
              self.keepStdioOpenArg],
-            usePTY=self.usePTY)
+            env=properEnv, usePTY=self.usePTY)
 
         def cbExited(args):
             failure, = args
@@ -804,7 +815,7 @@ def test_processCommandLineArguments(self):
         Arguments given to spawnProcess are passed to the child process as
         originally intended.
         """
-        us = FilePath(__file__).sibling(b"process_cli.py")
+        us = b"twisted.internet.test.process_cli"
 
         args = [b'hello', b'"', b' \t|<>^&', br'"\\"hello\\"', br'"foo\ bar baz\""']
         # Ensure that all non-NUL characters can be passed too.
@@ -829,7 +840,8 @@ def shutdown(result):
         def spawnChild():
             d = succeed(None)
             d.addCallback(lambda dummy: utils.getProcessOutputAndValue(
-                pyExe, [us.path] + args, reactor=reactor))
+                pyExe, [b"-m", us] + args, env=properEnv,
+                reactor=reactor))
             d.addCallback(processFinished)
             d.addBoth(shutdown)
 

diff --git a/twisted/python/dist3.py b/twisted/python/dist3.py
@@ -458,13 +458,14 @@
     "twisted.positioning.test.receiver",
     "twisted.python.test.pullpipe",
     "twisted.python.test.pullpipe",
-    "twisted.test._preamble",
     "twisted.test.plugin_basic",
     "twisted.test.plugin_extra1",
     "twisted.test.plugin_extra2",
     "twisted.test.process_cmdline",
     "twisted.test.process_echoer",
     "twisted.test.process_fds",
+    "twisted.test.process_getargv",
+    "twisted.test.process_getenv",
     "twisted.test.process_linger",
     "twisted.test.process_reader",
     "twisted.test.process_signal",

diff --git a/twisted/python/test/test_dist3.py b/twisted/python/test/test_dist3.py
@@ -49,4 +49,5 @@ def test_dataFileExist(self):
         root = os.path.dirname(os.path.dirname(twisted.__file__))
         for file in testDataFiles:
             self.assertTrue(os.path.exists(
-                os.path.join(root, os.path.sep.join(file.split(".")) + ".py")))
+                os.path.join(root, os.path.sep.join(file.split(".")) + ".py")),
+                            "Data file {0} does not exist".format(file))
diff --git a/twisted/test/_preamble.py b/twisted/test/_preamble.py
diff --git a/twisted/test/process_fds.py b/twisted/test/process_fds.py
@@ -7,35 +7,37 @@
 
 import os, sys
 
-debug = 0
+if __name__ == "__main__":
 
-if debug: stderr = os.fdopen(2, "w")
+    debug = 0
 
-if debug: print("this is stderr", file=stderr)
+    if debug: stderr = os.fdopen(2, "w")
 
-abcd = os.read(0, 4)
-if debug: print("read(0):", abcd, file=stderr)
-if abcd != b"abcd":
-    sys.exit(1)
+    if debug: print("this is stderr", file=stderr)
 
-if debug: print("os.write(1, righto)", file=stderr)
-os.write(1, b"righto")
+    abcd = os.read(0, 4)
+    if debug: print("read(0):", abcd, file=stderr)
+    if abcd != b"abcd":
+        sys.exit(1)
 
-efgh = os.read(3, 4)
-if debug: print("read(3):", file=stderr)
-if efgh != b"efgh":
-    sys.exit(2)
+    if debug: print("os.write(1, righto)", file=stderr)
+    os.write(1, b"righto")
 
-if debug: print("os.close(4)", file=stderr)
-os.close(4)
+    efgh = os.read(3, 4)
+    if debug: print("read(3):", file=stderr)
+    if efgh != b"efgh":
+        sys.exit(2)
 
-eof = os.read(5, 4)
-if debug: print("read(5):", eof, file=stderr)
-if eof != b"":
-    sys.exit(3)
+    if debug: print("os.close(4)", file=stderr)
+    os.close(4)
 
-if debug: print("os.write(1, closed)", file=stderr)
-os.write(1, b"closed")
+    eof = os.read(5, 4)
+    if debug: print("read(5):", eof, file=stderr)
+    if eof != b"":
+        sys.exit(3)
 
-if debug: print("sys.exit(0)", file=stderr)
-sys.exit(0)
+    if debug: print("os.write(1, closed)", file=stderr)
+    os.write(1, b"closed")
+
+    if debug: print("sys.exit(0)", file=stderr)
+    sys.exit(0)
diff --git a/twisted/test/process_getargv.py b/twisted/test/process_getargv.py
@@ -0,0 +1,15 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Used by L{twisted.test.test_process}.
+"""
+
+from __future__ import absolute_import, division
+
+from sys import stdout, argv
+
+if __name__ == "__main__":
+
+    stdout.write(chr(0).join(argv))
+    stdout.flush()
diff --git a/twisted/test/process_getenv.py b/twisted/test/process_getenv.py
@@ -0,0 +1,13 @@
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Used by L{twisted.test.test_process}.
+"""
+
+from sys import stdout
+from os import environ
+
+items = environ.items()
+stdout.write(chr(0).join([k + chr(0) + v for k, v in items]))
+stdout.flush()
diff --git a/twisted/test/stdio_test_consumer.py b/twisted/test/stdio_test_consumer.py
@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.python import log, reflect

diff --git a/twisted/test/stdio_test_halfclose.py b/twisted/test/stdio_test_halfclose.py
@@ -11,7 +11,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from zope.interface import implementer

diff --git a/twisted/test/stdio_test_hostpeer.py b/twisted/test/stdio_test_hostpeer.py
@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.internet import stdio, protocol

diff --git a/twisted/test/stdio_test_lastwrite.py b/twisted/test/stdio_test_lastwrite.py
@@ -11,7 +11,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.internet.protocol import Protocol

diff --git a/twisted/test/stdio_test_loseconn.py b/twisted/test/stdio_test_loseconn.py
@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.internet.error import ConnectionDone

diff --git a/twisted/test/stdio_test_producer.py b/twisted/test/stdio_test_producer.py
@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.internet import stdio, protocol

diff --git a/twisted/test/stdio_test_write.py b/twisted/test/stdio_test_write.py
@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.internet import stdio, protocol

diff --git a/twisted/test/stdio_test_writeseq.py b/twisted/test/stdio_test_writeseq.py
@@ -10,7 +10,6 @@
 
 from __future__ import absolute_import, division
 
-__import__('_preamble')
 import sys
 
 from twisted.internet import stdio, protocol
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		import coverage; coverage.process_startup() # A .pth hack to load coverage process startup in subprocesses spawned by Python, see http://nedbatchelder.com/blog/201001/running_code_at_python_startup.html