#!/usr/bin/env python3 from __future__ import annotations import subprocess from subprocess import Popen from sys import argv, executable, exit # Slow test suites CMDLINE = "PythonCmdline" PEP561 = "PEP561Suite" EVALUATION = "PythonEvaluation" DAEMON = "testdaemon" STUBGEN_CMD = "StubgenCmdLine" STUBGEN_PY = "StubgenPythonSuite" MYPYC_RUN = "TestRun" MYPYC_RUN_MULTI = "TestRunMultiFile" MYPYC_EXTERNAL = "TestExternal" MYPYC_COMMAND_LINE = "TestCommandLine" ERROR_STREAM = "ErrorStreamSuite" ALL_NON_FAST = [ CMDLINE, PEP561, EVALUATION, DAEMON, STUBGEN_CMD, STUBGEN_PY, MYPYC_RUN, MYPYC_RUN_MULTI, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, ERROR_STREAM, ] # This must be enabled by explicitly including 'pytest-extra' on the command line PYTEST_OPT_IN = [PEP561] # These must be enabled by explicitly including 'mypyc-extra' on the command line. MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI] # We split the pytest run into three parts to improve test # parallelization. Each run should have tests that each take a roughly similar # time to run. cmds = { # Self type check "self": [ executable, "-m", "mypy", "--config-file", "mypy_self_check.ini", "-p", "mypy", "-p", "mypyc", ], # Lint "lint": ["pre-commit", "run", "--all-files"], # Fast test cases only (this is the bulk of the test suite) "pytest-fast": ["pytest", "-q", "-k", f"not ({' or '.join(ALL_NON_FAST)})"], # Test cases that invoke mypy (with small inputs) "pytest-cmdline": [ "pytest", "-q", "-k", " or ".join([CMDLINE, EVALUATION, STUBGEN_CMD, STUBGEN_PY]), ], # Test cases that may take seconds to run each "pytest-slow": [ "pytest", "-q", "-k", " or ".join([DAEMON, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, ERROR_STREAM]), ], # Test cases that might take minutes to run "pytest-extra": ["pytest", "-q", "-k", " or ".join(PYTEST_OPT_IN)], # Mypyc tests that aren't run by default, since they are slow and rarely # fail for commits that don't touch mypyc "mypyc-extra": ["pytest", "-q", "-k", " or ".join(MYPYC_OPT_IN)], } # Stop run immediately if these commands fail FAST_FAIL = ["self", "lint"] EXTRA_COMMANDS = ("pytest-extra", "mypyc-extra") DEFAULT_COMMANDS = [cmd for cmd in cmds if cmd not in EXTRA_COMMANDS] assert all(cmd in cmds for cmd in FAST_FAIL) def run_cmd(name: str) -> int: status = 0 cmd = cmds[name] print(f"run {name}: {cmd}") proc = subprocess.run(cmd, stderr=subprocess.STDOUT) if proc.returncode: print("\nFAILED: %s" % name) status = proc.returncode if name in FAST_FAIL: exit(status) return status def start_background_cmd(name: str) -> Popen: cmd = cmds[name] proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) return proc def wait_background_cmd(name: str, proc: Popen) -> int: output = proc.communicate()[0] status = proc.returncode print(f"run {name}: {cmds[name]}") if status: print(output.decode().rstrip()) print("\nFAILED:", name) if name in FAST_FAIL: exit(status) return status def main() -> None: prog, *args = argv if not set(args).issubset(cmds): print("usage:", prog, " ".join(f"[{k}]" for k in cmds)) print() print( "Run the given tests. If given no arguments, run everything except" + " pytest-extra and mypyc-extra." ) exit(1) if not args: args = DEFAULT_COMMANDS.copy() status = 0 if "self" in args and "lint" in args: # Perform lint and self check in parallel as it's faster. proc = start_background_cmd("lint") cmd_status = run_cmd("self") if cmd_status: status = cmd_status cmd_status = wait_background_cmd("lint", proc) if cmd_status: status = cmd_status args = [arg for arg in args if arg not in ("self", "lint")] for arg in args: cmd_status = run_cmd(arg) if cmd_status: status = cmd_status exit(status) if __name__ == "__main__": main()