tests: custom venv tester (scikit-build#879)

* tests: custom venv tester Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: handle pytest 3.12 warnings Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: use virtualenv instead Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * WIP: print out info in setup.py Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: fix Windows virtualenv Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: Correctly load coverage and use build[virtualenv] Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * fix(types): enable test types Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * chore: 3.12 in metadata --------- Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
holzschu · May 30, 2023 · 03fa85c · 03fa85c
1 parent d9d795a
commit 03fa85c
Show file tree

Hide file tree

Showing 14 changed files with 203 additions and 78 deletions.
diff --git a/.distro/python-scikit-build.spec b/.distro/python-scikit-build.spec
@@ -73,9 +73,8 @@ Provides:       bundled(cmake(UsePythonExtensions))
 # so we clean them.
 export CFLAGS=' '
 export CXXFLAGS=' '
-# pep518 tests are disabled because they require internet
-%pytest -k "not pep518" \
-        -m "not deprecated and not nosetuptoolsscm"
+# isolated tests are disabled because they require internet
+%pytest -m "not isolated and not deprecated and not nosetuptoolsscm"
 
 
 %files -n python3-scikit-build -f %{pyproject_files}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -68,7 +68,7 @@ repos:
   rev: "v1.3.0"
   hooks:
   - id: mypy
-    files: ^(skbuild)  # TODO: add |tests
+    files: ^(skbuild|tests)
     exclude: ^tests/samples
     additional_dependencies:
     - cmake

diff --git a/noxfile.py b/noxfile.py
@@ -45,6 +45,8 @@ def tests(session: nox.Session) -> None:
 
     numpy = [] if "pypy" in session.python or "3.12" in session.python else ["numpy"]
     install_spec = "-e.[test,cov,doctest]" if "--cov" in posargs else ".[test,doctest]"
+    if "--cov" in posargs:
+        posargs.append("--cov-config=pyproject.toml")
 
     # Latest versions may break things, so grab them for testing!
     session.install("-U", "setuptools", "wheel")

diff --git a/pyproject.toml b/pyproject.toml
@@ -24,6 +24,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Typing :: Typed",
 ]
 dependencies = [
@@ -56,7 +57,6 @@ test = [
     'cython>=0.25.1',
     'importlib-metadata;python_version<"3.8"',
     'pytest-mock>=1.10.4',
-    'pytest-virtualenv>=1.2.5',
     'pytest>=6.0.0',
     'requests',
     'virtualenv',
@@ -112,17 +112,19 @@ line-length = 120
 
 [tool.mypy]
 files = ["skbuild", "tests"]
+exclude = ["tests/samples"]
 python_version = "3.7"
 warn_unused_configs = true
 show_error_codes = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
 strict = true
 disallow_untyped_defs = false
-exclude = ["tests/samples"]
+disallow_untyped_calls = false
 
 [[tool.mypy.overrides]]
 module = ["skbuild.*"]
 disallow_untyped_defs = true
+disallow_untyped_calls = true
 
 [[tool.mypy.overrides]]
 module = [
@@ -181,6 +183,7 @@ filterwarnings = [
   'default:subprocess .* is still running:ResourceWarning',
   'ignore: pkg_resources is deprecated as an API:DeprecationWarning',
   'ignore:onerror argument is deprecated, use onexc instead:DeprecationWarning',  # Caused by wheel and Python 3.12
+  'ignore:(ast.Str|Attribute s|ast.NameConstant|ast.Num) is deprecated:DeprecationWarning:_pytest',
 ]
 log_cli_level = "info"
 markers = [

diff --git a/skbuild/setuptools_wrap.py b/skbuild/setuptools_wrap.py
@@ -394,7 +394,7 @@ def setup(
     cmake_process_manifest_hook: Callable[[list[str]], list[str]] | None = None,
     cmake_install_target: str = "install",
     **kw: Any,
-) -> None:
+) -> upstream_Distribution:
     """This function wraps setup() so that we can run cmake, make,
     CMake build, then proceed as usual with setuptools, appending the
     CMake-generated output as necessary.
@@ -510,7 +510,7 @@ def setup(
             print('Arguments following a "--" are passed directly to CMake (e.g. -DMY_VAR:BOOL=TRUE).')
             print('Arguments following a second "--" are passed directly to the build tool.')
             print(flush=True)
-        return setuptools.setup(**kw)
+        return setuptools.setup(**kw)  # type: ignore[no-any-return, func-returns-value]
 
     developer_mode = "develop" in commands or "test" in commands or build_ext_inplace
 
@@ -778,7 +778,7 @@ def has_ext_modules(self) -> bool:  # pylint: disable=no-self-use,missing-functi
 
     print(flush=True)
 
-    return setuptools.setup(**kw)
+    return setuptools.setup(**kw)  # type: ignore[no-any-return, func-returns-value]
 
 
 def _collect_package_prefixes(package_dir: dict[str, str], packages: list[Any | str]) -> list[Any | tuple[str, str]]:

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -31,6 +31,18 @@
     "samples",
 )
 
+__all__ = [
+    "SAMPLES_DIR",
+    "execute_setup_py",
+    "get_cmakecache_variables",
+    "initialize_git_repo_and_commit",
+    "list_ancestors",
+    "prepare_project",
+    "project_setup_py_test",
+    "push_dir",
+    "push_env",
+]
+
 
 @contextmanager
 def push_argv(argv):
@@ -66,7 +78,7 @@ def prepend_sys_path(paths):
     sys.path = saved_paths
 
 
-def _tmpdir(basename):
+def _tmpdir(basename: str) -> py.path.local:
     """This function returns a temporary directory similar to the one
     returned by the ``tmpdir`` pytest fixture.
     The difference is that the `basetemp` is not configurable using
@@ -80,7 +92,7 @@ def _tmpdir(basename):
 
     # Adapted from _pytest.tmpdir.TempdirFactory.getbasetemp()
     try:
-        basetemp = _tmpdir._basetemp
+        basetemp = _tmpdir._basetemp  # type: ignore[attr-defined]
     except AttributeError:
         temproot = py.path.local.get_temproot()
         user = _pytest.tmpdir.get_user()
@@ -205,7 +217,7 @@ def execute_setup_py(project_dir, setup_args, disable_languages_test=False):
     """
 
     # See https://stackoverflow.com/questions/9160227/dir-util-copy-tree-fails-after-shutil-rmtree
-    distutils.dir_util._path_created = {}
+    distutils.dir_util._path_created = {}  # type: ignore[attr-defined]
 
     # Clear _PYTHON_HOST_PLATFORM to ensure value sets in skbuild.setuptools_wrap.setup() does not
     # influence other tests.
@@ -215,7 +227,7 @@ def execute_setup_py(project_dir, setup_args, disable_languages_test=False):
     with push_dir(str(project_dir)), push_argv(["setup.py", *setup_args]), prepend_sys_path([str(project_dir)]):
         # Restore master working set that is reset following call to "python setup.py test"
         # See function "project_on_sys_path()" in setuptools.command.test
-        pkg_resources._initialize_master_working_set()
+        pkg_resources._initialize_master_working_set()  # type: ignore[attr-defined]
 
         with open("setup.py") as fp:
             setup_code = compile(fp.read(), "setup.py", mode="exec")
@@ -241,22 +253,22 @@ def project_setup_py_test(project, setup_args, tmp_dir=None, verbose_git=True, d
     def dec(fun):
         @functools.wraps(fun)
         def wrapped(*iargs, **ikwargs):
-            if wrapped.tmp_dir is None:
-                wrapped.tmp_dir = _tmpdir(fun.__name__)
-                prepare_project(wrapped.project, wrapped.tmp_dir)
-                initialize_git_repo_and_commit(wrapped.tmp_dir, verbose=wrapped.verbose_git)
+            if wrapped.tmp_dir is None:  # type: ignore[attr-defined]
+                wrapped.tmp_dir = _tmpdir(fun.__name__)  # type: ignore[attr-defined]
+                prepare_project(wrapped.project, wrapped.tmp_dir)  # type: ignore[attr-defined]
+                initialize_git_repo_and_commit(wrapped.tmp_dir, verbose=wrapped.verbose_git)  # type: ignore[attr-defined]
 
-            with execute_setup_py(wrapped.tmp_dir, wrapped.setup_args, disable_languages_test=disable_languages_test):
+            with execute_setup_py(wrapped.tmp_dir, wrapped.setup_args, disable_languages_test=disable_languages_test):  # type: ignore[attr-defined]
                 result2 = fun(*iargs, **ikwargs)
 
             if ret:
-                return wrapped.tmp_dir, result2
+                return wrapped.tmp_dir, result2  # type: ignore[attr-defined]
             return None
 
-        wrapped.project = project
-        wrapped.setup_args = setup_args
-        wrapped.tmp_dir = tmp_dir
-        wrapped.verbose_git = verbose_git
+        wrapped.project = project  # type: ignore[attr-defined]
+        wrapped.setup_args = setup_args  # type: ignore[attr-defined]
+        wrapped.tmp_dir = tmp_dir  # type: ignore[attr-defined]
+        wrapped.verbose_git = verbose_git  # type: ignore[attr-defined]
 
         return wrapped
 

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -2,34 +2,62 @@
 
 import importlib.util
 import os
+import shutil
 import subprocess
 import sys
+from collections.abc import Generator
+from pathlib import Path
 
 import pytest
+import virtualenv as _virtualenv
 
 if sys.version_info < (3, 8):
     import importlib_metadata as metadata
+    from typing_extensions import Literal, overload
 else:
     from importlib import metadata
+    from typing import Literal, overload
 
 
 HAS_SETUPTOOLS_SCM = importlib.util.find_spec("setuptools_scm") is not None
 
-DIR = os.path.dirname(os.path.abspath(__file__))
-BASE = os.path.dirname(DIR)
+DIR = Path(__file__).parent.resolve()
+BASE = DIR.parent
 
 
 pytest.register_assert_rewrite("tests.pytest_helpers")
 
 
 @pytest.fixture(scope="session")
-def pep518_wheelhouse(tmpdir_factory):
+def pep518_wheelhouse(tmp_path_factory: pytest.TempPathFactory) -> Path:
     numpy = ["numpy"] if sys.version_info < (3, 12) else []
-    wheelhouse = tmpdir_factory.mktemp("wheelhouse")
-    dist = tmpdir_factory.mktemp("dist")
-    subprocess.run([sys.executable, "-m", "build", "--wheel", "--outdir", str(dist)], cwd=BASE, check=True)
-    (wheel_path,) = dist.visit("*.whl")
-    subprocess.run([sys.executable, "-m", "pip", "download", "-q", "-d", str(wheelhouse), str(wheel_path)], check=True)
+    wheelhouse = tmp_path_factory.mktemp("wheelhouse")
+    subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "pip",
+            "wheel",
+            "--wheel-dir",
+            str(wheelhouse),
+            f"{BASE}",
+        ],
+        check=True,
+    )
+
+    # Hatch-* packages only required for test_distribution
+    packages = [
+        "build",
+        "setuptools",
+        "virtualenv",
+        "wheel",
+        "ninja",
+        "cmake",
+        "hatch-fancy-pypi-readme",
+        "hatch-vcs",
+        "hatchling",
+    ]
+
     subprocess.run(
         [
             sys.executable,
@@ -39,31 +67,114 @@ def pep518_wheelhouse(tmpdir_factory):
             "-q",
             "-d",
             str(wheelhouse),
-            "build",
-            "setuptools",
-            "wheel",
-            "ninja",
-            "cmake",
             *numpy,
+            *packages,
         ],
         check=True,
     )
-    return str(wheelhouse)
+    return wheelhouse
+
+
+class VEnv:
+    def __init__(self, env_dir: Path, *, wheelhouse: Path | None = None) -> None:
+        cmd = [str(env_dir), "--no-setuptools", "--no-wheel", "--activators", ""]
+        result = _virtualenv.cli_run(cmd, setup_logging=False)
+        self.wheelhouse = wheelhouse
+        self.executable = Path(result.creator.exe)
+        self.dest = env_dir.resolve()
+
+    @overload
+    def run(self, *args: str | os.PathLike[str], capture: Literal[True], cwd: Path | None = ...) -> str:
+        ...
+
+    @overload
+    def run(self, *args: str | os.PathLike[str], capture: Literal[False] = ..., cwd: Path | None = ...) -> None:
+        ...
+
+    def run(self, *args: str | os.PathLike[str], capture: bool = False, cwd: Path | None = None) -> str | None:
+        __tracebackhide__ = True
+        env = os.environ.copy()
+        env["PATH"] = f"{self.executable.parent}{os.pathsep}{env['PATH']}"
+        env["VIRTUAL_ENV"] = str(self.dest)
+        env["PIP_DISABLE_PIP_VERSION_CHECK"] = "ON"
+        if self.wheelhouse is not None:
+            env["PIP_NO_INDEX"] = "ON"
+            env["PIP_FIND_LINKS"] = str(self.wheelhouse)
+
+        str_args = [os.fspath(a) for a in args]
+
+        # Windows does not make a python shortcut in venv
+        if str_args[0] in {"python", "python3"}:
+            str_args[0] = str(self.executable)
+
+        if capture:
+            result = subprocess.run(
+                str_args,
+                check=False,
+                capture_output=True,
+                text=True,
+                env=env,
+                cwd=cwd,
+            )
+            if result.returncode != 0:
+                print(result.stdout, file=sys.stdout)
+                print(result.stderr, file=sys.stderr)
+                print("FAILED RUN:", *str_args, file=sys.stderr)
+                raise SystemExit(result.returncode)
+            return result.stdout.strip()
+
+        result_bytes = subprocess.run(
+            str_args,
+            check=False,
+            env=env,
+            cwd=cwd,
+        )
+        if result_bytes.returncode != 0:
+            print("FAILED RUN:", *str_args, file=sys.stderr)
+            raise SystemExit(result_bytes.returncode)
+        return None
+
+    def execute(self, command: str, cwd: Path | None = None) -> str:
+        return self.run(str(self.executable), "-c", command, capture=True, cwd=cwd)
+
+    @overload
+    def module(self, *args: str | os.PathLike[str], capture: Literal[False] = ..., cwd: Path | None = ...) -> None:
+        ...
+
+    @overload
+    def module(self, *args: str | os.PathLike[str], capture: Literal[True], cwd: Path | None = ...) -> str:
+        ...
+
+    def module(self, *args: str | os.PathLike[str], capture: bool = False, cwd: Path | None = None) -> None | str:
+        return self.run(str(self.executable), "-m", *args, capture=capture, cwd=cwd)  # type: ignore[no-any-return,call-overload]
+
+    def install(self, *args: str | os.PathLike[str]) -> None:
+        self.module("pip", "install", *args)
 
 
 @pytest.fixture()
 def pep518(pep518_wheelhouse, monkeypatch):
-    monkeypatch.setenv("PIP_FIND_LINKS", pep518_wheelhouse)
+    monkeypatch.setenv("PIP_FIND_LINKS", str(pep518_wheelhouse))
     monkeypatch.setenv("PIP_NO_INDEX", "true")
     return pep518_wheelhouse
 
 
+@pytest.fixture()
+def isolated(tmp_path: Path, pep518_wheelhouse: Path) -> Generator[VEnv, None, None]:
+    path = tmp_path / "venv"
+    try:
+        yield VEnv(path, wheelhouse=pep518_wheelhouse)
+    finally:
+        shutil.rmtree(path, ignore_errors=True)
+
+
 def pytest_report_header() -> str:
     interesting_packages = {
         "build",
         "distro",
         "packaging",
         "pip",
+        "scikit-build",
         "setuptools",
         "setuptools_scm",
         "virtualenv",
@@ -72,7 +183,7 @@ def pytest_report_header() -> str:
     valid = []
     for package in interesting_packages:
         try:
-            version = metadata.version(package)  # type: ignore[no-untyped-call]
+            version = metadata.version(package)
         except ModuleNotFoundError:
             continue
         valid.append(f"{package}=={version}")