scikit-learn · ogrisel · Nov 18, 2020 · Nov 9, 2020 · Nov 9, 2020 · Nov 9, 2020
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -78,23 +78,19 @@ jobs:
 
       - name: Build and test wheels
         env:
-          # Set the directory where the wheel is unpacked
+          CONFTEST_PATH: ${{ github.workspace }}/conftest.py
+          CONFTEST_NAME: conftest.py
           CIBW_ENVIRONMENT: WHEEL_DIRNAME=scikit_learn-$SCIKIT_LEARN_VERSION
                             OMP_NUM_THREADS=2
                             OPENBLAS_NUM_THREADS=2
                             SKLEARN_SKIP_NETWORK_TESTS=1
                             SKLEARN_BUILD_PARALLEL=3
           CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} 
+          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
           CIBW_TEST_REQUIRES: pytest pandas threadpoolctl
-          # Test that there are no links to system libraries
-          CIBW_TEST_COMMAND: pytest --pyargs sklearn &&
-                             python -m threadpoolctl -i sklearn
-          # By default, the Windows wheels are not repaired.
-          # In this case, we need to vendor the vcomp140.dll
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: wheel unpack {wheel} &&
-                                             python build_tools/github/vendor_vcomp140.py %WHEEL_DIRNAME% &&
-                                             wheel pack %WHEEL_DIRNAME% -d {dest_dir} &&
-                                             rmdir /s /q %WHEEL_DIRNAME%
+          CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh
+          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }}
 
         run: bash build_tools/github/build_wheels.sh
 
@@ -117,8 +113,17 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
 
-      - name: Build and test source distribution
+      - name: Build source distribution
         run: bash build_tools/github/build_source.sh
+        env:
+          SKLEARN_BUILD_PARALLEL: 3
+
+      - name: Test source distribution
+        run: bash build_tools/github/test_source.sh
+        env:
+          OMP_NUM_THREADS: 2
+          OPENBLAS_NUM_THREADS: 2
+          SKLEARN_SKIP_NETWORK_TESTS: 1
 
       - name: Store artifacts
         uses: actions/upload-artifact@v2
@@ -130,7 +135,7 @@ jobs:
     name: Upload to Anaconda
     runs-on: ubuntu-latest
     needs: [build_wheels, build_sdist]
-    # The artifacts are not be uploaded on PRs
+    # The artifacts cannot be uploaded on PRs
     if: github.event_name != 'pull_request'
 
     steps:

diff --git a/build_tools/github/Windows b/build_tools/github/Windows
@@ -0,0 +1,15 @@
+# Get the Python version of the base image from a build argument
+ARG PYTHON_VERSION
+FROM winamd64/python:$PYTHON_VERSION-windowsservercore
+
+ARG WHEEL_NAME
+ARG CONFTEST_NAME
+ARG CIBW_TEST_REQUIRES
+
+# Copy and install the Windows wheel
+COPY $WHEEL_NAME $WHEEL_NAME
+COPY $CONFTEST_NAME $CONFTEST_NAME
+RUN pip install $env:WHEEL_NAME
+
+# Install the testing dependencies
+RUN pip install $env:CIBW_TEST_REQUIRES.split(" ")
diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+set -x
+
+PYTHON_VERSION=$1
+BITNESS=$2
+
+if [[ "$PYTHON_VERSION" == "36" || "$BITNESS" == "32" ]]; then
+    # Python 3.6 and 32-bit architectures are not supported
+    # by the official Docker images: Tests will just be run
+    # on the host (instead of the minimal Docker container).
+    exit 0
+fi
+
+TEMP_FOLDER="$HOME/AppData/Local/Temp"
+WHEEL_PATH=$(ls -d $TEMP_FOLDER/*/repaired_wheel/*)
+WHEEL_NAME=$(basename $WHEEL_PATH)
+
+cp $WHEEL_PATH $WHEEL_NAME
+
+# Dot the Python version for identyfing the base Docker image
+PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2})
+
+# Build a minimal Windows Docker image for testing the wheels
+docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \
+             --build-arg WHEEL_NAME=$WHEEL_NAME \
+             --build-arg CONFTEST_NAME=$CONFTEST_NAME \
+             --build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \
+             -f build_tools/github/Windows \
+             -t scikit-learn/minimal-windows .
diff --git a/build_tools/github/build_source.sh b/build_tools/github/build_source.sh
@@ -3,15 +3,18 @@
 set -e
 set -x
 
+# Move up two levels to create the virtual
+# environment outside of the source folder
+cd ../../
+
+python -m venv build_env
+source build_env/bin/activate
+
 python -m pip install numpy scipy cython
 python -m pip install twine
-python -m pip install pytest pandas
 
+cd scikit-learn/scikit-learn
 python setup.py sdist
-python -m pip install dist/*.tar.gz
-python setup.py build_ext -i
-
-pytest --pyargs sklearn
 
 # Check whether the source distribution will render correctly
 twine check dist/*.tar.gz
diff --git a/build_tools/github/build_wheels.sh b/build_tools/github/build_wheels.sh
@@ -4,7 +4,7 @@ set -e
 set -x
 
 # OpenMP is not present on macOS by default
-if [ "$RUNNER_OS" == "macOS" ]; then
+if [[ "$RUNNER_OS" == "macOS" ]]; then
     brew install libomp
     export CC=/usr/bin/clang
     export CXX=/usr/bin/clang++

diff --git a/build_tools/github/repair_windows_wheels.sh b/build_tools/github/repair_windows_wheels.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -x
+
+WHEEL=$1
+DEST_DIR=$2
+BITNESS=$3
+
+# By default, the Windows wheels are not repaired.
+# In this case, we need to vendor VCRUNTIME140.dll
+wheel unpack "$WHEEL"
+python build_tools/github/vendor.py "$WHEEL_DIRNAME" "$BITNESS"
+wheel pack "$WHEEL_DIRNAME" -d "$DEST_DIR"
+rm -rf "$WHEEL_DIRNAME"
diff --git a/build_tools/github/test_source.sh b/build_tools/github/test_source.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+set -x
+
+cd ../../
+
+python -m venv test_env
+source test_env/bin/activate
+
+python -m pip install scikit-learn/scikit-learn/dist/*.tar.gz
+python -m pip install pytest pandas
+
+# Run the tests on the installed source distribution
+mkdir tmp_for_test
+cp scikit-learn/scikit-learn/conftest.py tmp_for_test
+cd tmp_for_test
+
+pytest --pyargs sklearn
diff --git a/build_tools/github/test_wheels.sh b/build_tools/github/test_wheels.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+set -x
+
+if [[ "$OSTYPE" != "linux-gnu" ]]; then
+    # The Linux test environment is run in a Docker container and
+    # it is not possible to copy the test configuration file (yet)
+    cp $CONFTEST_PATH $CONFTEST_NAME
+fi
+
+pytest --pyargs sklearn
+
+# Test that there are no links to system libraries
+python -m threadpoolctl -i sklearn
diff --git a/build_tools/github/test_windows_wheels.sh b/build_tools/github/test_windows_wheels.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+set -x
+
+PYTHON_VERSION=$1
+BITNESS=$2
+
+if [[ "$PYTHON_VERSION" == "36" || "$BITNESS" == "32" ]]; then
+    # For Python 3.6 and 32-bit architecture use the regular
+    # test command (outside of the minimal Docker container)
+    cp $CONFTEST_PATH $CONFTEST_NAME
+    pytest --pyargs sklearn
+    python -m threadpoolctl -i sklearn
+else
+    docker container run -e SKLEARN_SKIP_NETWORK_TESTS=1 \
+                         -e OMP_NUM_THREADS=2 \
+                         -e OPENBLAS_NUM_THREADS=2 \
+                         --rm scikit-learn/minimal-windows \
+                         powershell -Command "pytest --pyargs sklearn"
+
+    docker container run --rm scikit-learn/minimal-windows \
+                         powershell -Command "python -m threadpoolctl -i sklearn"
+fi
diff --git a/build_tools/github/vendor.py b/build_tools/github/vendor.py
@@ -0,0 +1,148 @@
+"""Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll.
+
+Note that vcruntime140_1.dll is only required (and available)
+for 64-bit architectures.
+"""
+
+
+import os
+import os.path as op
+import shutil
+import sys
+import textwrap
+
+
+TARGET_FOLDER = op.join("sklearn", ".libs")
+DISTRIBUTOR_INIT = op.join("sklearn", "_distributor_init.py")
+VCOMP140_SRC_PATH = "C:\\Windows\\System32\\vcomp140.dll"
+VCRUNTIME140_SRC_PATH = "C:\\Windows\\System32\\vcruntime140.dll"
+VCRUNTIME140_1_SRC_PATH = "C:\\Windows\\System32\\vcruntime140_1.dll"
+
+
+def make_distributor_init_32_bits(distributor_init,
+                                  vcomp140_dll_filename,
+                                  vcruntime140_dll_filename):
+    """Create a _distributor_init.py file for 32-bit architectures.
+
+    This file is imported first when importing the sklearn package
+    so as to pre-load the vendored vcomp140.dll and vcruntime140.dll.
+    """
+    with open(distributor_init, "wt") as f:
+        f.write(textwrap.dedent("""
+            '''Helper to preload vcomp140.dll and vcruntime140.dll to
+            prevent "not found" errors.
+
+            Once vcomp140.dll and vcruntime140.dll are preloaded, the
+            namespace is made available to any subsequent vcomp140.dll
+            and vcruntime140.dll. This is created as part of the scripts
+            that build the wheel.
+            '''
+
+
+            import os
+            import os.path as op
+            from ctypes import WinDLL
+
+
+            if os.name == "nt":
+                # Load vcomp140.dll and vcruntime140.dll
+                libs_path = op.join(op.dirname(__file__), ".libs")
+                vcomp140_dll_filename = op.join(libs_path, "{0}")
+                vcruntime140_dll_filename = op.join(libs_path, "{1}")
+                WinDLL(op.abspath(vcomp140_dll_filename))
+                WinDLL(op.abspath(vcruntime140_dll_filename))
+            """.format(vcomp140_dll_filename, vcruntime140_dll_filename)))
+
+
+def make_distributor_init_64_bits(distributor_init,
+                                  vcomp140_dll_filename,
+                                  vcruntime140_dll_filename,
+                                  vcruntime140_1_dll_filename):
+    """Create a _distributor_init.py file for 64-bit architectures.
+
+    This file is imported first when importing the sklearn package
+    so as to pre-load the vendored vcomp140.dll, vcruntime140.dll
+    and vcruntime140_1.dll.
+    """
+    with open(distributor_init, "wt") as f:
+        f.write(textwrap.dedent("""
+            '''Helper to preload vcomp140.dll, vcruntime140.dll and
+            vcruntime140_1.dll to prevent "not found" errors.
+
+            Once vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll are
+            preloaded, the namespace is made available to any subsequent
+            vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll. This is
+            created as part of the scripts that build the wheel.
+            '''
+
+
+            import os
+            import os.path as op
+            from ctypes import WinDLL
+
+
+            if os.name == "nt":
+                # Load vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll
+                libs_path = op.join(op.dirname(__file__), ".libs")
+                vcomp140_dll_filename = op.join(libs_path, "{0}")
+                vcruntime140_dll_filename = op.join(libs_path, "{1}")
+                vcruntime140_1_dll_filename = op.join(libs_path, "{2}")
+                WinDLL(op.abspath(vcomp140_dll_filename))
+                WinDLL(op.abspath(vcruntime140_dll_filename))
+                WinDLL(op.abspath(vcruntime140_1_dll_filename))
+            """.format(vcomp140_dll_filename,
+                       vcruntime140_dll_filename,
+                       vcruntime140_1_dll_filename)))
+
+
+def main(wheel_dirname, bitness):
+    """Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll."""
+    if not op.exists(VCOMP140_SRC_PATH):
+        raise ValueError(f"Could not find {VCOMP140_SRC_PATH}.")
+
+    if not op.exists(VCRUNTIME140_SRC_PATH):
+        raise ValueError(f"Could not find {VCRUNTIME140_SRC_PATH}.")
+
+    if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64":
+        raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.")
+
+    if not op.isdir(wheel_dirname):
+        raise RuntimeError(f"Could not find {wheel_dirname} file.")
+
+    vcomp140_dll_filename = op.basename(VCOMP140_SRC_PATH)
+    vcruntime140_dll_filename = op.basename(VCRUNTIME140_SRC_PATH)
+    vcruntime140_1_dll_filename = op.basename(VCRUNTIME140_1_SRC_PATH)
+
+    target_folder = op.join(wheel_dirname, TARGET_FOLDER)
+    distributor_init = op.join(wheel_dirname, DISTRIBUTOR_INIT)
+
+    # Create the "sklearn/.libs" subfolder
+    if not op.exists(target_folder):
+        os.mkdir(target_folder)
+
+    print(f"Copying {VCOMP140_SRC_PATH} to {target_folder}.")
+    shutil.copy2(VCOMP140_SRC_PATH, target_folder)
+
+    print(f"Copying {VCRUNTIME140_SRC_PATH} to {target_folder}.")
+    shutil.copy2(VCRUNTIME140_SRC_PATH, target_folder)
+
+    if bitness == "64":
+        print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.")
+        shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder)
+
+    # Generate the _distributor_init file in the source tree
+    print("Generating the '_distributor_init.py' file.")
+    if bitness == "32":
+        make_distributor_init_32_bits(distributor_init,
+                                      vcomp140_dll_filename,
+                                      vcruntime140_dll_filename)
+    else:
+        make_distributor_init_64_bits(distributor_init,
+                                      vcomp140_dll_filename,
+                                      vcruntime140_dll_filename,
+                                      vcruntime140_1_dll_filename)
+
+
+if __name__ == "__main__":
+    _, wheel_file, bitness = sys.argv
+    main(wheel_file, bitness)