Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generate/test Manylinux wheels, clean up wheel/build process #1105

Merged
merged 8 commits into from
Jul 31, 2020
Merged

Generate/test Manylinux wheels, clean up wheel/build process #1105

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5b50584
install auditwheel from git, add test_wheel script, make sure manylin…
sc1f Jun 23, 2020
2d6bd91
remove platform flag from azure
sc1f Jun 26, 2020
acd1143
verbose copy in wheel script
sc1f Jun 26, 2020
77b4776
pass macos flag, remove py2 mac limitation for wheel test
sc1f Jun 29, 2020
0ac6145
run python3 auditwheel aghainst python2 wheel
sc1f Jun 29, 2020
83ac68c
Test and upload py2 wheel
sc1f Jun 29, 2020
0950248
Remove hack to find pyarrow relative to install path from linux builds
sc1f Jul 29, 2020
61ad11d
update datetime docs in py user guide
sc1f Jul 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
install auditwheel from git, add test_wheel script, make sure manylin… 
…ux images work
  • Loading branch information
@sc1f
sc1f committed Jul 29, 2020
commit 5b505842cacfca8f3d9df5380abb28d757de0aee
96 changes: 16 additions & 80 deletions azure-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,15 +123,20 @@ jobs:
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'

# Build a python wheel for Manylinux
- bash: yarn _wheel_python --platform manylinux --ci $(python_flag) $(manylinux_flag)
condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
- bash: yarn _wheel_python --ci $(python_flag) $(manylinux_flag)
condition: and(succeeded(), ne(variables['manylinux_flag'], ''), ne(variables['python_flag'], '--python2'))
displayName: 'Build Manylinux Wheel'
env:
PSP_DOCKER: 1

# Test the wheel
- bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) $(manylinux_flag)
condition: and(succeeded(), ne(variables['manylinux_flag'], ''), ne(variables['python_flag'], '--python2'))
displayName: 'Test Manylinux Wheel'

# Save the artifact to Azure storage
- task: PublishPipelineArtifact@1
condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
condition: and(succeeded(), ne(variables['manylinux_flag'], ''), ne(variables['python_flag'], '--python2'))
inputs:
targetPath: '$(System.DefaultWorkingDirectory)/python/perspective/wheelhouse/'
artifactName: '$(artifact_name)'
Expand Down Expand Up @@ -270,43 +275,10 @@ jobs:
- bash: yarn _wheel_python --ci $(python_flag) --platform osx
displayName: 'Build wheel'

# Start a new virtual environment for the Perspective wheel, so as to
# isolate it from previously installed dependencies
- bash: python -m pip install virtualenv && python -m virtualenv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python2'))
displayName: "Create virtualenv (Python 2)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], ''))
displayName: "Create virtualenv (Python 3.7)"

- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python36'))
displayName: "Create virtualenv (Python 3.6)"

- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python37'))
displayName: "Create virtualenv (Python 3.7)"

- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python38'))
displayName: "Create virtualenv (Python 3.8)"

- bash: source ./temp_venv/bin/activate
displayName: "Activate virtualenv"

# Make sure that Perspective is not installed
- bash: "python -c 'from __future__ import print_function\ntry: import perspective; print(perspective.is_libpsp())\nexcept:print(\"Perspective not installed\")'"
displayName: 'Verify that Perspective is not installed'

# Install the mac wheel
- bash: python -m pip install --force-reinstall $(System.DefaultWorkingDirectory)/python/perspective/dist/*.whl
displayName: 'Install from wheel'

# Make sure that the binaries are valid and can be imported
- bash: "python -c 'from __future__ import print_function\nimport perspective; print(perspective.is_libpsp())'"
displayName: 'Verify successful installation'
# Test the wheel
- bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag)
condition: and(succeeded(), ne(variables['python_flag'], '--python2'))
displayName: 'Test Mac Wheel'

# Save the artifact to Azure storage
- task: PublishPipelineArtifact@1
Expand Down Expand Up @@ -390,46 +362,10 @@ jobs:
- bash: yarn _wheel_python --ci $(python_flag) --platform osx
displayName: 'Build wheel'

# Start a new virtual environment for the Perspective wheel, so as to
# isolate it from previously installed dependencies
- bash: python -m pip install virtualenv && python -m virtualenv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python2'))
displayName: "Create virtualenv (Python 2)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], ''))
displayName: "Create virtualenv (Python 3.7)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python36'))
displayName: "Create virtualenv (Python 3.6)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python37'))
displayName: "Create virtualenv (Python 3.7)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python38'))
displayName: "Create virtualenv (Python 3.8)"

- bash: source ./temp_venv/bin/activate
displayName: "Activate virtualenv"

# Make sure that Perspective is not installed
- bash: "python -c 'from __future__ import print_function\ntry: import perspective; print(perspective.is_libpsp())\nexcept:print(\"Perspective not installed\")'"
displayName: 'Verify that Perspective is not installed'

# Install the mac wheel
- bash: python -m pip install --force-reinstall $(System.DefaultWorkingDirectory)/python/perspective/dist/*.whl
displayName: 'Install from wheel'

# Make sure that the binaries are valid and can be imported
- bash: "python -c 'from __future__ import print_function\nimport perspective; print(perspective.is_libpsp())'"
displayName: 'Verify successful installation'
# Test the wheel
- bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag)
condition: and(succeeded(), ne(variables['python_flag'], '--python2'))
displayName: 'Test Mac Wheel'

# Save the artifact to Azure storage
- task: PublishPipelineArtifact@1
Expand Down
2 changes: 0 additions & 2 deletions cmake/modules/FindPyArrow.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@ find_path(PYTHON_PYARROW_INCLUDE_DIR arrow/python/api.h
set(PYTHON_PYARROW_LIBRARY_DIR ${__pyarrow_library_dirs})

# Figure out the major version for the .so/.dylibs
message(${__pyarrow_version})
string(REPLACE "." ";" PYARROW_VERSION_LIST ${__pyarrow_version})
message(${PYARROW_VERSION_LIST})
list(GET PYARROW_VERSION_LIST 0 PYARROW_VERSION_MAJOR)
list(GET PYARROW_VERSION_LIST 1 PYARROW_VERSION_MINOR)
list(GET PYARROW_VERSION_LIST 2 PYARROW_VERSION_PATCH)
Expand Down
14 changes: 9 additions & 5 deletions cpp/perspective/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,11 +616,15 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
target_compile_options(binding PRIVATE -Wdeprecated-declarations)
# Add a relative path to search for PyArrow - when Perspective is
# installed from a wheel, PyArrow may not be in the same directory
# as the PyArrow which was used to build the wheel. Assuming that
# both Pyarrow and Perspective are installed in `site-packages`,
# the relative search path should be able to pick up pyarrow.
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR})
set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR})
# as the PyArrow which was used to build the wheel.
#
# Assuming that both Pyarrow and Perspective are installed in
# `site-packages`, the relative search path should be able to pick
# up pyarrow. This is only enabled for MacOS, as `auditwheel`
# will not delocate libarrow properly if it is in the rpath.
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${PYTHON_PYARROW_LIBRARY_DIR})
set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${PYTHON_PYARROW_LIBRARY_DIR})

endif()

target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
Expand Down
14 changes: 9 additions & 5 deletions docker/python/manylinux2010/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,15 @@ RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock flake8 pyt
RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil
RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil

# Force reinstall auditwheel
RUN python2.7 -m pip install --ignore-installed auditwheel
RUN python3.6 -m pip install --ignore-installed auditwheel
RUN python3.7 -m pip install --ignore-installed auditwheel
RUN python3.8 -m pip install --ignore-installed auditwheel
# Force reinstall auditwheel from Git master, as it does not overwrite
# rpaths - this was breaking manylinux wheels. Auditwheel is only for
# python 3.5+.
# https://github.com/pypa/auditwheel/pull/245#issuecomment-648132311
RUN git clone https://github.com/pypa/auditwheel.git ./auditwheel \
&& cd auditwheel \
&& python3.6 -m pip install --ignore-installed . \
&& python3.7 -m pip install --ignore-installed . \
&& python3.8 -m pip install --ignore-installed .

# install boost
RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1
Expand Down
12 changes: 8 additions & 4 deletions docker/python/manylinux2014/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,14 @@ RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock flake8 pyt
RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil
RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil

# Force reinstall auditwheel
RUN python3.6 -m pip install --ignore-installed auditwheel
RUN python3.7 -m pip install --ignore-installed auditwheel
RUN python3.8 -m pip install --ignore-installed auditwheel
# Force reinstall auditwheel from Git master, as it does not overwrite
# rpaths - this was breaking manylinux wheels.
# https://github.com/pypa/auditwheel/pull/245#issuecomment-648132311
RUN git clone https://github.com/pypa/auditwheel.git ./auditwheel \
&& cd auditwheel \
&& python3.6 -m pip install --ignore-installed . \
&& python3.7 -m pip install --ignore-installed . \
&& python3.8 -m pip install --ignore-installed .

# install boost
RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1
Expand Down
111 changes: 111 additions & 0 deletions python/perspective/scripts/test_wheels.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/bash
#
# Copyright (c) 2019, the Perspective Authors.
#
# This file is part of the Perspective library, distributed under the terms of
# the Apache License 2.0. The full license can be found in the LICENSE file.
#
# Creates a new virtual environment and installs Perspective from a wheel,
# testing whether C++ bindings are imported properly.
#
# Usage:
# 1. Make sure wheels are built and in `python/perspective/wheelhouse`:
# $ yarn _wheel_python --{manylinux2010|manylinux2014|macos} --{python36|python38}
# 2. Run this script, passing in in the Python version you want to run
# against (defaults to Python 3.7):
# $ ./test_wheels.sh --{manylinux2010|manylinux2014|macos} --{python36|python38}

HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PYTHON_INTERPRETER="python3.7"
PYTHON_TAG="cp37"
PLATFORM="manylinux2010"

# Test one version of python with one platform at a time - on Azure, all wheels
# are built individually and so we don't need to test manylinux2010 and 2014
# or different versions of Python in the same command.
while :; do
case $1 in
"--python36")
PYTHON_INTERPRETER="python3.6"
PYTHON_TAG="cp36"
;;
"--python38")
PYTHON_INTERPRETER="python3.8"
PYTHON_TAG="cp38"
;;
"--manylinux2010")
PLATFORM="manylinux2010"
;;
"--manylinux2014")
PLATFORM="manylinux2010"
;;
"--macos")
PLATFORM="macos"
;;
*) break
esac
shift
done

echo "Testing ${PYTHON_INTERPRETER} wheels for ${PLATFORM}"

if [ ! -d ${HERE}/../wheelhouse ]; then
echo "wheelhouse directory does not exist; run yarn _wheel_python first to generate wheels."
exit 1
fi

echo "Creating ${PYTHON_INTERPRETER} virtualenv in python/perspective/test_wheel_venvs"

if [ ! -d ${HERE}/../test_wheel_venvs ]; then
echo "Creating test_wheel_venvs directory"
mkdir ${HERE}/../test_wheel_venvs
fi

cd ${HERE}/../test_wheel_venvs

VENV_DIR=${PYTHON_INTERPRETER//.}_venv

if [ -d ${VENV_DIR} ]; then
echo "Cleaning up ${VENV_DIR}"
rm -rf ${VENV_DIR}
fi

# Create a new virtualenv
mkdir ${VENV_DIR}
cd ${VENV_DIR}
${PYTHON_INTERPRETER} -m venv .
source ./bin/activate
echo "${PYTHON_INTERPRETER} virtualenv activated"
cd ..

cd ${HERE}/../wheelhouse

# Look for wheels based on Python version/platform
WHEELS=$(ls . | grep "${PYTHON_TAG}-${PYTHON_TAG}m-${PLATFORM}" --include .whl)
SAVEIFS=${IFS}
IFS=$'\n'
WHEELS_LIST=(${WHEELS})
IFS=${SAVEIFS}

for wheel in ${WHEELS_LIST}; do
echo "Installing ${wheel}..."
pip install --force-reinstall ${wheel}

echo "-----------------------"
echo "Testing ${wheel}"

IS_LIBPSP=$(${PYTHON_INTERPRETER} -c "import perspective;print(perspective.is_libpsp())")

if [ "${IS_LIBPSP}" != "True" ]; then
echo "${wheel} failed to import with error: ${IS_LIBPSP}"
echo "${VENV_DIR} has been preserved for debugging"
exit 1
else
echo "${wheel} imported correctly!"
fi
done

echo "Cleaning up ${VENV_DIR}"
rm -rf ${HERE}/../test_wheel_venvs/${VENV_DIR}

exit 0
42 changes: 20 additions & 22 deletions scripts/_wheel_python.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,33 @@
*
*/

const {execute, docker, clean, resolve, getarg, bash, python_image} = require("./script_utils.js");
const {execute, execute_throw, docker, clean, resolve, getarg, bash, python_image} = require("./script_utils.js");
const fs = require("fs-extra");
const IS_DOCKER = process.env.PSP_DOCKER;
const IS_MACOS = getarg("--macos");
const IS_PY2 = getarg("--python2");
const PYTHON = IS_PY2 ? "python2" : getarg("--python38") ? "python3.8" : getarg("--python36") ? "python3.6" : "python3.7";

let IMAGE = "manylinux2014";
let MANYLINUX_VERSION;

if (IS_DOCKER) {
// defaults to 2010
let MANYLINUX_VERSION = "manylinux2010";
MANYLINUX_VERSION = "manylinux2010";
if (!IS_PY2) {
// switch to 2014 only on python3
MANYLINUX_VERSION = getarg("--manylinux2010") ? "manylinux2010" : getarg("--manylinux2014") ? "manylinux2014" : "manylinux2014";
}
IMAGE = python_image(MANYLINUX_VERSION, PYTHON);
}

const PLATFORM = getarg("--platform");

/**
* Using Perspective's docker images, create a wheel built for the image
* architecture and output it to the local filesystem.
* architecture and output it to the local filesystem. A thin wrapper around
* `bdist_wheel`, except it also automatically calls `auditwheel` (Linux) or
* `delocate` on Mac.
*/
try {
// Determine the platform - either `manylinux` or `osx`
if (!PLATFORM || !["manylinux", "osx"].includes(PLATFORM)) {
throw new Error(`Invalid platform ${PLATFORM} - Supported platforms are "manylinux" and "osx"`);
}

console.log("Copying assets to `dist` folder");
const dist = resolve`${__dirname}/../python/perspective/dist`;
const cpp = resolve`${__dirname}/../cpp/perspective`;
Expand All @@ -56,29 +53,30 @@ try {

if (IS_PY2) {
// shutil_which is required in setup.py
cmd = bash`${PYTHON} -m pip install backports.shutil_which && `;
cmd += bash`${PYTHON} -m pip install backports.shutil_which && `;
} else {
cmd = bash``;
cmd += bash``;
}

// Create a wheel
cmd += `${PYTHON} setup.py bdist_wheel`;

if (PLATFORM === "manylinux") {
// Use auditwheel on Linux
cmd += "&& auditwheel -v show ./dist/*.whl && auditwheel -v repair -L .lib ./dist/*.whl";
} else if (PLATFORM === "osx") {
// Use delocate on MacOS
cmd += "&& delocate-listdeps --all ./dist/*.whl && delocate-wheel -v ./dist/*.whl";
} else {
throw new Error("Unsupported platform specified for wheel build.");
if (MANYLINUX_VERSION && !IS_PY2) {
// Use auditwheel on Linux - repaired wheels are in
// `python/perspective/wheelhouse`.
cmd += `&& ${PYTHON} -m auditwheel -v show ./dist/*.whl && ${PYTHON} -m auditwheel -v repair -L .lib ./dist/*.whl`;
} else if (IS_MACOS) {
cmd += " && mkdir -p ./wheelhouse && cp ./dist/*.whl ./wheelhouse";
}

// TODO: MacOS wheel processed with delocate segfaults on
// `import perspective`.

if (IS_DOCKER) {
console.log(`Building wheel for \`perspective-python\` for platform \`${PLATFORM}\` using image \`${IMAGE}\` in Docker`);
console.log(`Building wheel for \`perspective-python\` using image \`${IMAGE}\` in Docker`);
execute`${docker(IMAGE)} bash -c "cd python/perspective && ${cmd}"`;
} else {
console.log(`Building wheel for \`perspective-python\` for platform \`${PLATFORM}\``);
console.log(`Building wheel for \`perspective-python\``);
const python_path = resolve`${__dirname}/../python/perspective`;
execute`cd ${python_path} && ${cmd}`;
}
Expand Down