Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generate/test Manylinux wheels, clean up wheel/build process #1105

Merged
merged 8 commits into from
Jul 31, 2020
Merged

Generate/test Manylinux wheels, clean up wheel/build process #1105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5b50584
install auditwheel from git, add test_wheel script, make sure manylin…
sc1f Jun 23, 2020
2d6bd91
remove platform flag from azure
sc1f Jun 26, 2020
acd1143
verbose copy in wheel script
sc1f Jun 26, 2020
77b4776
pass macos flag, remove py2 mac limitation for wheel test
sc1f Jun 29, 2020
0ac6145
run python3 auditwheel aghainst python2 wheel
sc1f Jun 29, 2020
83ac68c
Test and upload py2 wheel
sc1f Jun 29, 2020
0950248
Remove hack to find pyarrow relative to install path from linux builds
sc1f Jul 29, 2020
61ad11d
update datetime docs in py user guide
sc1f Jul 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 14 additions & 80 deletions azure-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,12 +123,17 @@ jobs:
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'

# Build a python wheel for Manylinux
- bash: yarn _wheel_python --platform manylinux --ci $(python_flag) $(manylinux_flag)
- bash: yarn _wheel_python --ci $(python_flag) $(manylinux_flag)
condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
displayName: 'Build Manylinux Wheel'
env:
PSP_DOCKER: 1

# Test the wheel
- bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) $(manylinux_flag)
condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
displayName: 'Test Manylinux Wheel'

# Save the artifact to Azure storage
- task: PublishPipelineArtifact@1
condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
Expand Down Expand Up @@ -267,46 +272,12 @@ jobs:
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'

# Build a python wheel for Mac 10.14
- bash: yarn _wheel_python --ci $(python_flag) --platform osx
- bash: yarn _wheel_python --ci $(python_flag) --macos
displayName: 'Build wheel'

# Start a new virtual environment for the Perspective wheel, so as to
# isolate it from previously installed dependencies
- bash: python -m pip install virtualenv && python -m virtualenv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python2'))
displayName: "Create virtualenv (Python 2)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], ''))
displayName: "Create virtualenv (Python 3.7)"

- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python36'))
displayName: "Create virtualenv (Python 3.6)"

- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python37'))
displayName: "Create virtualenv (Python 3.7)"

- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python38'))
displayName: "Create virtualenv (Python 3.8)"

- bash: source ./temp_venv/bin/activate
displayName: "Activate virtualenv"

# Make sure that Perspective is not installed
- bash: "python -c 'from __future__ import print_function\ntry: import perspective; print(perspective.is_libpsp())\nexcept:print(\"Perspective not installed\")'"
displayName: 'Verify that Perspective is not installed'

# Install the mac wheel
- bash: python -m pip install --force-reinstall $(System.DefaultWorkingDirectory)/python/perspective/dist/*.whl
displayName: 'Install from wheel'

# Make sure that the binaries are valid and can be imported
- bash: "python -c 'from __future__ import print_function\nimport perspective; print(perspective.is_libpsp())'"
displayName: 'Verify successful installation'
# Test the wheel
- bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) --macos
displayName: 'Test Mac Wheel'

# Save the artifact to Azure storage
- task: PublishPipelineArtifact@1
Expand Down Expand Up @@ -387,49 +358,12 @@ jobs:
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'

# Build a python wheel for Mac 10.14
- bash: yarn _wheel_python --ci $(python_flag) --platform osx
- bash: yarn _wheel_python --ci $(python_flag) --macos
displayName: 'Build wheel'

# Start a new virtual environment for the Perspective wheel, so as to
# isolate it from previously installed dependencies
- bash: python -m pip install virtualenv && python -m virtualenv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python2'))
displayName: "Create virtualenv (Python 2)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], ''))
displayName: "Create virtualenv (Python 3.7)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python36'))
displayName: "Create virtualenv (Python 3.6)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python37'))
displayName: "Create virtualenv (Python 3.7)"

# Use venv for python3
- bash: python -m venv ./temp_venv
condition: and(succeeded(), eq(variables['python_flag'], '--python38'))
displayName: "Create virtualenv (Python 3.8)"

- bash: source ./temp_venv/bin/activate
displayName: "Activate virtualenv"

# Make sure that Perspective is not installed
- bash: "python -c 'from __future__ import print_function\ntry: import perspective; print(perspective.is_libpsp())\nexcept:print(\"Perspective not installed\")'"
displayName: 'Verify that Perspective is not installed'

# Install the mac wheel
- bash: python -m pip install --force-reinstall $(System.DefaultWorkingDirectory)/python/perspective/dist/*.whl
displayName: 'Install from wheel'

# Make sure that the binaries are valid and can be imported
- bash: "python -c 'from __future__ import print_function\nimport perspective; print(perspective.is_libpsp())'"
displayName: 'Verify successful installation'
# Test the wheel
- bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) --macos
displayName: 'Test Mac Wheel'

# Save the artifact to Azure storage
- task: PublishPipelineArtifact@1
Expand Down
2 changes: 0 additions & 2 deletions cmake/modules/FindPyArrow.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@ find_path(PYTHON_PYARROW_INCLUDE_DIR arrow/python/api.h
set(PYTHON_PYARROW_LIBRARY_DIR ${__pyarrow_library_dirs})

# Figure out the major version for the .so/.dylibs
message(${__pyarrow_version})
string(REPLACE "." ";" PYARROW_VERSION_LIST ${__pyarrow_version})
message(${PYARROW_VERSION_LIST})
list(GET PYARROW_VERSION_LIST 0 PYARROW_VERSION_MAJOR)
list(GET PYARROW_VERSION_LIST 1 PYARROW_VERSION_MINOR)
list(GET PYARROW_VERSION_LIST 2 PYARROW_VERSION_PATCH)
Expand Down
18 changes: 12 additions & 6 deletions cpp/perspective/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,15 +612,21 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)

# .dll not importable
set_property(TARGET binding PROPERTY SUFFIX .pyd)
else()
elseif (MACOS)
target_compile_options(binding PRIVATE -Wdeprecated-declarations)

# Add a relative path to search for PyArrow - when Perspective is
# installed from a wheel, PyArrow may not be in the same directory
# as the PyArrow which was used to build the wheel. Assuming that
# both Pyarrow and Perspective are installed in `site-packages`,
# the relative search path should be able to pick up pyarrow.
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR})
set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR})
# as the PyArrow which was used to build the wheel.
#
# Assuming that both Pyarrow and Perspective are installed in
# `site-packages`, the relative search path should be able to pick
# up pyarrow. This is only enabled for MacOS, as `auditwheel`
# will not delocate libarrow properly if it is in the rpath.
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${PYTHON_PYARROW_LIBRARY_DIR})
set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${PYTHON_PYARROW_LIBRARY_DIR})
else()
target_compile_options(binding PRIVATE -Wdeprecated-declarations)
endif()

target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
Expand Down
3 changes: 1 addition & 2 deletions docker/python/manylinux2010/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock flake8 pyt
RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil
RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil

# Force reinstall auditwheel
RUN python2.7 -m pip install --ignore-installed auditwheel
# Install Auditwheel - not available on Python 2
RUN python3.6 -m pip install --ignore-installed auditwheel
RUN python3.7 -m pip install --ignore-installed auditwheel
RUN python3.8 -m pip install --ignore-installed auditwheel
Expand Down
2 changes: 1 addition & 1 deletion docker/python/manylinux2014/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock flake8 pyt
RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil
RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil

# Force reinstall auditwheel
# Install Auditwheel - not available on Python 2
RUN python3.6 -m pip install --ignore-installed auditwheel
RUN python3.7 -m pip install --ignore-installed auditwheel
RUN python3.8 -m pip install --ignore-installed auditwheel
Expand Down
28 changes: 13 additions & 15 deletions docs/md/python.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,26 +159,24 @@ two methods into your object:

#### Time Zone Handling

Columns with the `datetime` type are stored internally as UTC timestamps in milliseconds since epoch (Unix Time),
and are serialized to the user as `datetime.datetime` objects in _local time_ according to the Python runtime.
- ["Naive"](https://docs.python.org/3/library/datetime.html#aware-and-naive-objects)
datetimes are assumed to be local time.
- ["Aware"](https://docs.python.org/3/library/datetime.html#aware-and-naive-objects)
datetimes use the timezone specified in the `tzinfo`.

Both ["naive" and "aware" datetimes](https://docs.python.org/3/library/datetime.html#aware-and-naive-objects) will be
serialized to local time by Perspective, with the conversion determined by the `tzinfo` attribute:
All `datetime` columns (regardless of input timezone) are output to the user as
`datetime.datetime` objects in _local time_ according to the Python runtime.

- "Naive" datetimes are assumed to be already in local time and are serialized as-is.
- "Aware" datetimes will be converted to UTC from their original timezone, and then converted to local time
from UTC.

This behavior is consistent with Perspective's behavior in Javascript. For more details, see this
in-depth [explanation](https://github.com/finos/perspective/pull/867) of `perspective-python` semantics around time zone handling.
This behavior is consistent with Perspective's behavior in Javascript. For more
details, see this in-depth [explanation](https://github.com/finos/perspective/pull/867)
of `perspective-python` semantics around time zone handling.

##### Pandas Timestamps

`pandas.Timestamp` objects stored in a `pandas.DataFrame` are _always_ treated
as UTC times, and will be converted to local time when serialized to the user.

To treat a `Timestamp` in a `DataFrame` as local time, use `tz_localize` or
`tz_convert` to provide the `Timestamp` with a time zone.
- Naive `pandas.Timestamp` objects are _always_ treated as UTC times, and will
be converted to local time when output to the user.
- Aware `pandas.Timestamp` objects use the timezone specified in `tzinfo`. Use
`tz_localize` or `tz_convert` to provide the `Timestamp` with a time zone.

### Callbacks and Events

Expand Down
1 change: 0 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@
"build_python": "node scripts/build_python.js",
"bench": "node scripts/bench.js",
"bench_python": "python3 python/perspective/bench/perspective_benchmark.py",
"_test_wheel_python": "node scripts/_test_wheel_python.js",
"_wheel_python": "node scripts/_wheel_python.js",
"setup": "node scripts/setup.js",
"docs": "node scripts/docs.js",
Expand Down
111 changes: 111 additions & 0 deletions python/perspective/scripts/test_wheels.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/bash
#
# Copyright (c) 2019, the Perspective Authors.
#
# This file is part of the Perspective library, distributed under the terms of
# the Apache License 2.0. The full license can be found in the LICENSE file.
#
# Creates a new virtual environment and installs Perspective from a wheel,
# testing whether C++ bindings are imported properly.
#
# Usage:
# 1. Make sure wheels are built and in `python/perspective/wheelhouse`:
# $ yarn _wheel_python --{manylinux2010|manylinux2014|macos} --{python36|python38}
# 2. Run this script, passing in in the Python version you want to run
# against (defaults to Python 3.7):
# $ ./test_wheels.sh --{manylinux2010|manylinux2014|macos} --{python36|python38}

HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PYTHON_INTERPRETER="python3.7"
PYTHON_TAG="cp37"
PLATFORM="manylinux2010"

# Test one version of python with one platform at a time - on Azure, all wheels
# are built individually and so we don't need to test manylinux2010 and 2014
# or different versions of Python in the same command.
while :; do
case $1 in
"--python36")
PYTHON_INTERPRETER="python3.6"
PYTHON_TAG="cp36"
;;
"--python38")
PYTHON_INTERPRETER="python3.8"
PYTHON_TAG="cp38"
;;
"--manylinux2010")
PLATFORM="manylinux2010"
;;
"--manylinux2014")
PLATFORM="manylinux2010"
;;
"--macos")
PLATFORM="macos"
;;
*) break
esac
shift
done

echo "Testing ${PYTHON_INTERPRETER} wheels for ${PLATFORM}"

if [ ! -d ${HERE}/../wheelhouse ]; then
echo "wheelhouse directory does not exist; run yarn _wheel_python first to generate wheels."
exit 1
fi

echo "Creating ${PYTHON_INTERPRETER} virtualenv in python/perspective/test_wheel_venvs"

if [ ! -d ${HERE}/../test_wheel_venvs ]; then
echo "Creating test_wheel_venvs directory"
mkdir ${HERE}/../test_wheel_venvs
fi

cd ${HERE}/../test_wheel_venvs

VENV_DIR=${PYTHON_INTERPRETER//.}_venv

if [ -d ${VENV_DIR} ]; then
echo "Cleaning up ${VENV_DIR}"
rm -rf ${VENV_DIR}
fi

# Create a new virtualenv
mkdir ${VENV_DIR}
cd ${VENV_DIR}
${PYTHON_INTERPRETER} -m venv .
source ./bin/activate
echo "${PYTHON_INTERPRETER} virtualenv activated"
cd ..

cd ${HERE}/../wheelhouse

# Look for wheels based on Python version/platform
WHEELS=$(ls . | grep "${PYTHON_TAG}-${PYTHON_TAG}m-${PLATFORM}" --include .whl)
SAVEIFS=${IFS}
IFS=$'\n'
WHEELS_LIST=(${WHEELS})
IFS=${SAVEIFS}

for wheel in ${WHEELS_LIST}; do
echo "Installing ${wheel}..."
pip install --force-reinstall ${wheel}

echo "-----------------------"
echo "Testing ${wheel}"

IS_LIBPSP=$(${PYTHON_INTERPRETER} -c "import perspective;print(perspective.is_libpsp())")

if [ "${IS_LIBPSP}" != "True" ]; then
echo "${wheel} failed to import with error: ${IS_LIBPSP}"
echo "${VENV_DIR} has been preserved for debugging"
exit 1
else
echo "${wheel} imported correctly!"
fi
done

echo "Cleaning up ${VENV_DIR}"
rm -rf ${HERE}/../test_wheel_venvs/${VENV_DIR}

exit 0
Loading