diff --git a/.gitattributes b/.gitattributes index 4fd428b8c7..24aac431d6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,6 @@ scripts/* linguist-documentation tools/**/* linguist-documentation cmake/** linguist-documentation CMakeLists.txt linguist-documentation -docker/** linguist-documentation **/*.sh linguist-documentation * text=auto eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3ffb8f7e9e..ed572b079a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -669,8 +669,6 @@ jobs: ############### - name: WebAssembly Test run: yarn test_js --quiet --ci - env: - PSP_DOCKER: 1 # FIXME run in container directly ########################################################################################################################## ########################################################################################################################## diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile deleted file mode 100644 index 276efdbd80..0000000000 --- a/docker/python/manylinux2010/Dockerfile +++ /dev/null @@ -1,81 +0,0 @@ -# ***************************************************************************** -# -# Copyright (c) 2020, the Perspective Authors. -# -# This file is part of the Perspective library, distributed under the terms of -# the Apache License 2.0. The full license can be found in the LICENSE file. -# - -# https://github.com/pypa/manylinux/issues/428#issuecomment-774729597 -FROM quay.io/pypa/manylinux2010_x86_64:2021-02-06-3d322a5 -RUN yum -y install rapidjson-devel sudo wget - -# Build new cmake -RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q -RUN mkdir /opt/cmake -RUN printf "y\nn\n" | sh cmake-3.15.4-Linux-x86_64.sh --prefix=/opt/cmake > /dev/null -RUN rm -fr cmake*.sh /opt/cmake/doc -RUN rm -fr /usr/local/bin/cmake -RUN rm -fr /usr/local/bin/ctest -RUN rm -fr /opt/cmake/bin/cmake-gui -RUN rm -fr /opt/cmake/bin/ccmake -RUN rm -fr /opt/cmake/bin/cpack -RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -RUN ln -s /opt/cmake/bin/ctest /usr/local/bin/ctest - -# Remove auditwheel for reinstall -RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel -RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel -RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel -RUN rm -rf /opt/python/cp39-cp39/bin/auditwheel - - -# Copy assets -RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/ -RUN cp -arf /opt/python/cp37-cp37m/* /usr/local/ -RUN cp -arf /opt/python/cp38-cp38/* /usr/local/ -RUN cp -arf /opt/python/cp39-cp39/* /usr/local/ - -ENV PATH=/usr/local/bin:$PATH - -# Install dependencies -RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil -RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil -RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil -RUN python3.9 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil - -# Install Auditwheel - not available on Python 2 -RUN python3.6 -m pip install --ignore-installed auditwheel -RUN python3.7 -m pip install --ignore-installed auditwheel -RUN python3.8 -m pip install --ignore-installed auditwheel -RUN python3.9 -m pip install --ignore-installed auditwheel - -# install boost -RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz --no-check-certificate >/dev/null 2>&1 -RUN tar xfz boost_1_71_0.tar.gz -# https://github.com/boostorg/build/issues/468 -RUN cd boost_1_71_0 && ./bootstrap.sh -RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install - -RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' -RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' -RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' -RUN python3.9 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' - -# install node -RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash - -RUN yum install -y nodejs - -RUN npm install --global yarn -RUN yarn --version - -# install flatbuffers -RUN mkdir -p /usr/local \ - && cd /usr/local \ - && git clone https://github.com/google/flatbuffers.git \ - && cd flatbuffers \ - && cmake -G "Unix Makefiles" \ - && make \ - && cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include \ - && ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc \ - && chmod +x /usr/local/flatbuffers/flatc diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile deleted file mode 100644 index 735ce07b61..0000000000 --- a/docker/python/manylinux2014/Dockerfile +++ /dev/null @@ -1,80 +0,0 @@ -# ***************************************************************************** -# -# Copyright (c) 2020, the Perspective Authors. -# -# This file is part of the Perspective library, distributed under the terms of -# the Apache License 2.0. The full license can be found in the LICENSE file. -# - -FROM quay.io/pypa/manylinux2014_x86_64 -RUN yum -y install rapidjson-devel sudo wget - -# Build new cmake -RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q -RUN mkdir /opt/cmake -RUN printf "y\nn\n" | sh cmake-3.15.4-Linux-x86_64.sh --prefix=/opt/cmake > /dev/null -RUN rm -fr cmake*.sh /opt/cmake/doc -RUN rm -fr /usr/local/bin/cmake -RUN rm -fr /usr/local/bin/ctest -RUN rm -fr /opt/cmake/bin/cmake-gui -RUN rm -fr /opt/cmake/bin/ccmake -RUN rm -fr /opt/cmake/bin/cpack -RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -RUN ln -s /opt/cmake/bin/ctest /usr/local/bin/ctest - -# Remove auditwheel for reinstall -RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel -RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel -RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel -RUN rm -rf /opt/python/cp39-cp39/bin/auditwheel - - -# Copy assets -RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/ -RUN cp -arf /opt/python/cp37-cp37m/* /usr/local/ -RUN cp -arf /opt/python/cp38-cp38/* /usr/local/ -RUN cp -arf /opt/python/cp39-cp39/* /usr/local/ - -ENV PATH=/usr/local/bin:$PATH - -# Install dependencies -RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil -RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil -RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil -RUN python3.9 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil - -# Install Auditwheel - not available on Python 2 -RUN python3.6 -m pip install --ignore-installed auditwheel -RUN python3.7 -m pip install --ignore-installed auditwheel -RUN python3.8 -m pip install --ignore-installed auditwheel -RUN python3.9 -m pip install --ignore-installed auditwheel - -# install boost -RUN wget https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz --no-check-certificate >/dev/null 2>&1 -RUN tar xfz boost_1_71_0.tar.gz -# https://github.com/boostorg/build/issues/468 -RUN cd boost_1_71_0 && ./bootstrap.sh -RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install - -RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' -RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' -RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' -RUN python3.9 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=8.0.0' - -# install node -RUN curl -sL https://rpm.nodesource.com/setup_14.x | sudo bash - -RUN yum install -y nodejs - -RUN npm install --global yarn -RUN yarn --version - -# install flatbuffers -RUN mkdir -p /usr/local \ - && cd /usr/local \ - && git clone https://github.com/google/flatbuffers.git \ - && cd flatbuffers \ - && cmake -G "Unix Makefiles" \ - && make \ - && cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include \ - && ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc \ - && chmod +x /usr/local/flatbuffers/flatc diff --git a/docs/md/development.md b/docs/md/development.md index 4a88d7c4ba..af405d188d 100644 --- a/docs/md/development.md +++ b/docs/md/development.md @@ -138,13 +138,6 @@ yarn build `perspective-python` supports Python 3.7 and upwards. -### Docker - -[Docker](https://docs.docker.com/install/) images with pre-built development -environments are provided for the Python libraries. - -To build Perspective using Docker, select the option in `yarn setup`. - --- ## System-Specific Instructions diff --git a/scripts/_wheel_python.js b/scripts/_wheel_python.js index 3e8deca319..d899c36ff2 100644 --- a/scripts/_wheel_python.js +++ b/scripts/_wheel_python.js @@ -21,13 +21,16 @@ const { const fs = require("fs-extra"); const IS_DOCKER = process.env.PSP_DOCKER; const IS_MACOS = getarg("--macos"); -const PYTHON = python_version(); let IMAGE = "manylinux2014"; let MANYLINUX_VERSION; +let PYTHON; if (IS_DOCKER) { MANYLINUX_VERSION = manylinux_version(); - IMAGE = python_image(MANYLINUX_VERSION, PYTHON); + PYTHON = python_version(true); + IMAGE = python_image(MANYLINUX_VERSION, "xxx"); +} else { + PYTHON = python_version(); } /** @@ -57,30 +60,42 @@ try { // Create a wheel if (MANYLINUX_VERSION) { // install deps + const boost = [ + `yum -y install wget`, + `wget https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null`, + `tar xfz boost_1_71_0.tar.gz`, + "cd boost_1_71_0", + `./bootstrap.sh`, + `./b2 -j8 --with-program_options --with-filesystem --with-system install`, + `cd ..`, + ].join(" && "); + + // This always runs in docker for now so install boost + cmd += ` ${boost} && `; // These are system deps that may only be in place from pep-517/518 so // lets reinstall them to be sure - cmd += `${PYTHON} -m pip install -U 'numpy>=1.13.1' wheel twine && `; + cmd += `${PYTHON} -m pip install -U 'numpy>=1.13.1' wheel twine auditwheel && `; // remove the build folder so we completely rebuild (and pick up the // libs we just installed above, since this build method won't use // pep-517/518) - cmd += `rm -rf build/ &&`; + cmd += `rm -rf build/ && `; // now build the wheel in place - cmd += `${PYTHON} setup.py build_ext bdist_wheel`; + cmd += `${PYTHON} setup.py build_ext bdist_wheel `; // Use auditwheel on Linux - repaired wheels are in // `python/perspective/wheelhouse`. - cmd += `&& ${PYTHON} -m auditwheel -v show ./dist/*.whl && ${PYTHON} -m auditwheel -v repair -L .lib ./dist/*.whl`; + cmd += `&& ${PYTHON} -m auditwheel -v show ./dist/*.whl && ${PYTHON} -m auditwheel -v repair -L .lib ./dist/*.whl `; } else if (IS_MACOS) { // Don't need to do any cleaning here since we will reuse the cmake // cache and numpy paths from the pep-517/518 build in build_python.js - cmd += `${PYTHON} setup.py build_ext bdist_wheel`; - cmd += " && mkdir -p ./wheelhouse && cp -v ./dist/*.whl ./wheelhouse"; + cmd += `${PYTHON} setup.py build_ext bdist_wheel `; + cmd += " && mkdir -p ./wheelhouse && cp -v ./dist/*.whl ./wheelhouse "; } else { // Windows - cmd += `${PYTHON} setup.py build_ext bdist_wheel`; + cmd += `${PYTHON} setup.py build_ext bdist_wheel `; } // TODO: MacOS wheel processed with delocate segfaults on diff --git a/scripts/install_tools.js b/scripts/install_tools.js index 52b514da99..58a2264493 100644 --- a/scripts/install_tools.js +++ b/scripts/install_tools.js @@ -17,17 +17,6 @@ try { process.chdir("boost_1_71_0"); execute`./bootstrap.sh`; execute`./b2 -j8 --with-program_options --with-filesystem --with-system install `; - - console.log("-- Installing Flatbuffers"); - execute`mkdir -p /usr/local`; - process.chdir("/usr/local"); - execute`git clone --depth 1 --branch v1.12.1 https://github.com/google/flatbuffers.git`; - process.chdir("/usr/local/flatbuffers"); - execute`cmake -G "Unix Makefiles"`; - execute`make`; - execute`cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include`; - execute`ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc`; - execute`chmod +x /usr/local/flatbuffers/flatc`; } } catch (e) { console.log(e.message); diff --git a/scripts/script_utils.js b/scripts/script_utils.js index 5502ed1fa9..09775735df 100644 --- a/scripts/script_utils.js +++ b/scripts/script_utils.js @@ -262,14 +262,14 @@ const getarg = (exports.getarg = function (flag, ...args) { * execute`${docker()} echo "Hello from Docker"`; */ exports.docker = function docker(image = "puppeteer") { - console.log(`-- Creating perspective/${image} docker image`); + console.log(`-- Creating quay.io/pypa/${image}_x86_64 docker image`); const IS_WRITE = getarg("--write") || process.env.WRITE_TESTS; const CPUS = parseInt(process.env.PSP_CPU_COUNT); const PACKAGE = process.env.PACKAGE; const CWD = process.cwd(); const IS_CI = getarg("--ci"); const IS_MANYLINUX = image.indexOf("manylinux") > -1 ? true : false; - const IMAGE = `perspective/${image}`; + const IMAGE = `quay.io/pypa/${image}_x86_64`; let env_vars = bash`-eWRITE_TESTS=${IS_WRITE} \ -ePACKAGE="${PACKAGE}"`; let flags = IS_CI ? bash`--rm` : bash`--rm -it`; @@ -292,19 +292,38 @@ exports.docker = function docker(image = "puppeteer") { * * @returns {string} The python version to use */ -exports.python_version = function python_version() { - if (process.env.PYTHON_VERSION) { - return `python${process.env.PYTHON_VERSION}`; - } else if (getarg("--python310")) { - return "python3.10"; - } else if (getarg("--python39")) { - return "python3.9"; - } else if (getarg("--python38")) { - return "python3.8"; - } else if (getarg("--python37")) { - return "python3.7"; +exports.python_version = function python_version(manylinux) { + if (manylinux) { + if (process.env.PYTHON_VERSION) { + const v = process.env.PYTHON_VERSION.replace(".", ""); + return `/opt/python/cp${v}-cp${v}${ + v === "37" || v === "36" ? "m" : "" + }/bin/python`; + } else if (getarg("--python310")) { + return `/opt/python/cp310-cp310/bin/python`; + } else if (getarg("--python39")) { + return `/opt/python/cp39-cp39/bin/python`; + } else if (getarg("--python38")) { + return `/opt/python/cp38-cp38/bin/python`; + } else if (getarg("--python37")) { + return `/opt/python/cp37-cp37m/bin/python`; + } else { + return `/opt/python/cp37-cp37m/bin/python`; + } } else { - return "python3"; + if (process.env.PYTHON_VERSION) { + return `python${process.env.PYTHON_VERSION}`; + } else if (getarg("--python310")) { + return "python3.10"; + } else if (getarg("--python39")) { + return "python3.9"; + } else if (getarg("--python38")) { + return "python3.8"; + } else if (getarg("--python37")) { + return "python3.7"; + } else { + return "python3"; + } } }; @@ -319,13 +338,7 @@ exports.python_image = function python_image(image = "", python = "") { console.log( `-- Getting image for image: '${image}' and python: '${python}'` ); - if (python == "python2") { - if (image == "manylinux2014") { - throw "Python2 not supported for manylinux2014"; - } else { - return "manylinux2010"; - } - } + return `${image}`; };