Skip to content

pyopenms-wheels-and-packages #1219

pyopenms-wheels-and-packages

pyopenms-wheels-and-packages #1219

# This builds python wheels on manylinux, macOS and Windows and uploads them
# to public pypi for release and for nightlies also to pypi.cs.uni-tuebingen.de
name: pyopenms-wheels-and-packages
# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch
on:
workflow_dispatch:
inputs:
upload-to-pypi:
type: boolean
description: actually upload the release package to pypi.org
default: false
push:
tags:
- 'Release*'
branches: [ nightly, py_nightly_test ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
build-win:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
with:
path: OpenMS
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Reduce PATH # This fixes a bug described in https://github.com/facebook/watchman/commit/59bcfbf91d2a24ab580b8a78e384f133cc202383
run: echo "DISTUTILS_USE_SDK=1"
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Install Qt
uses: jurplel/install-qt-action@v4
with:
version: '6.*'
host: 'windows' # default: win64_msvc2017_64
target: 'desktop'
install-deps: 'true'
archives: 'qtsvg qtbase'
cache: 'false'
- name: Install Miniconda
shell: bash # uses git bash on windows
run: |
MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S //D=$HOME/miniconda3
- name: Download contrib build from archive (Windows)
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
cd OpenMS/contrib
# Download the file using the URL fetched from GitHub
gh release download -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz'
# Extract the archive
7z x -so contrib_build-Windows.tar.gz | 7z x -si -ttar
rm contrib_build-Windows.tar.gz
ls -la
# https://github.com/marketplace/actions/visual-studio-shell
- name: Set up Visual Studio shell
uses: egor-tensin/vs-shell@v2
with:
arch: x64
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Build on Windows
shell: bash
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
CMAKE_PREFIX_PATH="$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}"
#mkdir bld
#pushd bld
# TODO: set generator via variable, then we can share this step
# cmake --version
# cmake -G "Visual Studio 17 2022" -A x64 -DOPENMS_CONTRIB_LIBS="$GITHUB_WORKSPACE/OpenMS/contrib" -DCMAKE_PREFIX_PATH="$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" ../OpenMS
# Note: multiple --targets only supported by CMake 3.15+
# cmake --build . --config Release --target OpenMS
#ctest --progress --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
export CXXFLAGS="-D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR $CXXFLAGS"
mkdir pyopenms_whls
# The growing PATH leads to a failure in the last iteration of the pyopenms build process.
# We will store the PATH variable in the beginning and reset it with every iteration.
RESET_PATH=$PATH
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
# clean previous pyopenms libs
find . -name "_pyopenms*.pyd" -exec rm -rf {} \;
# reset the path
PATH=$RESET_PATH
# create and activate conda environment
py=$(echo "$py" | tr -d " \r\n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-"${pynodot}" python="${py}"
source activate pyoms-bld-"${pynodot}"
# set current python executable
CURRENT_PYTHON_EXECUTABLE=$(which python)
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip
# pip install all the stuff
${CURRENT_PYTHON_EXECUTABLE} -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt
# Get the Python root directory
export Python_ROOT_DIR=$(conda info --json | jq -r '.sys.prefix')
echo "Python root directory: $Python_ROOT_DIR"
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive
# build pyopenms distribution
ctest --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake --config Release
echo "Current dir is `pwd`"
echo "wheels are in `find . | grep whl`"
echo "current ls is `ls -la`"
# copy to directory
cp OpenMS/bld/pyOpenMS/dist/*.whl pyopenms_whls/
# clean up
#conda deactivate
#conda remove --name "pyoms-bld-${pynodot}" --all
done
env:
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CMAKE_GENERATOR: "Visual Studio 17 2022"
CMAKE_GENERATOR_PLATFORM: "x64"
OPENMS_CONTRIB_LIBS: "${{ github.workspace }}/OpenMS/contrib"
#CMAKE_PREFIX_PATH: "$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" #TODO move this to env
BUILD_TYPE: "Release"
CI_PROVIDER: "GitHub-Actions"
CMAKE_CCACHE_EXE: "ccache"
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_COMPRESS: true
CCACHE_COMPRESSLEVEL: 12
CCACHE_MAXSIZE: 400M
PYOPENMS: "ON"
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }}
ENABLE_TUTORIALS: "OFF"
ENABLE_DOCS: "OFF"
ENABLE_PIPELINE_TESTING: "OFF"
ENABLE_STYLE_TESTING: "OFF"
ENABLE_TOPP_TESTING: "OFF"
ENABLE_CLASS_TESTING: "OFF"
- name: Run Unit Tests
shell: bash
run: |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake -C Release
env:
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CI_PROVIDER: "GitHub-Actions"
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}"
BUILD_TYPE: "Release"
# The rest of the vars should be saved in the CMakeCache
- uses: actions/upload-artifact@v4
name: Upload artifacts
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: pyopenms_whls
build-macos-arm:
runs-on: macos-latest
env:
SYSTEM_VERSION_COMPAT: 0 # workaround for mac see https://github.com/actions/setup-python/issues/469 for details.
steps:
- uses: actions/checkout@v4
with:
path: OpenMS
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Install Miniconda
shell: bash
run: |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-arm64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Force update all brew python
run: |
brew update
# GHA messes with links https://github.com/actions/setup-python/issues/577 so we just force replace their installs' with brew's
for pkg in $(brew list --formula | grep -E '^python(@[0-9]+\.[0-9]+)?$'); do
brew upgrade --force --overwrite "$pkg"
done
brew cleanup
- name: Install build tools
run: brew install --force --overwrite autoconf automake libtool
- name: Install contrib packages from brew
run: |
## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
brew install --force --overwrite --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
brew install --force --overwrite libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@6 libomp
echo "cmake_prefix=$(brew --prefix qt@6)/lib/cmake;$(brew --prefix qt@6)" >> $GITHUB_OUTPUT
echo "Qt6_DIR=$(brew --prefix qt@6)/lib/cmake/Qt6" >> $GITHUB_ENV
if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then
brew install --force --overwrite --quiet doxygen ghostscript graphviz
fi
- name: Build on macos arm
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
# export CC=clang
# export CXX=clang++
# export MACOSX_DEPLOYMENT_TARGET=12
# Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following
# options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e
export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS"
export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS"
# mkdir bld
# pushd bld
# Use -DCMAKE_FIND_DEBUG_MODE=ON for debug
# export OpenMP_ROOT="$(brew --prefix libomp)"
export CMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_DIR)/lib/cmake;${Qt6_DIR}"
#ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
#cmake -DCMAKE_BUILD_TYPE="Release" -DCMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" -DCMAKE_OSX_DEPLOYMENT_TARGET=12 -DBOOST_USE_STATIC=OFF ../OpenMS
#make -j${{ steps.cpu-cores.outputs.count }} OpenMS
mkdir pyopenms_whls
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
# clean previous pyopenms libs
# do at the top of the python version loop so that we leave the final libraries for testing
find . -name "_pyopenms*.so" -exec rm -rf {} \;
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-"${pynodot}" python="${py}"
source activate pyoms-bld-"${pynodot}"
# set current python executable
CURRENT_PYTHON_EXECUTABLE=$(which python)
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip
# pip install all the stuff
${CURRENT_PYTHON_EXECUTABLE} -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt
# Get the Python root directory
export Python_ROOT_DIR=$(conda info --json | jq -r '.sys.prefix')
echo "Python root directory: $Python_ROOT_DIR"
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive
# build pyopenms distribution (macOS)
ctest --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
# TODO REMOVE DEBUGGING
echo "Current dir is `pwd`"
echo "wheels are in `find . | grep whl`"
echo "current ls is `ls -la`"
# cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} .
# make -j${{ steps.cpu-cores.outputs.count }} pyopenms
# copy to directory
cp OpenMS/bld/pyOpenMS/dist/*.whl pyopenms_whls/
# clean up
# conda deactivate
# conda remove --name pyoms-bld-"${pynodot}" --all
done
env:
CC: "clang"
CXX: "clang++"
BOOST_USE_STATIC: "OFF" #TODO check if necessary?
MACOSX_DEPLOYMENT_TARGET: "12"
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CMAKE_GENERATOR: "Ninja"
OPENMS_CONTRIB_LIBS: "$GITHUB_WORKSPACE/OpenMS/contrib"
BUILD_TYPE: "Release"
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" #TODO make this more expressive
BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}"
CI_PROVIDER: "GitHub-Actions"
CMAKE_CCACHE_EXE: "ccache"
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_COMPRESS: true
CCACHE_COMPRESSLEVEL: 12
CCACHE_MAXSIZE: 400M
# MT_ENABLE_OPENMP: "ON"
PYOPENMS: "ON"
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }}
ENABLE_TUTORIALS: "OFF"
ENABLE_DOCS: "OFF"
ENABLE_PIPELINE_TESTING: "OFF"
ENABLE_STYLE_TESTING: "OFF"
ENABLE_TOPP_TESTING: "OFF"
ENABLE_CLASS_TESTING: "OFF"
- name: Run Unit Tests
shell: bash
run: |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake
env:
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CI_PROVIDER: "GitHub-Actions"
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}"
# The rest of the vars should be saved in the CMakeCache
- uses: actions/upload-artifact@v4
name: Upload artifacts
if: always()
with:
name: ${{ runner.os }}-buildfiles
path: |
CMakeCache.txt
pyOpenMS/env.py
- uses: actions/upload-artifact@v4
name: Upload wheels
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: pyopenms_whls
build-macos:
runs-on: macos-13
env:
SYSTEM_VERSION_COMPAT: 0 # workaround for mac see https://github.com/actions/setup-python/issues/469 for details.
steps:
- uses: actions/checkout@v4
with:
path: OpenMS
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Install Miniconda
shell: bash
run: |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Force update all brew python
run: |
brew update
# GHA messes with links https://github.com/actions/setup-python/issues/577 so we just force replace their installs' with brew's
for pkg in $(brew list --formula | grep -E '^python(@[0-9]+\.[0-9]+)?$'); do
brew upgrade --force --overwrite "$pkg"
done
brew cleanup
- name: Install build tools
run: brew install --force --overwrite autoconf automake libtool
- name: Install contrib packages from brew
run: |
## Update the package lists for Brew
## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
brew install --force --overwrite --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
brew install --force --overwrite libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@6 libomp
echo "cmake_prefix=$(brew --prefix qt@6)/lib/cmake;$(brew --prefix qt@6)" >> $GITHUB_OUTPUT
echo "Qt6_DIR=$(brew --prefix qt@6)/lib/cmake/Qt6" >> $GITHUB_ENV
if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then
brew install --force --overwrite --quiet doxygen ghostscript graphviz
fi
- name: Build on macos intel
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
export CC=clang
export CXX=clang++
export MACOSX_DEPLOYMENT_TARGET=12
# Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following
# options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e
export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS"
export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS"
# export OpenMP_ROOT="$(brew --prefix libomp)"
export CMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}"
#mkdir bld
#pushd bld
# Use -DCMAKE_FIND_DEBUG_MODE=ON for debug
#cmake -DCMAKE_BUILD_TYPE="Release" -DCMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" -DCMAKE_OSX_DEPLOYMENT_TARGET=12 -DBOOST_USE_STATIC=OFF ../OpenMS
#make -j${{ steps.cpu-cores.outputs.count }} OpenMS
mkdir pyopenms_whls
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
# clean previous pyopenms libs
# do at the top of the python version loop so that we leave the final libraries for testing
find . -name "_pyopenms*.so" -exec rm -rf {} \;
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-"${pynodot}" python="${py}"
source activate pyoms-bld-"${pynodot}"
# set current python executable
CURRENT_PYTHON_EXECUTABLE=$(which python)
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip
# pip install all the stuff
${CURRENT_PYTHON_EXECUTABLE} -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt
# Get the Python root directory
export Python_ROOT_DIR=$(conda info --json | jq -r '.sys.prefix')
echo "Python root directory: $Python_ROOT_DIR"
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive
# build pyopenms distribution (macOS)
ctest --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
# TODO REMOVE DEBUGGING
echo "Current dir is `pwd`"
echo "wheels are in `find . | grep whl`"
echo "current ls is `ls -la`"
# cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} .
# make -j${{ steps.cpu-cores.outputs.count }} pyopenms
# copy to directory
cp OpenMS/bld/pyOpenMS/dist/*.whl pyopenms_whls/
# clean up
#conda deactivate
#conda remove --name pyoms-bld-"${pynodot}" --all
# clean previous pyopenms libs
#find . -name "_pyopenms*.so" -exec rm -rf {} \;
done
env:
CC: "clang"
CXX: "clang++"
MACOSX_DEPLOYMENT_TARGET: "12"
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CMAKE_GENERATOR: "Ninja"
OPENMS_CONTRIB_LIBS: "$GITHUB_WORKSPACE/OpenMS/contrib"
BOOST_USE_STATIC: "OFF"
BUILD_TYPE: "Release"
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" #TODO make this more expressive
BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}"
CI_PROVIDER: "GitHub-Actions"
CMAKE_CCACHE_EXE: "ccache"
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_COMPRESS: true
CCACHE_COMPRESSLEVEL: 12
CCACHE_MAXSIZE: 400M
#MT_ENABLE_OPENMP: "ON"
PYOPENMS: "ON"
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }}
ENABLE_TUTORIALS: "OFF"
ENABLE_DOCS: "OFF"
ENABLE_STYLE_TESTING: "OFF"
ENABLE_PIPELINE_TESTING: "OFF"
ENABLE_TOPP_TESTING: "OFF"
ENABLE_CLASS_TESTING: "OFF"
- name: Test
shell: bash
run: |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake
env:
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CI_PROVIDER: "GitHub-Actions"
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}"
# The rest of the vars should be saved in the CMakeCache
- uses: actions/upload-artifact@v4
name: Upload wheels
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: pyopenms_whls
build-lnx:
runs-on: [ubuntu-latest]
container: ghcr.io/openms/contrib_manylinux_2_34:latest
steps:
# Cancels older builds if still running
- uses: rokroskar/workflow-run-cleanup-action@master
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
# Disable warning for this line
if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'" # noqa
- uses: actions/checkout@v4
name: Checkout sources
with:
path: OpenMS
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v2
id: cpu-cores
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@v2
with:
cmake-version: '3.25.x'
- name: Build on manylinux for broad compatibility
shell: bash
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
yum install -y jq
yum install -y ninja-build # Install Ninja for build files
cmake --version
mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/
mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/before_fix/
LD_OLD_LIBRARY_PATH=$LD_LIBRARY_PATH
# compile and configure OpenMS
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
# Check if there is already and existing build,
# if so: clean previous pyopenms libs,
# do at the top of the python version loop so that we leave the final libraries for testing
if [ -d "OpenMS/bld/pyOpenMS/pyopenms" ]; then
rm OpenMS/bld/pyOpenMS/pyopenms/_pyopenms*.so
fi
py=$(echo "$py" | tr -d " \n")
pynodot=$(echo "$py" | tr -d ".")
PYBIN=$(echo /opt/python/cp*${pynodot})
# Create a venv for this python version
"$PYBIN/bin/python" -m venv $pynodot
source $pynodot/bin/activate
"$PYBIN/bin/pip" install --upgrade pip
# pip install all the stuff
"$PYBIN/bin/pip" install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt
# configure (don't copy deps since we use auditwheel)
export Python_ROOT_DIR=$PYBIN
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
# ensure auditwheel can find the libraries
export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH:`pwd`/OpenMS/bld/lib
# strip the libraries before repairing
strip --strip-all OpenMS/bld/lib/libOpenMS.so
strip --strip-all OpenMS/bld/lib/libOpenSwathAlgo.so
pushd OpenMS/bld/pyOpenMS
# Bundle stripped plus external shared libraries into the wheels
for whl in dist/pyopenms*.whl; do
auditwheel repair "$whl" -w wheelhouse/
done
mv wheelhouse/* $GITHUB_WORKSPACE/data/wheelhouse/
popd
export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH
# deactivate the venv
deactivate
done
env:
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CMAKE_GENERATOR: "Ninja"
OPENMS_CONTRIB_LIBS: "/contrib-build/"
BUILD_TYPE: "Release"
BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}"
NO_DEPENDENCIES: "ON"
Python_FIND_STRATEGY: "LOCATION"
PYOPENMS: "ON"
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }}
CI_PROVIDER: "GitHub-Actions"
CMAKE_CXX_COMPILER: "g++"
CMAKE_PREFIX_PATH: "/contrib-build/"
ENABLE_TUTORIALS: "OFF"
ENABLE_DOCS: "OFF"
ENABLE_STYLE_TESTING: "OFF"
ENABLE_PIPELINE_TESTING: "OFF"
ENABLE_TOPP_TESTING: "OFF"
ENABLE_CLASS_TESTING: "OFF"
- name: Test
shell: bash
run: |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake
env:
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
CI_PROVIDER: "GitHub-Actions"
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}"
# The rest of the vars should be saved in the CMakeCache
- uses: actions/upload-artifact@v4
name: Upload artifacts
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
path: data/wheelhouse/
# Tests the build workflows
test:
needs: [build-win, build-macos, build-lnx, build-macos-arm]
runs-on: ${{ matrix.os }}
env:
SYSTEM_VERSION_COMPAT: 0 #courtesy of https://github.com/actions/setup-python/issues/469 in lieu of an actual solution.
strategy:
matrix:
os: [ubuntu-latest, macos-13, windows-latest, macos-latest]
steps:
- uses: actions/checkout@v4
name: Checkout sources
with:
path: OpenMS
- name: Install Miniconda
shell: bash
run: |
if [[ "${{ runner.os }}" == "Linux" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-Linux-x86_64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
elif [[ "${{ runner.os }}" == "macOS" && "${{ runner.arch }}" == "X64" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
elif [[ "${{ runner.os }}" == "macOS" && "${{ runner.arch }}" == "ARM64" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-arm64.sh
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
elif [[ "${{ runner.os }}" == "Windows" ]]; then
MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S
fi
- name: Setup conda paths
shell: bash
run: |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: ${{ runner.os }}-${{ runner.arch }}-wheels
- name: Test
shell: bash
run: |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
echo $PYTHON_VERSIONS
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
py=$(echo "$py" | tr -d " \r\n")
pynodot=$(echo "$py" | tr -d ".")
conda create -n pyoms-bld-$pynodot python=$py
source activate pyoms-bld-$pynodot
# Find .whl for current python version
CURRENT_VERSION=$(echo $py | sed "s/\.//g") # remove "." for find
echo "Testing python version $CURRENT_VERSION"
CURRENT_WHL=$(find $GITHUB_WORKSPACE -name "*cp$CURRENT_VERSION*.whl") # Probably $GITHUB_WORKSPACE
if [[ "$CURRENT_WHL" == '' ]]; then
continue
fi
echo "Found wheel $CURRENT_WHL : Testing.."
# pip install all the stuff
python -m pip install --upgrade pip
python -m pip install $CURRENT_WHL
# check if package was installed
python -c "import importlib.util; p_name = 'pyopenms'; package = importlib.util.find_spec(p_name); print(p_name + ' was sucessfully installed! Nice one!') if package is not None else print('Something seems to be wrong!')"
# run test
python -c "exec(\"import pyopenms\nprint(pyopenms.EmpiricalFormula(\'C6H12O6\').getMonoWeight())\")"
conda deactivate
# clean up
conda remove --name pyoms-bld-$pynodot --all
done
# Upload the wheels (the action only runs on ubuntu slaves)
publish:
needs: [test] # doesn't really NEED it but we want the tests to pass first
runs-on: ubuntu-latest
steps:
# haha github is just ridiculous.. not getting their actions to do basic stuff for years
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: Linux-X64-wheels
path: /home/runner/work/OpenMS/OpenMS/Linux-X64-wheels
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: Windows-X64-wheels
path: /home/runner/work/OpenMS/OpenMS/Windows-X64-wheels
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: macOS-X64-wheels
path: /home/runner/work/OpenMS/OpenMS/macOS-X64-wheels
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: macOS-ARM64-wheels
path: /home/runner/work/OpenMS/OpenMS/macOS-ARM64-wheels
- name: Merge all artifacts
run: |
mkdir wheels
cp -lr *-wheels/* wheels/
pwd
ls -la wheels/
- name: Publish package to nightly PyPI
# Upload the wheels to our pypi server for develop, nightly and release branches (unless it's explictly stated to upload it to pypi.org)
if: contains(github.ref, 'develop') || contains(github.ref, 'nightly') || (contains(github.ref, 'Release') && inputs.upload-to-pypi == 'false')
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://pypi.cs.uni-tuebingen.de/
user: openms
password: ${{ secrets.abiservices_pypi_pw }}
packages-dir: ${{ github.workspace }}/wheels
- name: Publish package to PyPI
if: contains(github.ref, 'Release') && inputs.upload-to-pypi
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
repository-url: https://pypi.org/
password: ${{ secrets.pypi_api_token_release }}
packages-dir: ${{ github.workspace }}/wheels