pyopenms-wheels-and-packages #1219
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This builds python wheels on manylinux, macOS and Windows and uploads them | |
# to public pypi for release and for nightlies also to pypi.cs.uni-tuebingen.de | |
name: pyopenms-wheels-and-packages | |
# Controls when the action will run. Triggers the workflow on push or pull request | |
# events but only for the master branch | |
on: | |
workflow_dispatch: | |
inputs: | |
upload-to-pypi: | |
type: boolean | |
description: actually upload the release package to pypi.org | |
default: false | |
push: | |
tags: | |
- 'Release*' | |
branches: [ nightly, py_nightly_test ] | |
# A workflow run is made up of one or more jobs that can run sequentially or in parallel | |
jobs: | |
build-win: | |
runs-on: windows-latest | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
path: OpenMS | |
- name: Setup cmake | |
uses: jwlawson/actions-setup-cmake@v2 | |
with: | |
cmake-version: '3.25.x' | |
- name: Reduce PATH # This fixes a bug described in https://github.com/facebook/watchman/commit/59bcfbf91d2a24ab580b8a78e384f133cc202383 | |
run: echo "DISTUTILS_USE_SDK=1" | |
- name: Get number of CPU cores | |
uses: SimenB/github-actions-cpu-cores@v2 | |
id: cpu-cores | |
- name: Install Qt | |
uses: jurplel/install-qt-action@v4 | |
with: | |
version: '6.*' | |
host: 'windows' # default: win64_msvc2017_64 | |
target: 'desktop' | |
install-deps: 'true' | |
archives: 'qtsvg qtbase' | |
cache: 'false' | |
- name: Install Miniconda | |
shell: bash # uses git bash on windows | |
run: | | |
MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S //D=$HOME/miniconda3 | |
- name: Download contrib build from archive (Windows) | |
shell: bash | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
run: | | |
cd OpenMS/contrib | |
# Download the file using the URL fetched from GitHub | |
gh release download -R OpenMS/contrib --pattern 'contrib_build-Windows.tar.gz' | |
# Extract the archive | |
7z x -so contrib_build-Windows.tar.gz | 7z x -si -ttar | |
rm contrib_build-Windows.tar.gz | |
ls -la | |
# https://github.com/marketplace/actions/visual-studio-shell | |
- name: Set up Visual Studio shell | |
uses: egor-tensin/vs-shell@v2 | |
with: | |
arch: x64 | |
- name: Setup conda paths | |
shell: bash | |
run: | | |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH | |
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH | |
- name: Build on Windows | |
shell: bash | |
run: | | |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json) | |
CMAKE_PREFIX_PATH="$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" | |
#mkdir bld | |
#pushd bld | |
# TODO: set generator via variable, then we can share this step | |
# cmake --version | |
# cmake -G "Visual Studio 17 2022" -A x64 -DOPENMS_CONTRIB_LIBS="$GITHUB_WORKSPACE/OpenMS/contrib" -DCMAKE_PREFIX_PATH="$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" ../OpenMS | |
# Note: multiple --targets only supported by CMake 3.15+ | |
# cmake --build . --config Release --target OpenMS | |
#ctest --progress --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake | |
export CXXFLAGS="-D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR $CXXFLAGS" | |
mkdir pyopenms_whls | |
# The growing PATH leads to a failure in the last iteration of the pyopenms build process. | |
# We will store the PATH variable in the beginning and reset it with every iteration. | |
RESET_PATH=$PATH | |
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do | |
# clean previous pyopenms libs | |
find . -name "_pyopenms*.pyd" -exec rm -rf {} \; | |
# reset the path | |
PATH=$RESET_PATH | |
# create and activate conda environment | |
py=$(echo "$py" | tr -d " \r\n") | |
pynodot=$(echo "$py" | tr -d ".") | |
conda create -n pyoms-bld-"${pynodot}" python="${py}" | |
source activate pyoms-bld-"${pynodot}" | |
# set current python executable | |
CURRENT_PYTHON_EXECUTABLE=$(which python) | |
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip | |
# pip install all the stuff | |
${CURRENT_PYTHON_EXECUTABLE} -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt | |
# Get the Python root directory | |
export Python_ROOT_DIR=$(conda info --json | jq -r '.sys.prefix') | |
echo "Python root directory: $Python_ROOT_DIR" | |
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive | |
# build pyopenms distribution | |
ctest --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake --config Release | |
echo "Current dir is `pwd`" | |
echo "wheels are in `find . | grep whl`" | |
echo "current ls is `ls -la`" | |
# copy to directory | |
cp OpenMS/bld/pyOpenMS/dist/*.whl pyopenms_whls/ | |
# clean up | |
#conda deactivate | |
#conda remove --name "pyoms-bld-${pynodot}" --all | |
done | |
env: | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CMAKE_GENERATOR: "Visual Studio 17 2022" | |
CMAKE_GENERATOR_PLATFORM: "x64" | |
OPENMS_CONTRIB_LIBS: "${{ github.workspace }}/OpenMS/contrib" | |
#CMAKE_PREFIX_PATH: "$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" #TODO move this to env | |
BUILD_TYPE: "Release" | |
CI_PROVIDER: "GitHub-Actions" | |
CMAKE_CCACHE_EXE: "ccache" | |
CCACHE_BASEDIR: ${{ github.workspace }} | |
CCACHE_DIR: ${{ github.workspace }}/.ccache | |
CCACHE_COMPRESS: true | |
CCACHE_COMPRESSLEVEL: 12 | |
CCACHE_MAXSIZE: 400M | |
PYOPENMS: "ON" | |
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }} | |
ENABLE_TUTORIALS: "OFF" | |
ENABLE_DOCS: "OFF" | |
ENABLE_PIPELINE_TESTING: "OFF" | |
ENABLE_STYLE_TESTING: "OFF" | |
ENABLE_TOPP_TESTING: "OFF" | |
ENABLE_CLASS_TESTING: "OFF" | |
- name: Run Unit Tests | |
shell: bash | |
run: | | |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake -C Release | |
env: | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CI_PROVIDER: "GitHub-Actions" | |
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" | |
BUILD_TYPE: "Release" | |
# The rest of the vars should be saved in the CMakeCache | |
- uses: actions/upload-artifact@v4 | |
name: Upload artifacts | |
with: | |
name: ${{ runner.os }}-${{ runner.arch }}-wheels | |
path: pyopenms_whls | |
build-macos-arm: | |
runs-on: macos-latest | |
env: | |
SYSTEM_VERSION_COMPAT: 0 # workaround for mac see https://github.com/actions/setup-python/issues/469 for details. | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
path: OpenMS | |
- name: Get number of CPU cores | |
uses: SimenB/github-actions-cpu-cores@v2 | |
id: cpu-cores | |
- name: Setup cmake | |
uses: jwlawson/actions-setup-cmake@v2 | |
with: | |
cmake-version: '3.25.x' | |
- name: Install Miniconda | |
shell: bash | |
run: | | |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-arm64.sh | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3 | |
- name: Setup conda paths | |
shell: bash | |
run: | | |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH | |
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH | |
- name: Force update all brew python | |
run: | | |
brew update | |
# GHA messes with links https://github.com/actions/setup-python/issues/577 so we just force replace their installs' with brew's | |
for pkg in $(brew list --formula | grep -E '^python(@[0-9]+\.[0-9]+)?$'); do | |
brew upgrade --force --overwrite "$pkg" | |
done | |
brew cleanup | |
- name: Install build tools | |
run: brew install --force --overwrite autoconf automake libtool | |
- name: Install contrib packages from brew | |
run: | | |
## Needed for Qt. Install before to overwrite the default softlinks on the GH runners | |
brew install --force --overwrite --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache | |
brew install --force --overwrite libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@6 libomp | |
echo "cmake_prefix=$(brew --prefix qt@6)/lib/cmake;$(brew --prefix qt@6)" >> $GITHUB_OUTPUT | |
echo "Qt6_DIR=$(brew --prefix qt@6)/lib/cmake/Qt6" >> $GITHUB_ENV | |
if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then | |
brew install --force --overwrite --quiet doxygen ghostscript graphviz | |
fi | |
- name: Build on macos arm | |
run: | | |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json) | |
# export CC=clang | |
# export CXX=clang++ | |
# export MACOSX_DEPLOYMENT_TARGET=12 | |
# Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following | |
# options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e | |
export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS" | |
export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS" | |
# mkdir bld | |
# pushd bld | |
# Use -DCMAKE_FIND_DEBUG_MODE=ON for debug | |
# export OpenMP_ROOT="$(brew --prefix libomp)" | |
export CMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_DIR)/lib/cmake;${Qt6_DIR}" | |
#ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake | |
#cmake -DCMAKE_BUILD_TYPE="Release" -DCMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" -DCMAKE_OSX_DEPLOYMENT_TARGET=12 -DBOOST_USE_STATIC=OFF ../OpenMS | |
#make -j${{ steps.cpu-cores.outputs.count }} OpenMS | |
mkdir pyopenms_whls | |
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do | |
# clean previous pyopenms libs | |
# do at the top of the python version loop so that we leave the final libraries for testing | |
find . -name "_pyopenms*.so" -exec rm -rf {} \; | |
py=$(echo "$py" | tr -d " \n") | |
pynodot=$(echo "$py" | tr -d ".") | |
conda create -n pyoms-bld-"${pynodot}" python="${py}" | |
source activate pyoms-bld-"${pynodot}" | |
# set current python executable | |
CURRENT_PYTHON_EXECUTABLE=$(which python) | |
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip | |
# pip install all the stuff | |
${CURRENT_PYTHON_EXECUTABLE} -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt | |
# Get the Python root directory | |
export Python_ROOT_DIR=$(conda info --json | jq -r '.sys.prefix') | |
echo "Python root directory: $Python_ROOT_DIR" | |
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive | |
# build pyopenms distribution (macOS) | |
ctest --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake | |
# TODO REMOVE DEBUGGING | |
echo "Current dir is `pwd`" | |
echo "wheels are in `find . | grep whl`" | |
echo "current ls is `ls -la`" | |
# cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} . | |
# make -j${{ steps.cpu-cores.outputs.count }} pyopenms | |
# copy to directory | |
cp OpenMS/bld/pyOpenMS/dist/*.whl pyopenms_whls/ | |
# clean up | |
# conda deactivate | |
# conda remove --name pyoms-bld-"${pynodot}" --all | |
done | |
env: | |
CC: "clang" | |
CXX: "clang++" | |
BOOST_USE_STATIC: "OFF" #TODO check if necessary? | |
MACOSX_DEPLOYMENT_TARGET: "12" | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CMAKE_GENERATOR: "Ninja" | |
OPENMS_CONTRIB_LIBS: "$GITHUB_WORKSPACE/OpenMS/contrib" | |
BUILD_TYPE: "Release" | |
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" #TODO make this more expressive | |
BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}" | |
CI_PROVIDER: "GitHub-Actions" | |
CMAKE_CCACHE_EXE: "ccache" | |
CCACHE_BASEDIR: ${{ github.workspace }} | |
CCACHE_DIR: ${{ github.workspace }}/.ccache | |
CCACHE_COMPRESS: true | |
CCACHE_COMPRESSLEVEL: 12 | |
CCACHE_MAXSIZE: 400M | |
# MT_ENABLE_OPENMP: "ON" | |
PYOPENMS: "ON" | |
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }} | |
ENABLE_TUTORIALS: "OFF" | |
ENABLE_DOCS: "OFF" | |
ENABLE_PIPELINE_TESTING: "OFF" | |
ENABLE_STYLE_TESTING: "OFF" | |
ENABLE_TOPP_TESTING: "OFF" | |
ENABLE_CLASS_TESTING: "OFF" | |
- name: Run Unit Tests | |
shell: bash | |
run: | | |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake | |
env: | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CI_PROVIDER: "GitHub-Actions" | |
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" | |
# The rest of the vars should be saved in the CMakeCache | |
- uses: actions/upload-artifact@v4 | |
name: Upload artifacts | |
if: always() | |
with: | |
name: ${{ runner.os }}-buildfiles | |
path: | | |
CMakeCache.txt | |
pyOpenMS/env.py | |
- uses: actions/upload-artifact@v4 | |
name: Upload wheels | |
with: | |
name: ${{ runner.os }}-${{ runner.arch }}-wheels | |
path: pyopenms_whls | |
build-macos: | |
runs-on: macos-13 | |
env: | |
SYSTEM_VERSION_COMPAT: 0 # workaround for mac see https://github.com/actions/setup-python/issues/469 for details. | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
path: OpenMS | |
- name: Get number of CPU cores | |
uses: SimenB/github-actions-cpu-cores@v2 | |
id: cpu-cores | |
- name: Setup cmake | |
uses: jwlawson/actions-setup-cmake@v2 | |
with: | |
cmake-version: '3.25.x' | |
- name: Install Miniconda | |
shell: bash | |
run: | | |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3 | |
- name: Setup conda paths | |
shell: bash | |
run: | | |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH | |
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH | |
- name: Force update all brew python | |
run: | | |
brew update | |
# GHA messes with links https://github.com/actions/setup-python/issues/577 so we just force replace their installs' with brew's | |
for pkg in $(brew list --formula | grep -E '^python(@[0-9]+\.[0-9]+)?$'); do | |
brew upgrade --force --overwrite "$pkg" | |
done | |
brew cleanup | |
- name: Install build tools | |
run: brew install --force --overwrite autoconf automake libtool | |
- name: Install contrib packages from brew | |
run: | | |
## Update the package lists for Brew | |
## Needed for Qt. Install before to overwrite the default softlinks on the GH runners | |
brew install --force --overwrite --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache | |
brew install --force --overwrite libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@6 libomp | |
echo "cmake_prefix=$(brew --prefix qt@6)/lib/cmake;$(brew --prefix qt@6)" >> $GITHUB_OUTPUT | |
echo "Qt6_DIR=$(brew --prefix qt@6)/lib/cmake/Qt6" >> $GITHUB_ENV | |
if [[ "${{ steps.set-vars.outputs.pkg_type }}" != "none" ]]; then | |
brew install --force --overwrite --quiet doxygen ghostscript graphviz | |
fi | |
- name: Build on macos intel | |
run: | | |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json) | |
export CC=clang | |
export CXX=clang++ | |
export MACOSX_DEPLOYMENT_TARGET=12 | |
# Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following | |
# options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e | |
export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS" | |
export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS" | |
# export OpenMP_ROOT="$(brew --prefix libomp)" | |
export CMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" | |
#mkdir bld | |
#pushd bld | |
# Use -DCMAKE_FIND_DEBUG_MODE=ON for debug | |
#cmake -DCMAKE_BUILD_TYPE="Release" -DCMAKE_PREFIX_PATH="$(brew --prefix libomp);$(echo $Qt6_Dir)/lib/cmake;${Qt6_Dir}" -DCMAKE_OSX_DEPLOYMENT_TARGET=12 -DBOOST_USE_STATIC=OFF ../OpenMS | |
#make -j${{ steps.cpu-cores.outputs.count }} OpenMS | |
mkdir pyopenms_whls | |
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do | |
# clean previous pyopenms libs | |
# do at the top of the python version loop so that we leave the final libraries for testing | |
find . -name "_pyopenms*.so" -exec rm -rf {} \; | |
py=$(echo "$py" | tr -d " \n") | |
pynodot=$(echo "$py" | tr -d ".") | |
conda create -n pyoms-bld-"${pynodot}" python="${py}" | |
source activate pyoms-bld-"${pynodot}" | |
# set current python executable | |
CURRENT_PYTHON_EXECUTABLE=$(which python) | |
${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip | |
# pip install all the stuff | |
${CURRENT_PYTHON_EXECUTABLE} -m pip install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt | |
# Get the Python root directory | |
export Python_ROOT_DIR=$(conda info --json | jq -r '.sys.prefix') | |
echo "Python root directory: $Python_ROOT_DIR" | |
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive | |
# build pyopenms distribution (macOS) | |
ctest --output-on-failure -VV -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake | |
# TODO REMOVE DEBUGGING | |
echo "Current dir is `pwd`" | |
echo "wheels are in `find . | grep whl`" | |
echo "current ls is `ls -la`" | |
# cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON -DPY_NUM_THREADS=${{ steps.cpu-cores.outputs.count }} . | |
# make -j${{ steps.cpu-cores.outputs.count }} pyopenms | |
# copy to directory | |
cp OpenMS/bld/pyOpenMS/dist/*.whl pyopenms_whls/ | |
# clean up | |
#conda deactivate | |
#conda remove --name pyoms-bld-"${pynodot}" --all | |
# clean previous pyopenms libs | |
#find . -name "_pyopenms*.so" -exec rm -rf {} \; | |
done | |
env: | |
CC: "clang" | |
CXX: "clang++" | |
MACOSX_DEPLOYMENT_TARGET: "12" | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CMAKE_GENERATOR: "Ninja" | |
OPENMS_CONTRIB_LIBS: "$GITHUB_WORKSPACE/OpenMS/contrib" | |
BOOST_USE_STATIC: "OFF" | |
BUILD_TYPE: "Release" | |
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" #TODO make this more expressive | |
BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}" | |
CI_PROVIDER: "GitHub-Actions" | |
CMAKE_CCACHE_EXE: "ccache" | |
CCACHE_BASEDIR: ${{ github.workspace }} | |
CCACHE_DIR: ${{ github.workspace }}/.ccache | |
CCACHE_COMPRESS: true | |
CCACHE_COMPRESSLEVEL: 12 | |
CCACHE_MAXSIZE: 400M | |
#MT_ENABLE_OPENMP: "ON" | |
PYOPENMS: "ON" | |
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }} | |
ENABLE_TUTORIALS: "OFF" | |
ENABLE_DOCS: "OFF" | |
ENABLE_STYLE_TESTING: "OFF" | |
ENABLE_PIPELINE_TESTING: "OFF" | |
ENABLE_TOPP_TESTING: "OFF" | |
ENABLE_CLASS_TESTING: "OFF" | |
- name: Test | |
shell: bash | |
run: | | |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake | |
env: | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CI_PROVIDER: "GitHub-Actions" | |
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" | |
# The rest of the vars should be saved in the CMakeCache | |
- uses: actions/upload-artifact@v4 | |
name: Upload wheels | |
with: | |
name: ${{ runner.os }}-${{ runner.arch }}-wheels | |
path: pyopenms_whls | |
build-lnx: | |
runs-on: [ubuntu-latest] | |
container: ghcr.io/openms/contrib_manylinux_2_34:latest | |
steps: | |
# Cancels older builds if still running | |
- uses: rokroskar/workflow-run-cleanup-action@master | |
env: | |
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" | |
# Disable warning for this line | |
if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'" # noqa | |
- uses: actions/checkout@v4 | |
name: Checkout sources | |
with: | |
path: OpenMS | |
- name: Get number of CPU cores | |
uses: SimenB/github-actions-cpu-cores@v2 | |
id: cpu-cores | |
- name: Setup cmake | |
uses: jwlawson/actions-setup-cmake@v2 | |
with: | |
cmake-version: '3.25.x' | |
- name: Build on manylinux for broad compatibility | |
shell: bash | |
run: | | |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json) | |
yum install -y jq | |
yum install -y ninja-build # Install Ninja for build files | |
cmake --version | |
mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/ | |
mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/before_fix/ | |
LD_OLD_LIBRARY_PATH=$LD_LIBRARY_PATH | |
# compile and configure OpenMS | |
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do | |
# Check if there is already and existing build, | |
# if so: clean previous pyopenms libs, | |
# do at the top of the python version loop so that we leave the final libraries for testing | |
if [ -d "OpenMS/bld/pyOpenMS/pyopenms" ]; then | |
rm OpenMS/bld/pyOpenMS/pyopenms/_pyopenms*.so | |
fi | |
py=$(echo "$py" | tr -d " \n") | |
pynodot=$(echo "$py" | tr -d ".") | |
PYBIN=$(echo /opt/python/cp*${pynodot}) | |
# Create a venv for this python version | |
"$PYBIN/bin/python" -m venv $pynodot | |
source $pynodot/bin/activate | |
"$PYBIN/bin/pip" install --upgrade pip | |
# pip install all the stuff | |
"$PYBIN/bin/pip" install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt | |
# configure (don't copy deps since we use auditwheel) | |
export Python_ROOT_DIR=$PYBIN | |
export BUILD_NAME="pyopenms-wheels-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive | |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake | |
# ensure auditwheel can find the libraries | |
export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH:`pwd`/OpenMS/bld/lib | |
# strip the libraries before repairing | |
strip --strip-all OpenMS/bld/lib/libOpenMS.so | |
strip --strip-all OpenMS/bld/lib/libOpenSwathAlgo.so | |
pushd OpenMS/bld/pyOpenMS | |
# Bundle stripped plus external shared libraries into the wheels | |
for whl in dist/pyopenms*.whl; do | |
auditwheel repair "$whl" -w wheelhouse/ | |
done | |
mv wheelhouse/* $GITHUB_WORKSPACE/data/wheelhouse/ | |
popd | |
export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH | |
# deactivate the venv | |
deactivate | |
done | |
env: | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CMAKE_GENERATOR: "Ninja" | |
OPENMS_CONTRIB_LIBS: "/contrib-build/" | |
BUILD_TYPE: "Release" | |
BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}" | |
NO_DEPENDENCIES: "ON" | |
Python_FIND_STRATEGY: "LOCATION" | |
PYOPENMS: "ON" | |
PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }} | |
CI_PROVIDER: "GitHub-Actions" | |
CMAKE_CXX_COMPILER: "g++" | |
CMAKE_PREFIX_PATH: "/contrib-build/" | |
ENABLE_TUTORIALS: "OFF" | |
ENABLE_DOCS: "OFF" | |
ENABLE_STYLE_TESTING: "OFF" | |
ENABLE_PIPELINE_TESTING: "OFF" | |
ENABLE_TOPP_TESTING: "OFF" | |
ENABLE_CLASS_TESTING: "OFF" | |
- name: Test | |
shell: bash | |
run: | | |
ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake | |
env: | |
SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS" | |
CI_PROVIDER: "GitHub-Actions" | |
BUILD_NAME: "${{ runner.arch }}-class-pyopenms-${{ github.run_number }}" | |
# The rest of the vars should be saved in the CMakeCache | |
- uses: actions/upload-artifact@v4 | |
name: Upload artifacts | |
with: | |
name: ${{ runner.os }}-${{ runner.arch }}-wheels | |
path: data/wheelhouse/ | |
# Tests the build workflows | |
test: | |
needs: [build-win, build-macos, build-lnx, build-macos-arm] | |
runs-on: ${{ matrix.os }} | |
env: | |
SYSTEM_VERSION_COMPAT: 0 #courtesy of https://github.com/actions/setup-python/issues/469 in lieu of an actual solution. | |
strategy: | |
matrix: | |
os: [ubuntu-latest, macos-13, windows-latest, macos-latest] | |
steps: | |
- uses: actions/checkout@v4 | |
name: Checkout sources | |
with: | |
path: OpenMS | |
- name: Install Miniconda | |
shell: bash | |
run: | | |
if [[ "${{ runner.os }}" == "Linux" ]]; then | |
MINICONDA_FILENAME=Miniconda3-latest-Linux-x86_64.sh | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3 | |
elif [[ "${{ runner.os }}" == "macOS" && "${{ runner.arch }}" == "X64" ]]; then | |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3 | |
elif [[ "${{ runner.os }}" == "macOS" && "${{ runner.arch }}" == "ARM64" ]]; then | |
MINICONDA_FILENAME=Miniconda3-latest-MacOSX-arm64.sh | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3 | |
elif [[ "${{ runner.os }}" == "Windows" ]]; then | |
MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe | |
curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME" | |
./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S | |
fi | |
- name: Setup conda paths | |
shell: bash | |
run: | | |
echo "$HOME/miniconda3/bin" >> $GITHUB_PATH | |
echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH | |
- name: Download artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
name: ${{ runner.os }}-${{ runner.arch }}-wheels | |
- name: Test | |
shell: bash | |
run: | | |
PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json) | |
echo $PYTHON_VERSIONS | |
for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do | |
py=$(echo "$py" | tr -d " \r\n") | |
pynodot=$(echo "$py" | tr -d ".") | |
conda create -n pyoms-bld-$pynodot python=$py | |
source activate pyoms-bld-$pynodot | |
# Find .whl for current python version | |
CURRENT_VERSION=$(echo $py | sed "s/\.//g") # remove "." for find | |
echo "Testing python version $CURRENT_VERSION" | |
CURRENT_WHL=$(find $GITHUB_WORKSPACE -name "*cp$CURRENT_VERSION*.whl") # Probably $GITHUB_WORKSPACE | |
if [[ "$CURRENT_WHL" == '' ]]; then | |
continue | |
fi | |
echo "Found wheel $CURRENT_WHL : Testing.." | |
# pip install all the stuff | |
python -m pip install --upgrade pip | |
python -m pip install $CURRENT_WHL | |
# check if package was installed | |
python -c "import importlib.util; p_name = 'pyopenms'; package = importlib.util.find_spec(p_name); print(p_name + ' was sucessfully installed! Nice one!') if package is not None else print('Something seems to be wrong!')" | |
# run test | |
python -c "exec(\"import pyopenms\nprint(pyopenms.EmpiricalFormula(\'C6H12O6\').getMonoWeight())\")" | |
conda deactivate | |
# clean up | |
conda remove --name pyoms-bld-$pynodot --all | |
done | |
# Upload the wheels (the action only runs on ubuntu slaves) | |
publish: | |
needs: [test] # doesn't really NEED it but we want the tests to pass first | |
runs-on: ubuntu-latest | |
steps: | |
# haha github is just ridiculous.. not getting their actions to do basic stuff for years | |
- name: Download artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
name: Linux-X64-wheels | |
path: /home/runner/work/OpenMS/OpenMS/Linux-X64-wheels | |
- name: Download artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
name: Windows-X64-wheels | |
path: /home/runner/work/OpenMS/OpenMS/Windows-X64-wheels | |
- name: Download artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
name: macOS-X64-wheels | |
path: /home/runner/work/OpenMS/OpenMS/macOS-X64-wheels | |
- name: Download artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
name: macOS-ARM64-wheels | |
path: /home/runner/work/OpenMS/OpenMS/macOS-ARM64-wheels | |
- name: Merge all artifacts | |
run: | | |
mkdir wheels | |
cp -lr *-wheels/* wheels/ | |
pwd | |
ls -la wheels/ | |
- name: Publish package to nightly PyPI | |
# Upload the wheels to our pypi server for develop, nightly and release branches (unless it's explictly stated to upload it to pypi.org) | |
if: contains(github.ref, 'develop') || contains(github.ref, 'nightly') || (contains(github.ref, 'Release') && inputs.upload-to-pypi == 'false') | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
repository-url: https://pypi.cs.uni-tuebingen.de/ | |
user: openms | |
password: ${{ secrets.abiservices_pypi_pw }} | |
packages-dir: ${{ github.workspace }}/wheels | |
- name: Publish package to PyPI | |
if: contains(github.ref, 'Release') && inputs.upload-to-pypi | |
uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
user: __token__ | |
repository-url: https://pypi.org/ | |
password: ${{ secrets.pypi_api_token_release }} | |
packages-dir: ${{ github.workspace }}/wheels |