[api][ci] Introduce paimon_python_api and ci framework (#2)

apache · Aug 15, 2024 · 8e07985 · 8e07985
1 parent ae5f793
commit 8e07985
Show file tree

Hide file tree

Showing 16 changed files with 1,280 additions and 12 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,18 @@
+### Purpose
+
+<!-- Linking this pull request to the issue -->
+Linked issue: close #xxx
+
+<!-- What is the purpose of the change -->
+
+### Tests
+
+<!-- List UT and IT cases to verify this change -->
+
+### API and Format
+
+<!-- Does this change affect API or storage format -->
+
+### Documentation
+
+<!-- Does this change introduce a new feature -->
diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml
@@ -0,0 +1,41 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Check Code Style & Run Tests
+
+on:
+  push:
+  pull_request:
+    paths-ignore:
+      - 'dev/**'
+      - 'java-based-implementation/paimon-python-java-bridge/**'
+      - '**/*.md'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  lint-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run lint-python.sh
+        run: |
+          chmod +x dev/lint-python.sh
+          ./dev/lint-python.sh
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,16 @@
 .idea/
 !.idea/vcs.xml
+
+# paimon-python-java-bridge
 target
 dependency-reduced-pom.xml
+
+# Python building & testing
+.tox/
+build/
+dev/.conda
+dev/.stage.txt
+dev/download
+dev/log
+dist/
+paimon_python.egg-info/
diff --git a/dev/build-wheels.sh b/dev/build-wheels.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e -x
+
+## 1. install python env
+dev/lint-python.sh -s py_env
+
+PY_ENV_DIR=`pwd`/dev/.conda/envs
+py_env=("3.8" "3.9" "3.10" "3.11")
+## 2. install dependency
+for ((i=0;i<${#py_env[@]};i++)) do
+    echo "Installing dependencies for environment: ${py_env[i]}"
+    ${PY_ENV_DIR}/${py_env[i]}/bin/pip install -r dev/dev-requirements.txt
+done
+
+## 3. build wheels
+for ((i=0;i<${#py_env[@]};i++)) do
+    echo "Building wheel for environment: ${py_env[i]}"
+    if [[ "$(uname)" != "Darwin" ]]; then
+        # force the linker to use the older glibc version in Linux
+        export CFLAGS="-I. -include dev/glibc_version_fix.h"
+    fi
+    ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel
+done
+
+## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux
+if [[ "$(uname)" != "Darwin" ]]; then
+    echo "Converting linux_x86_64 wheel to manylinux1"
+    source `pwd`/dev/.conda/bin/activate
+    # 4.1 install patchelf
+    conda install -c conda-forge patchelf=0.11 -y
+    # 4.2 install auditwheel
+    pip install auditwheel==3.2.0
+    # 4.3 convert Linux wheel
+    for wheel_file in dist/*.whl; do
+        auditwheel repair ${wheel_file} -w dist
+        rm -f ${wheel_file}
+    done
+    source deactivate
+fi
+## see the result
+ls -al dist/
diff --git a/dev/dev-requirements.txt b/dev/dev-requirements.txt
@@ -0,0 +1,22 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+pip>=20.3
+setuptools>=18.0
+wheel
+pytest~=7.0
diff --git a/dev/install_command.sh b/dev/install_command.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+retry_times=3
+install_command="python -m pip install $@"
+${install_command}
+status=$?
+while [[ ${status} -ne 0 ]] && [[ ${retry_times} -gt 0 ]]; do
+    retry_times=$((retry_times-1))
+    # sleep 3 seconds and then reinstall.
+    sleep 3
+    ${install_command}
+    status=$?
+done
+
+exit ${status}