Build wheels for the release with GitHub actions (#11752)

This allows us to finish our migration from Travis to GitHub actions. ### Building pants.pex Note that we do not migrate over the building of pants.pex. To save on CI and S3 resources, we no longer want to deploy unstable PEXes. While unstable wheel files are consumed downstream with the `PANTS_SHA` feature that allows running from any arbitrary commit, that is not wired up to the PEX. The anticipated usage of unstable PEXes is too low to justify the cost. We do still want to release stable PEXes for now, but to facilitate landing this migration, we will have releases run `build-support/bin/release.sh -p` locally on their machine and then upload the artifact to GitHub releases, similar to the old workflow for Pex releases. This can be automated in a followup. ### macOS version We now build on macOS 10.15 instead of 10.11. This means that earlier macOS users will not be able to use the latest Pants releases, as announced in https://groups.google.com/u/1/g/pants-devel/c/Bfk4e_9JiT4.
pantsbuild · Apr 1, 2021 · e40cf56 · e40cf56
1 parent bb3fa05
commit e40cf56
Show file tree

Hide file tree

Showing 23 changed files with 257 additions and 1,435 deletions.
diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml
@@ -7,6 +7,7 @@ env:
   PANTS_CONFIG_FILES: +['pants.ci.toml']
   PANTS_REMOTE_CACHE_READ: 'true'
   PANTS_REMOTE_CACHE_WRITE: 'true'
+  RUST_BACKTRACE: all
 jobs:
   bootstrap_pants_linux:
     name: Bootstrap Pants, test+lint Rust (Linux)
@@ -131,7 +132,7 @@ jobs:
         python-version:
         - 3.8.3
   bootstrap_pants_macos:
-    name: Bootstrap Pants, test Rust (MacOS)
+    name: Bootstrap Pants, test Rust (macOS)
     runs-on: macos-10.15
     steps:
     - name: Check out code
@@ -356,7 +357,7 @@ jobs:
   test_python_macos:
     env:
       ARCHFLAGS: -arch x86_64
-    name: Test Python (MacOS)
+    name: Test Python (macOS)
     needs: bootstrap_pants_macos
     runs-on: macos-10.15
     steps:

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -7,6 +7,7 @@ env:
   PANTS_CONFIG_FILES: +['pants.ci.toml']
   PANTS_REMOTE_CACHE_READ: 'true'
   PANTS_REMOTE_CACHE_WRITE: 'true'
+  RUST_BACKTRACE: all
 jobs:
   bootstrap_pants_linux:
     name: Bootstrap Pants, test+lint Rust (Linux)
@@ -131,7 +132,7 @@ jobs:
         python-version:
         - 3.7.10
   bootstrap_pants_macos:
-    name: Bootstrap Pants, test Rust (MacOS)
+    name: Bootstrap Pants, test Rust (macOS)
     runs-on: macos-10.15
     steps:
     - name: Check out code
@@ -231,6 +232,131 @@ jobs:
       matrix:
         python-version:
         - 3.7.10
+  build_wheels_linux:
+    container: quay.io/pypa/manylinux2014_x86_64:latest
+    name: Build wheels and fs_util (Linux)
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 10
+    - if: github.event_name == 'push'
+      name: Get commit message for branch builds
+      run: 'echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+
+        echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV
+
+        echo "EOF" >> $GITHUB_ENV
+
+        '
+    - if: github.event_name == 'pull_request'
+      name: Get commit message for PR builds
+      run: 'echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+
+        echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV
+
+        echo "EOF" >> $GITHUB_ENV
+
+        '
+    - name: Install rustup
+      run: 'curl --proto ''=https'' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s --
+        -v -y --default-toolchain none
+
+        echo "PATH=${PATH}:${HOME}/.cargo/bin" >> $GITHUB_ENV
+
+        '
+    - name: Expose Pythons
+      run: echo "PATH=${PATH}:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin"
+        >> $GITHUB_ENV
+    - if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]')
+      name: Build wheels and fs_util
+      run: '[[ "${GITHUB_EVENT_NAME}" == "pull_request" ]] && export MODE=debug
+
+        ./build-support/bin/release.sh -n
+
+        USE_PY38=true ./build-support/bin/release.sh -n
+
+        ./build-support/bin/release.sh -f
+
+        '
+    - env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      if: github.event_name == 'push'
+      name: Deploy to S3
+      run: ./build-support/bin/deploy_to_s3.py
+  build_wheels_macos:
+    name: Build wheels and fs_util (macOS)
+    runs-on: macos-10.15
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 10
+    - if: github.event_name == 'push'
+      name: Get commit message for branch builds
+      run: 'echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+
+        echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV
+
+        echo "EOF" >> $GITHUB_ENV
+
+        '
+    - if: github.event_name == 'pull_request'
+      name: Get commit message for PR builds
+      run: 'echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+
+        echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV
+
+        echo "EOF" >> $GITHUB_ENV
+
+        '
+    - name: Expose Pythons
+      uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe
+    - name: Cache Rust toolchain
+      uses: actions/cache@v2
+      with:
+        key: ${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}
+        path: '~/.rustup/toolchains/1.49.0-*
+
+          ~/.rustup/update-hashes
+
+          ~/.rustup/settings.toml
+
+          '
+    - name: Cache Cargo
+      uses: actions/cache@v2
+      with:
+        key: '${{ runner.os }}-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'')
+          }}
+
+          '
+        path: '~/.cargo/registry
+
+          ~/.cargo/git
+
+          '
+        restore-keys: '${{ runner.os }}-cargo-${{ hashFiles(''rust-toolchain'') }}-
+
+          '
+    - if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]')
+      name: Build wheels and fs_util
+      run: '[[ "${GITHUB_EVENT_NAME}" == "pull_request" ]] && export MODE=debug
+
+        ./build-support/bin/release.sh -n
+
+        USE_PY38=true ./build-support/bin/release.sh -n
+
+        ./build-support/bin/release.sh -f
+
+        '
+    - env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      if: github.event_name == 'push'
+      name: Deploy to S3
+      run: ./build-support/bin/deploy_to_s3.py
   lint_python:
     name: Lint Python
     needs: bootstrap_pants_linux
@@ -356,7 +482,7 @@ jobs:
   test_python_macos:
     env:
       ARCHFLAGS: -arch x86_64
-    name: Test Python (MacOS)
+    name: Test Python (macOS)
     needs: bootstrap_pants_macos
     runs-on: macos-10.15
     steps:

diff --git a/.gitignore b/.gitignore
@@ -44,8 +44,6 @@ out/
 /build-support/twine-deps.venv/
 /dist/
 /lib/
-# This is created by the very handy `build-support/bin/get_failing_travis_targets_for_pr.sh`.
-/logs/
 arc.sh
 BUILD.release*
 /pants.pex