Skip to content

Commit

Permalink
Fix s3 deploy to use copies instead of a symlink. (#4814)
Browse files Browse the repository at this point in the history 
In addition, cleanup unused cruft surrounding bintray deploys and
re-enable native engine binary deploys for all green master commits. The
latter should be sane since s3 storage is cheap, we'll only store
changed native engine binaries in s3 (so noop on many master commits)
and transfers (downloads) will only happen for released versions of
pants as before this change.
  • Loading branch information
@jsirois
jsirois authored Aug 15, 2017
1 parent c7c0658 commit 9d21c96
Show file tree
Hide file tree
Showing 8 changed files with 127 additions and 163 deletions.
4 changes: 1 addition & 3 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,5 @@ GRTAGS
GSYMS
GTAGS

# Generated by build-support/bin/prepare-binary-deploy.sh for
# use by Travis-CI bintray deploys.
/native-engine.bintray.json
# Generated by build-support/bin/prepare-binary-deploy.sh for use by Travis-CI binary deploys.
/build-support/bin/native/s3-upload/
65 changes: 14 additions & 51 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,6 @@ env:
- PANTS_CONFIG_FILES="pants.travis-ci.ini"
- ANDROID_SDK_INSTALL_LOCATION="${HOME}/opt/android-sdk-install"
- ANDROID_HOME="$ANDROID_SDK_INSTALL_LOCATION/android-sdk-linux"
# Credentials for OSX syncing: GH_USER, GH_EMAIL, GH_TOKEN
# These are encrypted with a public key for our repo that only
# Travis-CI has the private key for. We are trusting Travis-CI
# here but no-one else.
#
# See: http://docs.travis-ci.com/user/encryption-keys/
- secure: VvwbndU++a2/iNAjk9cd67ATiipDwqcKnxDR4/J2Ik3GH10wHEDUhJ1+MK4WLhedfaOakDOEmarZQS3GwtgvCHO3knpTJuJc8d/bCfZovYuSqdi//BEv4dS7hDt6tQeJfkbBjG0T4yNjPJ3W9R9KDWCy/vj2CUm90BGg2CmxUbg=
# User for bintray deploys: BINTRAY_USER
- secure: eXGuKvbp297wi/z74jFqGyxzRDCFic9HUb0z2UYDXDmobErILZdgT0KCvqeyAx7QX/JQSp5oQAQNisE8RLrain5lXAIa2ZuswTPsh6yXGmFgwdx/X+Am7CPO27b0P5OxiJAN3kfYglN7qY+opRM1jud4anzEaTJnm7jENFfDXwk=
# Key for bintray deploys: BINTRAY_KEY
- secure: qid/ot1XIWOpNaN+RhgdJq8IEajcpHI5EFvy2ywkYHJO2hKYawyX2M4gFd0Vq8+xmeGB4MUmpPW8D8gijLi5JB+0aZ3+5JHs5r9NWkK7HVMpVnok3CywknzXBgeo+UoEQv9ugYvRr1Sm9Dj9IezAhM0tw1uS95Ap+JLbnWFB830=

before_cache:
# Ensure permissions to do the below removals, which happen with or without caching enabled.
Expand Down Expand Up @@ -58,7 +47,7 @@ matrix:
- os: osx
language: generic
env:
- SHARD="OSX Bintray Builder"
- SHARD="OSX Native Engine Binary Builder"
script:
- ./build-support/bin/native/prepare-binary-deploy.sh

Expand All @@ -73,7 +62,7 @@ matrix:
language: python
python: "2.7.13"
env:
- SHARD="Linux Bintray Builder"
- SHARD="Linux Native Engine Binary Builder"
# Use the standard python manylinux image for ideal binary compatibility.
- DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
before_install:
Expand Down Expand Up @@ -418,45 +407,19 @@ matrix:
- ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 6/7 "${SHARD}"

deploy:
# See: https://docs.travis-ci.com/user/deployment/bintray/
# TODO: Do we still need to deploy to bintray? If not we can remove this and also
# the code that generates ./native-engine.bintray.json.
- provider: bintray
# NB: This is generated in after_success in each shard above.
file: ./native-engine.bintray.json
user: ${BINTRAY_USER}
key: ${BINTRAY_KEY}
dry-run: false
on:
condition: -f ./native-engine.bintray.json
# NB: Deploys are always tagged as part of the deploy process encoded in
# `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary
# for all releases. Unfortunately, CI only runs after the release tag hits origin and so there
# will be a lag of roughly 30 minutes until a pypi release has its paired native engine version
# available on bintray for OSX and linux. This trade-off (vs releasing for all branch builds),
# helps us use bintray in a friendly way.
tags: true
repo: pantsbuild/pants
# See: https://docs.travis-ci.com/user/deployment/s3/
- provider: s3
access_key_id: AKIAIQHTQI5E42SQBSNA
secret_access_key:
secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0=
bucket: binaries.pantsbuild.org
local_dir: build-support/bin/native/s3-upload
skip_cleanup: true # Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail.
acl: public_read
on:
# TODO: Do we need this condition? If so, document why, as it superficially appears to be irrelevant.
condition: -f ./native-engine.bintray.json
# NB: Deploys are always tagged as part of the deploy process encoded in
# `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary
# for all releases. Unfortunately, CI only runs after the release tag hits origin and so there
# will be a lag of roughly 30 minutes until a pypi release has its paired native engine version
# available on s3 for OSX and linux. This trade-off (vs releasing for all branch builds),
# helps us use s3 in a friendly way.
tags: true
repo: pantsbuild/pants
provider: s3
access_key_id: AKIAIQHTQI5E42SQBSNA
secret_access_key:
secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0=
bucket: binaries.pantsbuild.org
local_dir: build-support/bin/native/s3-upload
# Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail.
skip_cleanup: true
acl: public_read
on:
branch: master
repo: pantsbuild/pants

# We accept the default travis-ci email author+committer notification
# for now which is enabled even with no `notifications` config.
Expand Down
12 changes: 12 additions & 0 deletions build-support/bin/check_native_engine_version.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ set -e

REPO_ROOT="$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../.. && pwd -P)"

# Defines:
# + CACHE_ROOT: The pants cache root dir.
# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory.
# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for
# the current OS.
# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS.
# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine
# version hash.
# Exposes:
# + calculate_current_hash: Calculates the current native engine version hash and echoes it to
# stdout.
# + bootstrap_native_code: Builds target-specific native engine binaries.
source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh"

readonly actual_native_engine_version="$(calculate_current_hash)"
Expand Down
34 changes: 22 additions & 12 deletions build-support/bin/native/bootstrap.sh
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,32 @@
#!/usr/bin/env bash

# Defines:
# + CACHE_TARGET_DIR: The directory containing all versions of the native engine for the current OS.
# + CACHE_ROOT: The pants cache root dir.
# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory.
# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for
# the current OS.
# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS.
# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine
# version hash.
# Exposes:
# + build_native_code: Builds target-specific native engine binaries.
# + calculate_current_hash: Calculates the current native engine version hash and echoes it to
# stdout.
# + bootstrap_native_code: Builds target-specific native engine binaries.

REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P)
source ${REPO_ROOT}/build-support/common.sh

# Defines:
# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7.
# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12
# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12.
# + LIB_EXTENSION: The extension of native libraries.
# + KERNEL: The lower-cased name of the kernel as reported by uname.
# + OS_NAME: The name of the OS as seen by pants.
# + OS_ID: The ID of the current OS as seen by pants.
# Exposes:
# + get_native_engine_version: Echoes the current native engine version.
# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line.
# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line.
# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line.
source ${REPO_ROOT}/build-support/bin/native/utils.sh

Expand All @@ -35,7 +44,8 @@ case "$MODE" in
esac

readonly CACHE_ROOT=${XDG_CACHE_HOME:-$HOME/.cache}/pants
readonly CACHE_TARGET_DIR=${CACHE_ROOT}/bin/native-engine/${OS_ID}
readonly NATIVE_ENGINE_CACHE_DIR=${CACHE_ROOT}/bin/native-engine
readonly NATIVE_ENGINE_CACHE_TARGET_DIR=${NATIVE_ENGINE_CACHE_DIR}/${OS_ID}

function calculate_current_hash() {
# Cached and unstaged files, with ignored files excluded.
Expand All @@ -50,12 +60,12 @@ function calculate_current_hash() {
)
}

function ensure_cffi_sources() {
function _ensure_cffi_sources() {
# N.B. Here we assume that higher level callers have already setup the pants' venv and $PANTS_SRCPATH.
PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" python "${CFFI_BOOTSTRAPPER}" "$@"
}

function ensure_build_prerequisites() {
function _ensure_build_prerequisites() {
# Control a pants-specific rust toolchain, optionally ensuring the given target toolchain is
# installed.
local readonly target=$1
Expand Down Expand Up @@ -83,11 +93,11 @@ function ensure_build_prerequisites() {
fi
}

function build_native_code() {
function _build_native_code() {
# Builds the native code, optionally taking an explicit target triple arg, and echos the path of
# the built binary.
local readonly target=$1
ensure_build_prerequisites ${target}
_ensure_build_prerequisites ${target}

local readonly cargo="${CARGO_HOME}/bin/cargo"
local readonly build_cmd="${cargo} build --manifest-path ${NATIVE_ROOT}/Cargo.toml ${MODE_FLAG}"
Expand All @@ -105,14 +115,14 @@ function bootstrap_native_code() {
# Bootstraps the native code and overwrites the native_engine_version to the resulting hash
# version if needed.
local native_engine_version="$(calculate_current_hash)"
local target_binary="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
local target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
local cffi_output_dir="${NATIVE_ROOT}/src/cffi"
local cffi_env_script="${cffi_output_dir}/${NATIVE_ENGINE_MODULE}.sh"
if [ ! -f "${target_binary}" ]
then
ensure_cffi_sources "${cffi_output_dir}"
_ensure_cffi_sources "${cffi_output_dir}"
source "${cffi_env_script}"
local readonly native_binary="$(build_native_code)"
local readonly native_binary="$(_build_native_code)"

# If bootstrapping the native engine fails, don't attempt to run pants
# afterwards.
Expand All @@ -123,7 +133,7 @@ function bootstrap_native_code() {

# Pick up Cargo.lock changes if any caused by the `cargo build`.
native_engine_version="$(calculate_current_hash)"
target_binary="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"

mkdir -p "$(dirname ${target_binary})"
cp "${native_binary}" "${target_binary}"
Expand Down
114 changes: 34 additions & 80 deletions build-support/bin/native/prepare-binary-deploy.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,108 +6,62 @@ REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P)

# Indirectly defines:
# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7.
# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12
# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12.
# + LIB_EXTENSION: The extension of native libraries.
# + KERNEL: The lower-cased name of the kernel as reported by uname.
# + OS_NAME: The name of the OS as seen by pants.
# + OS_ID: The ID of the current OS as seen by pants.
# Indirectly exposes:
# + get_native_engine_version: Echoes the current native engine version.
# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line.
# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line.
# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line.
# Defines:
# + CACHE_TARGET_DIR: The directory containing all versions of the native engine for the current OS.
# + CACHE_ROOT: The pants cache root dir.
# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory.
# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for
# the current OS.
# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS.
# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine
# version hash.
# Exposes:
# + build_native_code: Builds target-specific native engine binaries.
source ${REPO_ROOT}/build-support/bin/native/bootstrap.sh
# + calculate_current_hash: Calculates the current native engine version hash and echoes it to
# stdout.
# + bootstrap_native_code: Builds target-specific native engine binaries.
source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh"

readonly native_engine_version=$(get_native_engine_version)
readonly cached_bin_path="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"

cat << __EOF__ > ${REPO_ROOT}/native-engine.bintray.json
{
"package": {
"subject": "pantsbuild",
"repo": "bin",
"name": "native-engine",
"desc": "The pants native engine library.",
"website_url": "http://www.pantsbuild.org",
"issue_tracker_url": "https://github.com/pantsbuild/pants/issues",
"vcs_url": "https://github.com/pantsbuild/pants.git",
"licenses": ["Apache-2.0"],
"public_download_numbers": true,
"public_stats": true,
"github_use_tag_release_notes": false,
"attributes": [],
"labels": []
},
readonly s3_upload_root="${REPO_ROOT}/build-support/bin/native/s3-upload"
readonly s3_native_engine_dir="${s3_upload_root}/bin/native-engine"

"version": {
"name": "${native_engine_version}",
"desc": "The native engine at sha1: ${native_engine_version}",
"released": "$(date +'%Y-%m-%d')",
"vcs_tag": "$(git rev-parse HEAD)",
"attributes": [],
"gpgSign": false
},
"publish": true,
"files": [
__EOF__

function emit_osx_files() {
local readonly cached_bin_path="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
ensure_file_exists "${cached_bin_path}"
function prepare_chroot() {
rm -rf "${s3_upload_root}"
mkdir -p "$(dirname ${s3_native_engine_dir})"
cp -vpr "${NATIVE_ENGINE_CACHE_DIR}" "${s3_native_engine_dir}"
}

# Rust targets OSX 10.7+ as noted here: https://doc.rust-lang.org/book/getting-started.html#tier-1
for version in $(get_rust_osx_versions)
function prepare_osx_versions() {
for os_id in $(get_rust_osx_ids)
do
local cached_link_path="${cached_bin_path}.10.${version}"

if (( ${version} < ${OSX_MAX_VERSION} ))
if [ "${OS_ID}" != "${os_id}" ]
then
local sep=","
else
local sep=""
local target="${s3_native_engine_dir}/${os_id}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
mkdir -p "$(dirname ${target})"
cp -vp "${cached_bin_path}" "${target}"
fi
# It appears to be the case that upload de-dupes on includePattern keys; so we make a unique
# includePattern per uploadPattern via a symlink here per OSX version.
ln -fs "${cached_bin_path}" "${cached_link_path}"
cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json
{
"includePattern": "${cached_link_path}",
"uploadPattern": "build-support/bin/native-engine/mac/10.${version}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
}${sep}
__EOF__
done
}

function emit_linux_files() {
local readonly cached_bin_path="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
ensure_file_exists "${cached_bin_path}"
# Sanity check the locally built native engine binary exists in the first place.
ensure_file_exists "${cached_bin_path}"

cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json
{
"includePattern": "${cached_bin_path}",
"uploadPattern": "build-support/bin/native-engine/linux/x86_64/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
}
__EOF__
}
# Prepare a chroot for s3 deploy of the binary(ies).
prepare_chroot

# Maybe add copies of the mac native engine binary for the other supported OSX versions.
if [ "${OS_NAME}" == "mac" ]
then
emit_osx_files
else
emit_linux_files
fi

cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json
]
}
__EOF__

# Prepare a chroot for s3 deploy of the artifacts created above.
S3_UPLOAD_ROOT=${REPO_ROOT}/build-support/bin/native/s3-upload
rm -rf ${S3_UPLOAD_ROOT}
mkdir -p ${S3_UPLOAD_ROOT}/bin
ln -s ${CACHE_ROOT}/bin/native-engine ${S3_UPLOAD_ROOT}/bin/native-engine
prepare_osx_versions
fi
Loading

0 comments on commit 9d21c96

Please sign in to comment.