Revert modernization of virtualenv to fix pants.pex deploy in CI. (#1…

…0343) * Revert "Fix `run_virtualenv` to target the current `$PY`. (#10331)" This reverts commit 206b316. * Revert "Modernize virtualenv. (#10330)" This reverts commit f583d18.
pantsbuild · Jul 14, 2020 · 568e7d5 · 568e7d5
1 parent a5c7a4b
commit 568e7d5
Show file tree

Hide file tree

Showing 7 changed files with 90 additions and 86 deletions.
diff --git a/.gitignore b/.gitignore
@@ -37,9 +37,9 @@ out/
 # remove root-level /bin, which is created by Eclipse
 /bin/
 /build-support/*.pex
-/build-support/pex/
 /build-support/virtualenvs
-/build-support/virtualenv/
+/build-support/virtualenv-*
+/build-support/virtualenv.dist/
 /build-support/phab/
 /build-support/twine-deps.venv/
 /dist/

diff --git a/build-support/bin/check_shell.sh b/build-support/bin/check_shell.sh
@@ -17,8 +17,8 @@ if [[ -n "${bad_output}" ]]; then
   exit_code=1
 fi
 
-# Skip common.sh because it has a custom function for failing noisily.
-bad_output="$(find ./* -name '*.sh' -not -path './build-support/common.sh' -print0 \
+# Skip release.sh because it has a custom function for failing noisily.
+bad_output="$(find ./* -name '*.sh' -not -path './build-support/bin/release.sh' -print0 \
   | xargs -0 grep -H 'curl '| grep -v 'curl --fail' | cut -d: -f1 \
   | grep -v build-support/bin/check_shell.sh \
   || :)"

diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh
@@ -6,6 +6,18 @@ set -e
 
 ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd "$(git rev-parse --show-toplevel)" && pwd)
 
+function safe_curl() {
+  real_curl="$(command -v curl)"
+  set +e
+  "${real_curl}" --fail -SL "$@"
+  exit_code=$?
+  set -e
+  if [[ "${exit_code}" -ne 0 ]]; then
+    echo >&2 "Curl failed with args: $*"
+    exit 1
+  fi
+}
+
 # shellcheck source=build-support/common.sh
 source "${ROOT}/build-support/common.sh"
 
@@ -57,6 +69,29 @@ readonly DEPLOY_PANTS_WHEEL_DIR="${DEPLOY_DIR}/${DEPLOY_PANTS_WHEELS_PATH}"
 # only pants core is included.
 : "${PANTS_PEX_PACKAGES:="pantsbuild.pants"}"
 
+# URL from which pex release binaries can be downloaded.
+: "${PEX_DOWNLOAD_PREFIX:="https://github.com/pantsbuild/pex/releases/download"}"
+
+function requirement() {
+  package="$1"
+  grep "^${package}[^A-Za-z0-9]" "${ROOT}/3rdparty/python/requirements.txt" || die "Could not find requirement for ${package}"
+}
+
+function run_pex() {
+  # TODO: Cache this in case we run pex multiple times
+  (
+    PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"
+
+    pexdir="$(mktemp -d -t build_pex.XXXXX)"
+    trap 'rm -rf "${pexdir}"' EXIT
+
+    pex="${pexdir}/pex"
+
+    safe_curl -s "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex" > "${pex}"
+    "${PY}" "${pex}" "$@"
+  )
+}
+
 function run_packages_script() {
   (
     cd "${ROOT}"
@@ -142,7 +177,7 @@ function activate_tmp_venv() {
   # place, Shellcheck will not be able to find it so we tell Shellcheck to ignore the file.
   # shellcheck source=/dev/null
   VENV_DIR=$(mktemp -d -t pants.XXXXX) && \
-  run_virtualenv "$VENV_DIR" && \
+  "${ROOT}/build-support/virtualenv" "$VENV_DIR" && \
   source "$VENV_DIR/bin/activate"
 }
 
@@ -265,7 +300,7 @@ function activate_twine() {
   local -r venv_dir="${ROOT}/build-support/twine-deps.venv"
 
   rm -rf "${venv_dir}"
-  run_virtualenv "${venv_dir}"
+  "${ROOT}/build-support/virtualenv" "${venv_dir}"
   # Because the venv/bin/activate script's location is dynamic and not located in a fixed
   # place, Shellcheck will not be able to find it so we tell Shellcheck to ignore the file.
   # shellcheck source=/dev/null
@@ -275,13 +310,12 @@ function activate_twine() {
 
 function execute_pex() {
   run_pex \
-    --python "${PY}" \
-    --no-build \
-    --no-pypi \
-    --disable-cache \
-    -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-    -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-    "$@"
+      --no-build \
+      --no-pypi \
+      --disable-cache \
+      -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
+      -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
+      "$@"
 }
 
 function build_pex() {

diff --git a/build-support/bin/shellcheck.py b/build-support/bin/shellcheck.py
@@ -27,10 +27,12 @@ def run_shellcheck() -> None:
     targets = set(glob("./**/*.sh", recursive=True)) | {
         "./pants",
         "./build-support/pants_venv",
+        "./build-support/virtualenv",
         "./build-support/githooks/pre-commit",
         "./build-support/githooks/prepare-commit-msg",
     }
     targets -= set(glob("./build-support/bin/native/src/**/*.sh", recursive=True))
+    targets -= set(glob("./build-support/virtualenv.dist/**/*.sh", recursive=True))
     targets -= set(glob("./build-support/twine-deps.venv/**/*.sh", recursive=True))
     command = ["shellcheck", "--shell=bash", "--external-sources"] + sorted(targets)
     try:

diff --git a/build-support/common.sh b/build-support/common.sh
@@ -68,75 +68,3 @@ function git_merge_base() {
   # This prints the tracking branch if set and otherwise falls back to local "master".
   git rev-parse --symbolic-full-name --abbrev-ref HEAD@\{upstream\} 2>/dev/null || echo 'master'
 }
-
-function safe_curl() {
-  real_curl="$(command -v curl)"
-  set +e
-  "${real_curl}" --fail -SL "$@"
-  exit_code=$?
-  set -e
-  if [[ "${exit_code}" -ne 0 ]]; then
-    echo >&2 "Curl failed with args: $*"
-    exit 1
-  fi
-}
-
-function root_dir() {
-  cd "$(dirname "${BASH_SOURCE[0]}")" && cd "$(git rev-parse --show-toplevel)" && pwd
-}
-
-function requirement() {
-  package="$1"
-  grep "^${package}[^A-Za-z0-9]" "$(root_dir)/3rdparty/python/requirements.txt" || \
-    die "Could not find requirement for ${package}"
-}
-
-# URL from which pex release binaries can be downloaded.
-: "${PEX_DOWNLOAD_PREFIX:="https://github.com/pantsbuild/pex/releases/download"}"
-
-function run_pex() {
-  (
-    PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"
-    PEX_DIR="$(root_dir)/build-support/pex/${PEX_VERSION}"
-    if ! [ -x "${PEX_DIR}/pex" ]; then
-      pex_url="${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex"
-      log "Downloading pex==${PEX_VERSION} from ${pex_url}..."
-
-      pexdir="$(mktemp -d -t download_pex.XXXXX)"
-      trap 'rm -rf "${pexdir}"' EXIT
-
-      safe_curl -s "${pex_url}" > "${pexdir}/pex" && \
-      chmod +x "${pexdir}/pex" && \
-      rm -rf "${PEX_DIR}" && \
-      mkdir -p "$(dirname "${PEX_DIR}")" && \
-      mv "${pexdir}" "${PEX_DIR}"
-    fi
-
-    "${PEX_DIR}/pex" "$@"
-  )
-}
-
-
-: "${VIRTUALENV_VERSION:="20.0.26"}"
-
-function run_virtualenv() {
-  (
-    VIRTUALENV_DIR="$(root_dir)/build-support/virtualenv/${VIRTUALENV_VERSION}"
-    if ! [ -x "${VIRTUALENV_DIR}/virtualenv.pex" ]; then
-      log "Building PEX for virtualenv==${VIRTUALENV_VERSION}..."
-      pexdir="$(mktemp -d -t virtualenv_pex.XXXXX)"
-      trap 'rm -rf "${pexdir}"' EXIT
-
-      run_pex \
-        --python="${PY}" \
-        "virtualenv==${VIRTUALENV_VERSION}" \
-        -c virtualenv \
-        -o "${pexdir}/virtualenv.pex" && \
-      rm -rf "${VIRTUALENV_DIR}" && \
-      mkdir -p "$(dirname "${VIRTUALENV_DIR}")" && \
-      mv "${pexdir}" "${VIRTUALENV_DIR}"
-    fi
-
-    "${VIRTUALENV_DIR}/virtualenv.pex" --python "${PY}" "$@"
-  )
-}
diff --git a/build-support/pants_venv b/build-support/pants_venv
@@ -33,7 +33,7 @@ function remove_venv() {
 
 function create_venv() {
   remove_venv
-  run_virtualenv "$(venv_dir)"
+  "${REPO_ROOT}/build-support/virtualenv" "$(venv_dir)"
 }
 
 function ensure_gcc() {

diff --git a/build-support/virtualenv b/build-support/virtualenv
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Wrapper for self-bootstrapping virtualenv
+
+set -e
+
+VIRTUALENV_VERSION=16.6.2
+VIRTUALENV_PACKAGE_LOCATION=${VIRTUALENV_PACKAGE_LOCATION:-https://pypi.io/packages/source/v/virtualenv}
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && git rev-parse --show-toplevel)
+
+# shellcheck source=build-support/common.sh
+source "${REPO_ROOT}/build-support/common.sh"
+
+# Locate the binary's path to log to user or die gracefully if not found.
+if command -v "${PY}" >/dev/null; then
+  PY="$(command -v "${PY}")"
+else
+  die "No ${PY} interpreter found on the path. Python will not work!"
+fi
+
+log "Using python at ${PY}"
+
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+if ! [ -f "${HERE}/virtualenv.dist/BOOTSTRAPPED-${VIRTUALENV_VERSION}" ]; then
+  pushd "${HERE}" >/dev/null
+    VIRTUALENV_PACKAGE_FULL_URL="${VIRTUALENV_PACKAGE_LOCATION}/virtualenv-${VIRTUALENV_VERSION}.tar.gz"
+    log "Downloading ${VIRTUALENV_PACKAGE_FULL_URL}..."
+    curl -#L -O "${VIRTUALENV_PACKAGE_FULL_URL}" || \
+      die "Failed to download ${VIRTUALENV_PACKAGE_FULL_URL}."
+    rm -rf virtualenv-${VIRTUALENV_VERSION}
+    tar zxf virtualenv-${VIRTUALENV_VERSION}.tar.gz || \
+      die "Failed to extract ${VIRTUALENV_PACKAGE_FULL_URL}."
+    # TODO(ksweeney): Checksum
+    touch virtualenv-${VIRTUALENV_VERSION}/BOOTSTRAPPED-${VIRTUALENV_VERSION} && \
+      rm -rf virtualenv.dist && \
+      mv -f virtualenv-${VIRTUALENV_VERSION} virtualenv.dist
+  popd >/dev/null
+fi
+
+exec "${PY}" "${HERE}/virtualenv.dist/virtualenv.py" "$@"