Replace .travis.yml with two scripts for tag release and merge to master

automation/release.sh takes care of the release whenever it is run against a tagged commit. It then executes tests, creates the GitHub release in case it not exists, and attaches the release artifacts to the GitHub release. It then uploads the testing manifests to gcs so that the external tests can retrieve it from there and creates a stable.txt file that is also uploaded to gcs. Finally it updates the release api documentation and the go client repository. automation/postsubmit-master.sh takes care of checking and updating on merges to the master branch. It updates api documentation for master, the python client and the go client repository. Signed-off-by: Daniel Hiller <dhiller@redhat.com>
fossedihelm · Feb 24, 2021 · ecf5dff · ecf5dff
1 parent 3b8ed60
commit ecf5dff
Show file tree

Hide file tree

Showing 10 changed files with 175 additions and 165 deletions.
diff --git a/.travis.yml b/.travis.yml
diff --git a/Makefile b/Makefile
@@ -48,7 +48,7 @@ apidocs:
 	hack/dockerized "./hack/gen-swagger-doc/gen-swagger-docs.sh v1 html"
 
 client-python:
-	hack/dockerized "TRAVIS_TAG=${TRAVIS_TAG} ./hack/gen-client-python/generate.sh"
+	hack/dockerized "DOCKER_TAG=${DOCKER_TAG} ./hack/gen-client-python/generate.sh"
 
 go-build:
 	hack/dockerized "KUBEVIRT_VERSION=${KUBEVIRT_VERSION} ./hack/build-go.sh install ${WHAT}" && ./hack/build-copy-artifacts.sh ${WHAT}

diff --git a/automation/git-askpass.sh b/automation/git-askpass.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+cat "$GITHUB_TOKEN_PATH"
diff --git a/automation/postsubmit-master.sh b/automation/postsubmit-master.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+GIT_ASKPASS="$(pwd)/automation/git-askpass.sh"
+[ -f "$GIT_ASKPASS" ] || exit 1
+export GIT_ASKPASS
+
+export DOCKER_TAG=""
+
+make bazel-build-verify
+
+make build-verify # verify that we set version on the packages built by go (goveralls depends on go-build target)
+make apidocs
+make client-python
+make manifests DOCKER_PREFIX="$DOCKER_PREFIX" DOCKER_TAG="$DOCKER_TAG" # skip getting old CSVs here (no QUAY_REPOSITORY), verification might fail because of stricter rules over time; falls back to latest if not on a tag
+make olm-verify
+make prom-rules-verify
+
+make manifests
+make build-functests
+
+bash hack/gen-swagger-doc/deploy.sh
+bash hack/gen-client-python/deploy.sh
+hack/publish-staging.sh
diff --git a/automation/release.sh b/automation/release.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+function cleanup_gh_install() {
+    [ -n "${gh_cli_dir}" ] && [ -d "${gh_cli_dir}" ] && rm -rf "${gh_cli_dir:?}/"
+}
+
+function ensure_gh_cli_installed() {
+    if command -V gh; then
+        return
+    fi
+
+    trap 'cleanup_gh_install' EXIT SIGINT SIGTERM
+
+    # install gh cli for uploading release artifacts, with prompt disabled to enforce non-interactive mode
+    gh_cli_dir=$(mktemp -d)
+    (
+        cd  "$gh_cli_dir/"
+        curl -sSL "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_linux_amd64.tar.gz" -o "gh_${GH_CLI_VERSION}_linux_amd64.tar.gz"
+        tar xvf "gh_${GH_CLI_VERSION}_linux_amd64.tar.gz"
+    )
+    export PATH="$gh_cli_dir/gh_${GH_CLI_VERSION}_linux_amd64/bin:$PATH"
+    if ! command -V gh; then
+        echo "gh cli not installed successfully"
+        exit 1
+    fi
+    gh config set prompt disabled
+}
+
+function build_release_artifacts() {
+    make
+    make build-verify
+    make apidocs
+    make client-python
+    make manifests
+    make olm-verify
+    make prom-rules-verify
+
+    docker login -u="$QUAY_USER" -p="$QUAY_PASSWORD" quay.io
+    QUAY_REPOSITORY="kubevirt" PACKAGE_NAME="kubevirt-operatorhub" make bazel-push-images
+
+    make build-functests
+}
+
+function update_github_release() {
+    # note: for testing purposes we set the target repository, gh cli seems to always automatically choose the
+    # upstream repository automatically, even when you are in a fork
+
+    set +e
+    if ! gh release view --repo "$GITHUB_REPOSITORY" "$DOCKER_TAG" ; then
+        set -e
+        git show "$DOCKER_TAG" --format=format:%B > /tmp/tag_notes
+        gh release create --repo "$GITHUB_REPOSITORY" "$DOCKER_TAG" --prerelease --title="$DOCKER_TAG" --notes-file /tmp/tag_notes
+    else
+        set -e
+    fi
+
+    gh release upload --repo "$GITHUB_REPOSITORY" --clobber "$DOCKER_TAG" _out/cmd/virtctl/virtctl-v* \
+        _out/manifests/release/demo-content.yaml \
+        _out/manifests/release/kubevirt-operator.yaml \
+        _out/manifests/release/kubevirt-cr.yaml \
+        _out/templates/manifests/release/kubevirt-operator.yaml.j2 \
+        _out/templates/manifests/release/kubevirt-cr.yaml.j2 \
+        _out/templates/manifests/release/demo-content.yaml.j2 \
+        _out/manifests/release/olm/kubevirt-operatorsource.yaml \
+        "_out/manifests/release/olm/bundle/kubevirtoperator.$DOCKER_TAG.clusterserviceversion.yaml" \
+        _out/tests/tests.test \
+        _out/manifests/release/conformance.yaml \
+        _out/templates/manifests/release/conformance.yaml.j2 \
+        _out/manifests/testing/*
+}
+
+function upload_testing_manifests() {
+    # replaces periodic-kubevirt-update-release-x.y-testing-manifests periodics
+    if [ "$(gsutil ls 'gs://kubevirt-prow/devel/release/kubevirt/kubevirt/'|grep $DOCKER_TAG|wc -l)" -gt 0 ]; then
+        gsutil -m rm -r "gs://kubevirt-prow/devel/release/kubevirt/kubevirt/$DOCKER_TAG"
+    fi
+    gsutil cp -r "_out/manifests/testing" "gs://kubevirt-prow/devel/release/kubevirt/kubevirt/$DOCKER_TAG/manifests/"
+}
+
+function generate_stable_version_file() {
+    # will be available under http://storage.googleapis.com/kubevirt-prow/devel/release/kubevirt/kubevirt/stable.txt
+    (
+        gh release list --repo "$GITHUB_REPOSITORY" --limit 1000 |
+        awk '{ print $1 }' |
+        grep -v -E '\-(rc|alpha|beta)' |
+        sort -rV |
+        head -1
+    ) > _out/stable.txt
+    gsutil cp "_out/stable.txt" "gs://kubevirt-prow/devel/release/kubevirt/kubevirt/"
+}
+
+function main() {
+    DOCKER_TAG="$(git tag --points-at HEAD | head -1)"
+    if [ -z "$DOCKER_TAG" ]; then
+        echo "commit $(git show -s --format=%h) doesn't have a tag, exiting..."
+        exit 0
+    fi
+
+    export DOCKER_TAG
+
+    GIT_ASKPASS="$(pwd)/automation/git-askpass.sh"
+    [ -f "$GIT_ASKPASS" ] || exit 1
+    export GIT_ASKPASS
+
+    ensure_gh_cli_installed
+
+    gh auth login --with-token <"$GITHUB_TOKEN_PATH"
+
+    build_release_artifacts
+    update_github_release
+    upload_testing_manifests
+    generate_stable_version_file
+
+    bash hack/gen-swagger-doc/deploy.sh
+    hack/publish-staging.sh
+}
+
+main "$@"
diff --git a/automation/travisci-test.sh b/automation/travisci-test.sh
diff --git a/hack/gen-client-python/deploy.sh b/hack/gen-client-python/deploy.sh
@@ -1,16 +1,15 @@
 #!/usr/bin/env bash
 
-set -e
+set -exo pipefail
 
 source $(dirname "$0")/../common.sh
 
 GITHUB_FQDN=github.com
-CLIENT_PYTHON_REPO=kubevirt/client-python
+CLIENT_PYTHON_REPO=${CLIENT_PYTHON_REPO:-kubevirt/client-python}
 CLIENT_PYTHON_DIR=/tmp/kubevirt-client-python
 
-# Reusing API_REFERENCE_PUSH_TOKEN.
 git clone \
-    "https://${API_REFERENCE_PUSH_TOKEN}@${GITHUB_FQDN}/${CLIENT_PYTHON_REPO}.git" \
+    "https://${GIT_USER_NAME}@${GITHUB_FQDN}/${CLIENT_PYTHON_REPO}.git" \
     "${CLIENT_PYTHON_DIR}" >/dev/null 2>&1
 
 # Remove content under kubevirt, docs and test directories
@@ -20,8 +19,8 @@ cp -rf "${PYTHON_CLIENT_OUT_DIR}"/* "${CLIENT_PYTHON_DIR}/"
 
 cd "${CLIENT_PYTHON_DIR}"
 
-git config --global user.email "travis@travis-ci.org"
-git config --global user.name "Travis CI"
+git config user.email "${GIT_AUTHOR_NAME:-kubevirt-bot}"
+git config user.name "${GIT_AUTHOR_EMAIL:-rmohr+kubebot@redhat.com}"
 
 CLIENT_UPDATED="false"
 # Check api_client.py and configuration.py whether there are other changes
@@ -44,7 +43,7 @@ fi
 # Push only in case something got changed in code.
 if [ "${CLIENT_UPDATED}" = "true" ]; then
     git add -A .
-    git commit --message "Client Python update by Travis Build ${TRAVIS_BUILD_NUMBER}"
+    git commit --message "Client Python update by KubeVirt Prow build ${BUILD_ID}"
 
     git push origin master >/dev/null 2>&1
     echo "Client Python updated."

diff --git a/hack/gen-client-python/generate.sh b/hack/gen-client-python/generate.sh
@@ -14,8 +14,8 @@ CODEGEN_CONFIG="${PYTHON_CLIENT_OUT_DIR}/swagger-codegen-config.json"
 HARD_CODED_MODULES="${KUBEVIRT_DIR}/hack/gen-client-python/hard-coded-modules"
 
 # Define version of client
-if [ -n "${TRAVIS_TAG:-}" ]; then
-    CLIENT_PYTHON_VERSION="$TRAVIS_TAG"
+if [ -n "${DOCKER_TAG:-}" ]; then
+    CLIENT_PYTHON_VERSION="$DOCKER_TAG"
 else
     CLIENT_PYTHON_VERSION="$(git describe || echo 'none')"
 fi

diff --git a/hack/gen-swagger-doc/deploy.sh b/hack/gen-swagger-doc/deploy.sh
@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 
-set -e
+set -exo pipefail
 
 GITHUB_FQDN=github.com
-API_REF_REPO=kubevirt/api-reference
+API_REF_REPO=${API_REF_REPO:-kubevirt/api-reference}
 API_REF_DIR=/tmp/api-reference
 GITHUB_IO_FQDN="https://kubevirt.github.io/api-reference"
 
-TARGET_DIR="$TRAVIS_BRANCH"
-if [ -n "${TRAVIS_TAG}" ]; then
-    TARGET_DIR="$TRAVIS_TAG"
+TARGET_DIR="$PULL_BASE_REF"
+if [ -n "${DOCKER_TAG}" ]; then
+    TARGET_DIR="$DOCKER_TAG"
 fi
 
 git clone \
-    "https://${API_REFERENCE_PUSH_TOKEN}@${GITHUB_FQDN}/${API_REF_REPO}.git" \
+    "https://${GIT_USER_NAME}@${GITHUB_FQDN}/${API_REF_REPO}.git" \
     "${API_REF_DIR}" >/dev/null 2>&1
 rm -rf "${API_REF_DIR}/${TARGET_DIR:?}/"*
 mkdir -p ${API_REF_DIR}/${TARGET_DIR}
@@ -35,13 +35,13 @@ __EOF__
 find * -type d -regex "^v[0-9.]*" \
     -exec echo "* [{}](${GITHUB_IO_FQDN}/{}/index.html)" \; | sort -r --version-sort -t '[' --key 2 >>README.md
 
-git config --global user.email "travis@travis-ci.org"
-git config --global user.name "Travis CI"
+git config user.email "${GIT_AUTHOR_NAME:-kubevirt-bot}"
+git config user.name "${GIT_AUTHOR_EMAIL:-rmohr+kubebot@redhat.com}"
 
-# NOTE: exclude index.html from match, becasue it is static except commit hash.
+# NOTE: exclude index.html from match, because it is static except commit hash.
 if git status --porcelain | grep -v "index[.]html" | grep --quiet "^ [AM]"; then
     git add -A README.md "${TARGET_DIR}"/*.html
-    git commit --message "API Reference update by Travis Build ${TRAVIS_BUILD_NUMBER}"
+    git commit --message "API Reference update by KubeVirt Prow build ${BUILD_ID}"
 
     git push origin master >/dev/null 2>&1
     echo "API Reference updated for ${TARGET_DIR}."