Clean up pipeline build variables (algorand#1722)

Remove the scripts that compute the BRANCH and CHANNEL at runtime. Don't pass in the CHANNEL. This can be determined by the NETWORK instead (which is passed in, of course). For now, compute the VERSION. This needs to be determined in a different manner and is a separate issue. This also adds some improvements to the deploy scripts and sign script. Although supporting command substitution is undoubtedly a good thing in the env blocks of the agents, this is made moot for now by assigning the ARCH_TYPE variable in the necessary dockerfiles and by simply doing the following to get the OS_TYPE: OS_TYPE=$(uname) OS_TYPE=${OS_TYPE,} # lowercases the first letter Also, it turns out that the GOLANG_VERSION is not needed by the shell scripts, so this has been removed. If determined to be needed in the future, this can also be set in the Dockerfile (since both ARCH and GOLANG_VERSION are already docker build arguments).
Halgo-io · Dec 3, 2020 · ef37ae8 · ef37ae8
1 parent 88aefd8
commit ef37ae8
Show file tree

Hide file tree

Showing 19 changed files with 236 additions and 118 deletions.
diff --git a/docker/build/aptly.Dockerfile b/docker/build/aptly.Dockerfile
@@ -19,7 +19,7 @@ RUN curl https://releases.algorand.com/key.pub | gpg --no-default-keyring --keyr
     aptly repo create -distribution=beta -architectures=amd64 -component=main -comment=betanet beta && \
     aptly mirror update stable && \
     aptly mirror update beta && \
-    aptly repo import stable stable algorand algorand-devtools
+    aptly repo import stable stable algorand algorand-devtools && \
     aptly repo import beta beta algorand-beta algorand-devtools-beta
 
 CMD ["/bin/bash"]

diff --git a/docker/build/cicd.centos.Dockerfile b/docker/build/cicd.centos.Dockerfile
@@ -13,7 +13,8 @@ RUN wget https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz \
     && tar -xvf go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz && \
     mv go /usr/local
 ENV GOROOT=/usr/local/go \
-    GOPATH=$HOME/go
+    GOPATH=$HOME/go \
+    ARCH_TYPE=${ARCH}
 RUN mkdir -p $GOPATH/src/github.com/algorand
 COPY . $GOPATH/src/github.com/algorand/go-algorand
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH \

diff --git a/docker/build/cicd.ubuntu.Dockerfile b/docker/build/cicd.ubuntu.Dockerfile
@@ -10,7 +10,8 @@ RUN wget https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz \
     && tar -xvf go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz && \
     mv go /usr/local
 ENV GOROOT=/usr/local/go \
-    GOPATH=$HOME/go
+    GOPATH=$HOME/go \
+    ARCH_TYPE=${ARCH}
 RUN mkdir -p $GOPATH/src/github.com/algorand
 COPY . $GOPATH/src/github.com/algorand/go-algorand
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH \

diff --git a/docker/build/docker.ubuntu.Dockerfile b/docker/build/docker.ubuntu.Dockerfile
@@ -2,6 +2,7 @@ ARG ARCH="amd64"
 
 FROM ${ARCH}/ubuntu:18.04
 ARG GOLANG_VERSION
+ARG ARCH="amd64"
 RUN apt-get update && apt-get install curl python python3.7 python3-pip build-essential apt-transport-https ca-certificates software-properties-common -y && \
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
     add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \
@@ -18,7 +19,8 @@ RUN apt-get update && apt-get install -y autoconf bsdmainutils git libboost-all-
     mv go /usr/local
 
 ENV GOROOT=/usr/local/go \
-    GOPATH=$HOME/go
+    GOPATH=$HOME/go \
+    ARCH_TYPE=${ARCH}
 ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH
 
 WORKDIR /root

diff --git a/package-deploy.yaml b/package-deploy.yaml
@@ -1,7 +1,7 @@
 agents:
   - name: docker
     dockerFilePath: docker/build/docker.ubuntu.Dockerfile
-    image: algorand/docker-ubuntu
+    image: algorand/go-algorand-ci-linux-ubuntu
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
@@ -33,7 +33,7 @@ agents:
     env:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-      - CHANNEL=$CHANNEL
+      - NETWORK=$NETWORK
       - SNAPSHOT=$SNAPSHOT
       - STAGING=$STAGING
       - VERSION=$VERSION
@@ -44,14 +44,14 @@ agents:
 
   - name: rpm
     dockerFilePath: docker/build/cicd.centos.Dockerfile
-    image: algorand/cicd-centos
+    image: algorand/go-algorand-ci-linux-centos
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
     env:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-      - CHANNEL=$CHANNEL
+      - NETWORK=$NETWORK
       - PACKAGES_DIR=$PACKAGES_DIR
       - NO_DEPLOY=$NO_DEPLOY
       - S3_SOURCE=$S3_SOURCE
@@ -83,6 +83,13 @@ tasks:
     target: mule-deploy-rpm
 
 jobs:
+  package-deploy:
+    tasks:
+      - docker.Make.deb
+      - docker.Make.rpm
+      - docker.Make.docker
+      - docker.Make.releases-page
+
   package-deploy-deb:
     tasks:
       - docker.Make.deb

diff --git a/package-sign.yaml b/package-sign.yaml
@@ -8,8 +8,7 @@ agents:
     env:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-      - BRANCH=$BRANCH
-      - CHANNEL=$CHANNEL
+      - NETWORK=$NETWORK
       - S3_SOURCE=$S3_SOURCE
       - VERSION=$VERSION
     volumes:

diff --git a/package-test.yaml b/package-test.yaml
@@ -1,21 +1,18 @@
 agents:
   - name: deb
     dockerFilePath: docker/build/cicd.ubuntu.Dockerfile
-    image: algorand/mule-linux-debian
+    image: algorand/go-algorand-ci-linux-ubuntu
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
     env:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
       - BRANCH=$BRANCH
-      - CHANNEL=$CHANNEL
       - NETWORK=$NETWORK
       - S3_SOURCE=$S3_SOURCE
       - SHA=$SHA
       - VERSION=$VERSION
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
     workDir: $HOME/projects/go-algorand
 
   - name: rpm
@@ -28,13 +25,10 @@ agents:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
       - BRANCH=$BRANCH
-      - CHANNEL=$CHANNEL
       - NETWORK=$NETWORK
       - S3_SOURCE=$S3_SOURCE
       - SHA=$SHA
       - VERSION=$VERSION
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
     workDir: $HOME/projects/go-algorand
 
 tasks:

diff --git a/package.yaml b/package.yaml
@@ -5,6 +5,9 @@ agents:
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
+    env:
+      - NETWORK=$NETWORK
+      - VERSION=$VERSION
     workDir: $HOME/projects/go-algorand
 
   - name: rpm
@@ -13,6 +16,9 @@ agents:
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
+    env:
+      - NETWORK=$NETWORK
+      - VERSION=$VERSION
     workDir: $HOME/projects/go-algorand
 
   - name: docker
@@ -21,6 +27,9 @@ agents:
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
+    env:
+      - NETWORK=$NETWORK
+      - VERSION=$VERSION
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     workDir: $HOME/projects/go-algorand
@@ -56,7 +65,6 @@ jobs:
 
   package-deb:
     tasks:
-      - docker.Make.build
       - docker.Make.deb
 
   package-rpm:

diff --git a/scripts/release/mule/common/get_channel.sh b/scripts/release/mule/common/get_channel.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+NETWORK="$1"
+
+if [ "$NETWORK" = betanet ]
+then
+    echo beta
+elif [ "$NETWORK" = mainnet ] || [ "$NETWORK" = testnet ]
+then
+    echo stable
+else
+    echo dev
+fi
+
diff --git a/scripts/release/mule/common/ostype.sh b/scripts/release/mule/common/ostype.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+
+OS_TYPE=$(uname)
+# Lowercase the first letter.
+echo "${OS_TYPE,}"
+
diff --git a/scripts/release/mule/deploy/deb/deploy.sh b/scripts/release/mule/deploy/deb/deploy.sh
@@ -1,30 +1,21 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
-if [ -z "$STAGING" ]
+if [ -z "$NETWORK" ]
 then
-    echo "[$0] Staging is a required parameter."
+    echo "[$0] Network is a required parameter."
     exit 1
 fi
 
-if [ -z "$CHANNEL" ]
+if [ -z "$STAGING" ]
 then
-    echo "[$0] Channel is a required parameter."
+    echo "[$0] Staging is a required parameter."
     exit 1
 fi
 
-if [[ ! "$CHANNEL" =~ ^beta$|^stable$ ]]
-then
-    echo "[$0] Repository values must be either \`beta\` or \`stable\`."
-    exit 1
-fi
-
-if [ -z "$VERSION" ]
-then
-    echo "[$0] Version is a required parameter."
-    exit 1
-fi
+CHANNEL=$("./scripts/release/mule/common/get_channel.sh" "$NETWORK")
+VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)}
 
 if [ -z "$SNAPSHOT" ]
 then
@@ -41,16 +32,30 @@ aptly mirror update beta
 aptly repo import stable stable algorand algorand-devtools
 aptly repo import beta beta algorand-beta algorand-devtools-beta
 
-KEY_PREFIX="releases/$CHANNEL/$VERSION"
+KEY_PREFIX="$CHANNEL/$VERSION"
 FILENAME_SUFFIX="${CHANNEL}_linux-amd64_${VERSION}.deb"
 ALGORAND_KEY="$KEY_PREFIX/algorand_${FILENAME_SUFFIX}"
 DEVTOOLS_KEY="$KEY_PREFIX/algorand-devtools_${FILENAME_SUFFIX}"
 
+# `STAGING` could contain a "path" (i.e. "my_bucket/foo/bar"), but the
+# `s3api` api expects it to be only the bucket name (i.e., "my_bucket").
+BUCKET=$(awk -F/ '{ print $1 }' <<< "$STAGING")
+
+# If the strings match then the objects are in the top-level of the bucket.
+if [ "$STAGING" = "$BUCKET" ]
+then
+    BUCKET_PREFIX_PATH="$STAGING"
+else
+    # Remove matching prefix.
+    BUCKET_PREFIX_PATH=${STAGING#$BUCKET"/"}
+fi
+
 for key in {"$ALGORAND_KEY","$DEVTOOLS_KEY"}
 do
-    if aws s3api head-object --bucket "$STAGING" --key "$key"
+    key="$BUCKET_PREFIX_PATH/$key"
+    if aws s3api head-object --bucket "$BUCKET" --key "$key"
     then
-        aws s3 cp "s3://$STAGING/$key" "$PACKAGES_DIR"
+        aws s3 cp "s3://$BUCKET/$key" "$PACKAGES_DIR"
     else
         echo "[$0] The package \`$key\` failed to download."
     fi
@@ -61,7 +66,12 @@ then
     aptly repo add "$CHANNEL" "$PACKAGES_DIR"/*.deb
     aptly repo show -with-packages "$CHANNEL"
     aptly snapshot create "$SNAPSHOT" from repo "$CHANNEL"
-    aptly publish snapshot -gpg-key=dev@algorand.com -origin=Algorand -label=Algorand "$SNAPSHOT" s3:algorand-releases:
+    if ! aptly publish show "$CHANNEL" s3:algorand-releases: &> /dev/null
+    then
+        aptly publish snapshot -gpg-key=dev@algorand.com -origin=Algorand -label=Algorand "$SNAPSHOT" s3:algorand-releases:
+    else
+        aptly publish switch "$CHANNEL" s3:algorand-releases: "$SNAPSHOT"
+    fi
 else
     echo "[$0] The packages directory is empty, so there is nothing to add the \`$CHANNEL\` repo."
     exit 1

diff --git a/scripts/release/mule/deploy/docker/docker.sh b/scripts/release/mule/deploy/docker/docker.sh
@@ -8,8 +8,10 @@
 #
 set -ex
 
-NETWORK=${NEWORK:-mainnet}
-VERSION=${VERSION:-latest}
+if [ -z "$NETWORK" ] || [ -z "$VERSION" ]; then
+    echo "[$0] NETWORK=$NETWORK or VERSION=$VERSION is missing."
+    exit 1
+fi
 
 if [[ ! "$NETWORK" =~ ^mainnet$|^testnet$|^betanet$ ]]
 then

diff --git a/scripts/release/mule/deploy/rpm/deploy.sh b/scripts/release/mule/deploy/rpm/deploy.sh
@@ -7,12 +7,16 @@ echo
 date "+build_release begin DEPLOY rpm stage %Y%m%d_%H%M%S"
 echo
 
-ARCH_TYPE=$(./scripts/archtype.sh)
-OS_TYPE=$(./scripts/ostype.sh)
-CHANNEL=${CHANNEL:-stable}
+if [ -z "$NETWORK" ]; then
+    echo "[$0] NETWORK is missing."
+    exit 1
+fi
+
+CHANNEL=$(./scripts/release/mule/common/get_channel.sh "$NETWORK")
+VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)}
 NO_DEPLOY=${NO_DEPLOY:-false}
+OS_TYPE=$(./scripts/release/mule/common/ostype.sh)
 PACKAGES_DIR=${PACKAGES_DIR:-"./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"}
-VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)}
 
 if [ -n "$S3_SOURCE" ]
 then

diff --git a/scripts/release/mule/package/deb/package.sh b/scripts/release/mule/package/deb/package.sh
@@ -7,14 +7,17 @@ echo
 date "+build_release begin PACKAGE DEB stage %Y%m%d_%H%M%S"
 echo
 
-BRANCH=${BRANCH:-$(./scripts/compute_branch.sh)}
-CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")}
+if [ -z "$NETWORK" ]; then
+    echo "[$0] NETWORK is missing."
+    exit 1
+fi
+
+CHANNEL=$("./scripts/release/mule/common/get_channel.sh" "$NETWORK")
 VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)}
 # A make target in Makefile.mule may pass the name as an argument.
 PACKAGE_NAME="$1"
 
-DEFAULTNETWORK=${DEFAULTNETWORK:-$(./scripts/compute_branch_network.sh "$BRANCH")}
-DEFAULT_RELEASE_NETWORK=$("./scripts/compute_branch_release_network.sh" "$DEFAULTNETWORK")
+DEFAULT_RELEASE_NETWORK=$("./scripts/compute_branch_release_network.sh" "$NETWORK")
 export DEFAULT_RELEASE_NETWORK
 
 find tmp/node_pkgs -name "*${CHANNEL}*linux*${VERSION}*.tar.gz" | cut -d '/' -f3-4 | sort --unique | while read OS_ARCH; do

diff --git a/scripts/release/mule/package/docker/package.sh b/scripts/release/mule/package/docker/package.sh
@@ -6,12 +6,15 @@ echo
 date "+build_release begin PACKAGE DOCKER stage %Y%m%d_%H%M%S"
 echo
 
-ARCH_TYPE=$(./scripts/archtype.sh)
-OS_TYPE=$(./scripts/ostype.sh)
-BRANCH=${BRANCH:-$(./scripts/compute_branch.sh "$BRANCH")}
-CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")}
-PKG_ROOT_DIR="./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"
+if  [ -z "$NETWORK" ]; then
+    echo "[$0] NETWORK is missing."
+    exit 1
+fi
+
+CHANNEL=$(./scripts/release/mule/common/get_channel.sh "$NETWORK")
 VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)}
+OS_TYPE=$(./scripts/release/mule/common/ostype.sh)
+PKG_ROOT_DIR="./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"
 ALGOD_INSTALL_TAR_FILE="$PKG_ROOT_DIR/node_${CHANNEL}_${OS_TYPE}-${ARCH_TYPE}_${VERSION}.tar.gz"
 
 if [ -f "$ALGOD_INSTALL_TAR_FILE" ]; then
@@ -31,17 +34,10 @@ DOCKERFILE="./docker/build/algod.Dockerfile"
 START_ALGOD_FILE="./docker/release/start_algod_docker.sh"
 ALGOD_DOCKER_INIT="./docker/release/algod_docker_init.sh"
 
-# Use go version specified by get_golang_version.sh
-if ! GOLANG_VERSION=$(./scripts/get_golang_version.sh)
-then
-    echo "${GOLANG_VERSION}"
-    exit 1
-fi
-
 echo "building '$DOCKERFILE' with install file $ALGOD_INSTALL_TAR_FILE"
 cp "$ALGOD_INSTALL_TAR_FILE" "/tmp/$INPUT_ALGOD_TAR_FILE"
 cp "$ALGOD_DOCKER_INIT" /tmp
-docker build --build-arg ALGOD_INSTALL_TAR_FILE="$INPUT_ALGOD_TAR_FILE" --build-arg GOLANG_VERSION="${GOLANG_VERSION}" /tmp -t "$DOCKER_IMAGE" -f "$DOCKERFILE"
+docker build --build-arg ALGOD_INSTALL_TAR_FILE="$INPUT_ALGOD_TAR_FILE" /tmp -t "$DOCKER_IMAGE" -f "$DOCKERFILE"
 
 mkdir -p "/tmp/$NEW_PKG_DIR"