Created mule tasks for aptly, releases page and docker hub images (al…

…gorand#1630)
Halgo-io · Oct 29, 2020 · 3ae2f78 · 3ae2f78
1 parent 450fd8c
commit 3ae2f78
Show file tree

Hide file tree

Showing 20 changed files with 609 additions and 62 deletions.
diff --git a/.gitignore b/.gitignore
@@ -63,3 +63,6 @@ assets
 
 # test binaries
 *.test
+
+index.html
+
diff --git a/docker/build/releases-page.Dockerfile b/docker/build/releases-page.Dockerfile
@@ -0,0 +1,10 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install git python3 python3-pip -y && \
+    pip3 install awscli boto3
+
+WORKDIR /root
+
+CMD ["/bin/bash"]
+
diff --git a/package-deploy.yaml b/package-deploy.yaml
@@ -1,10 +1,23 @@
 agents:
-  - name: deb
-    dockerFilePath: docker/build/cicd.ubuntu.Dockerfile
-    image: algorand/cicd-ubuntu
+  - name: docker
+    dockerFilePath: docker/build/docker.ubuntu.Dockerfile
+    image: algorand/docker-ubuntu
     version: scripts/configure_dev-deps.sh
     buildArgs:
       - GOLANG_VERSION=`./scripts/get_golang_version.sh`
+    env:
+      - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+      - NETWORK=$NETWORK
+      - VERSION=$VERSION
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    workDir: $HOME/projects/go-algorand
+
+  - name: releases-page
+    dockerFilePath: docker/build/releases-page.Dockerfile
+    image: algorand/releases-page
+    version: scripts/configure_dev-deps.sh
     env:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
@@ -19,29 +32,42 @@ agents:
     env:
       - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+      - CHANNEL=$CHANNEL
       - PACKAGES_DIR=$PACKAGES_DIR
       - NO_DEPLOY=$NO_DEPLOY
+      - S3_SOURCE=$S3_SOURCE
       - VERSION=$VERSION
     volumes:
       - $XDG_RUNTIME_DIR/gnupg/S.gpg-agent:/root/.gnupg/S.gpg-agent
       - $HOME/.gnupg/pubring.kbx:/root/.gnupg/pubring.kbx
     workDir: $HOME/projects/go-algorand
 
 tasks:
+  - task: docker.Make
+    name: docker
+    agent: docker
+    target: mule-docker
+
+  - task: docker.Make
+    name: releases-page
+    agent: releases-page
+    target: mule-releases-page
+
   - task: docker.Make
     name: rpm
     agent: rpm
     target: mule-deploy-rpm
 
 jobs:
-  package-deploy:
+  package-deploy-rpm:
     tasks:
       - docker.Make.rpm
 
-  package-deploy-deb:
+  docker-hub:
     tasks:
-      - docker.Make.deb
+      - docker.Make.docker
 
-  package-deploy-rpm:
+  releases-page:
     tasks:
-      - docker.Make.rpm
+      - docker.Make.releases-page
+
diff --git a/package.yaml b/package.yaml
@@ -67,3 +67,4 @@ jobs:
   package-docker:
     tasks:
       - docker.Make.docker
+
diff --git a/scripts/release/mule/Makefile.mule b/scripts/release/mule/Makefile.mule
@@ -45,13 +45,20 @@ mule = $(shell ls -d scripts/release/mule/*/ | awk 'BEGIN { FS="/" ; OFS="-" } {
 mule-deploy-rpm:
 	scripts/release/mule/deploy/rpm/deploy.sh
 
+mule-docker:
+	scripts/release/mule/deploy/docker/docker.sh
+
 mule-package-%: PKG_TYPE=$*
 mule-package-%:
 	echo Building algorand package...
 	scripts/release/mule/package/$(PKG_TYPE)/package.sh
 	echo Building algorand-devtools package...
 	scripts/release/mule/package/$(PKG_TYPE)/package.sh algorand-devtools
 
+mule-releases-page:
+	echo Generating the releases page...
+	cd scripts/release/mule/deploy/releases_page && ./generate_releases_page.sh
+
 mule-sign:
 	scripts/release/mule/sign/sign.sh
 

diff --git a/scripts/release/mule/README.md b/scripts/release/mule/README.md
@@ -4,6 +4,7 @@
 - [Build Stages](#build-stages)
 - [Custom Builds](#custom-builds)
 - [Examples](#examples)
+- [Manual Deploy](#manual-deploy)
 
 # Environment Variables
 
@@ -17,6 +18,11 @@ These env vars generally don't change between stages. Here is a list of variable
 - `OS_TYPE`
 - `VERSION`
 
+In addition, make sure that the following AWS credentials are set in environment variables:
+
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+
 # Build Stages
 
 - [package](#package)
@@ -32,6 +38,7 @@ These env vars generally don't change between stages. Here is a list of variable
 #### `mule` jobs
 
     - package
+        + calls `ci-build` make target
         + packages `deb`, `rpm` and `docker`
 
     - package-deb
@@ -63,9 +70,12 @@ These env vars generally don't change between stages. Here is a list of variable
 
 - customizable environment variables:
 
+    + `BRANCH`
+    + `CHANNEL`
     + `NETWORK`
     + `S3_SOURCE`, i.e., the S3 bucket from which to download
     + `SHA`, i.e., the value from `git rev-parse HEAD` if not passed on CLI
+    + `VERSION`
 
 #### `mule` jobs
 
@@ -84,7 +94,10 @@ These env vars generally don't change between stages. Here is a list of variable
 
 - customizable environment variables:
 
+    + `BRANCH`
+    + `CHANNEL`
     + `S3_SOURCE`, i.e., the S3 bucket from which to download
+    + `VERSION`
 
 ### `mule` jobs
 
@@ -97,20 +110,23 @@ These env vars generally don't change between stages. Here is a list of variable
 
 - customizable environment variables:
 
+    + `CHANNEL`
     + `NETWORK`
     + `NO_DEPLOY`
     + `PACKAGES_DIR`
+    + `S3_SOURCE`
+    + `VERSION`
 
 #### `mule` jobs
 
-    - package-deploy
-        + deploys both `deb` and `rpm`
+    - package-deploy-rpm
+        + deploys `rpm`
 
-    - package-deploy-deb
-        + deploys only `deb`
+    - docker-hub
+        + pushes new image to docker hub
 
-    - package-deploy-rpm
-        + deploys only `rpm`
+    - releases-page
+        + creates and pushes new releases page to S3
 
 # Custom Builds
 
@@ -165,7 +181,7 @@ Let's look at some examples.
 
 ### Uploading
 
-    VERSION=latest mule -f package-upload.yaml package-upload
+    STAGING=the-staging-area CHANNEL=beta VERSION=latest mule -f package-upload.yaml package-upload
 
 ### Testing
 
@@ -177,9 +193,9 @@ Let's look at some examples.
 
         BRANCH=update_signing CHANNEL=dev VERSION=2.1.86615 mule -f package-test.yaml package-test
 
-1. Download packages from staging and test.  This will download the packages to the usual place, i.e., `./go-algorand/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE/`.
+1. By setting the `S3_SOURCE` variable, the script will know to download packages from staging (instead of getting them from the local filesystem) and test.  This will download the packages to the usual place, i.e., `./go-algorand/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE/`.
 
-    Note that this is used to test a pending official release.
+    Note that this can be used to test a pending official release.
 
         CHANNEL=beta S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-test.yaml package-test
 
@@ -193,23 +209,60 @@ Let's look at some examples.
 
         CHANNEL=dev VERSION=2.1.86615 mule -f package-sign.yaml package-sign
 
-1. Download packages from staging and sign.  This will download the packages to the usual place, i.e., `./go-algorand/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE/`.
+1. Download packages from staging and sign. Again, the script will know to download from S3 because the `S3_SOURCE` has been set.  This will download the packages to the usual place, i.e., `./go-algorand/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE/`.
 
         CHANNEL=beta S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-sign.yaml package-sign
 
 ### Deploying
 
-1. Packages will be automatically downloaded from staging. Each package will then be pushed to `s3:algorand-releases:`.
+1. The new rpm packages will be downloaded from staging if the `S3_SOURCE` variable is set. Each package will then be pushed to `s3:algorand-releases:`.
 
-        VERSION=2.1.6 mule -f package-deploy.yaml package-deploy
+        S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-deploy.yaml package-deploy-rpm
 
 1. Packages are not downloaded from staging but rather are copied from the location on the local filesystem specified by `PACKAGES_DIR` in the `mule` yaml file. Each package will then be pushed to `s3:algorand-releases:`.
 
-        PACKAGES_DIR=/packages_location/foo VERSION=2.1.86615 mule -f package-deploy.yaml package-deploy
+        PACKAGES_DIR=/packages_location/foo VERSION=2.1.86615 mule -f package-deploy.yaml package-deploy-rpm
 
 1. `NO_DEPLOY` is set to `true`. Instead of automatically pushing to `s3:algorand-releases:`, this will copy the `rpmrepo` directory that was created in the container to the `WORKDIR` in the host environment (the `WORKDIR` is set in the `mule` yaml file).
 
     This is handy when testing a deployment and not yet ready to deploy.
 
-        NO_DEPLOY=true VERSION=2.1.6 mule -f package-deploy.yaml package-deploy
+        NO_DEPLOY=true S3_SOURCE=the-staging-area VERSION=2.1.6 mule -f package-deploy.yaml package-deploy-rpm
+
+# Manual Deploy
+
+> Before any processes are run, make sure that the signing keys have been added to the `gpg-agent`. The `gpg_preset_passphrase.sh` helper script is provided just for this purpose.
+
+Currently, it is still necessary to run two stages manually: sign and deploy. This is for several reasons, though principally because GPG signing of the build assets occurs in both stages.
+
+The processes that make up both stages have been `mule-ified` as much as possible, and all but one can be run as a `mule` task (deploying deb packages, which are done in its own separate docker container).
+
+### Signing
+
+Usually, the packages are pulled down from S3 where the eks pipeline or the `mule` `package-upload` task had placed them. Issue the following command to download and sign them:
+
+```
+CHANNEL=stable S3_SOURCE=the-internal-area VERSION=2.1.6 mule -f package-sign.yaml package-sign
+```
+
+> These are downloaded to the usual location at `tmp/node_pkgs/OS_TYPE/ARCH/` on the local filesystem.
+
+### Misc
+
+The following is an example of several commands issued for all the stages when building locally:
+
+```
+mule -f package.yaml package
+CHANNEL=dev VERSION=2.1.87522 SHA=730b3fd0 mule -f package-test.yaml package-test
+CHANNEL=dev VERSION=2.1.87522 mule -f package-sign.yaml package-sign
+CHANNEL=dev VERSION=2.1.87522 mule -f package-upload.yaml package-upload
+CHANNEL=dev VERSION=2.1.87522 NO_DEPLOY=true mule -f package-deploy.yaml package-deploy
+mule -f package-deploy.yaml releases-page
+
+docker build --build-arg AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" --build-arg AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -t aptly-test .
+
+docker run --name aptly-algorand --rm -i -v "$XDG_RUNTIME_DIR/gnupg/S.gpg-agent":/root/.gnupg/S.gpg-agent -v "$HOME/.gnupg/pubring.kbx":/root/.gnupg/pubring.kbx -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e CHANNEL=dev -e REPO=algorand -e VERSION=2.1.87522 aptly-test bash create_and_push
+
+docker run --name aptly-algorand --rm -it aptly-test
+```
 
diff --git a/scripts/release/mule/deploy/README.md b/scripts/release/mule/deploy/README.md
@@ -0,0 +1,25 @@
+## Manual deploy steps for deb packages
+
+It's **very** important that the docker container is run **before** the `release-page` **and** the `docker-hub` `mule` tasks in `package-deploy`.
+
+The docker container will do the following (see the `create_and_push` shell script):
+
+1. Copy the new `algorand` and `algorand-devtools` packages from the `algorand-staging` to `algorand-internal` buckets so the `packages/` directory in the container will be properly synced with the `algorand-internal` bucket.
+
+1. Sync `algorand-internal/packages` -> `packages/` in the container.
+
+1. Add the deb packages to the appropriate `aptly` repo.
+
+1. Create the snapshot (naming convention is `CHANNEL-VERSION`).
+
+1. Switch out the old snapshot in the `algorand-releases/deb` location for this new one.
+
+1. Sync `algorand-staging` -> `algorand-dev-deb-repo`
+
+When that is finished, it is safe to run the following commands (order doesn't matter):
+
+- `mule -f package-deploy releases-page`
+- `mule -f package-deploy docker-hub`
+
+> Note that the releases page is built from the latest release in the `algorand-dev-deb-repo` bucket, hence the need to have first run the docker container which performs that sync operation.
+
diff --git a/scripts/release/mule/deploy/deb/.aptly.conf b/scripts/release/mule/deploy/deb/.aptly.conf
@@ -0,0 +1,30 @@
+{
+  "rootDir": "/root/aptly",
+  "downloadConcurrency": 4,
+  "downloadSpeedLimit": 0,
+  "architectures": [],
+  "dependencyFollowSuggests": false,
+  "dependencyFollowRecommends": false,
+  "dependencyFollowAllVariants": false,
+  "dependencyFollowSource": false,
+  "dependencyVerboseResolve": false,
+  "gpgDisableSign": false,
+  "gpgDisableVerify": false,
+  "gpgProvider": "gpg",
+  "downloadSourcePackages": false,
+  "skipLegacyPool": true,
+  "ppaDistributorID": "ubuntu",
+  "ppaCodename": "",
+  "skipContentsPublishing": false,
+  "FileSystemPublishEndpoints": {},
+  "S3PublishEndpoints": {
+    "algorand-releases": {
+      "region":"us-east-1",
+      "bucket":"algorand-releases",
+      "acl":"public-read",
+      "prefix":"deb"
+    }
+  },
+  "SwiftPublishEndpoints": {}
+}
+
diff --git a/scripts/release/mule/deploy/deb/Dockerfile b/scripts/release/mule/deploy/deb/Dockerfile
@@ -0,0 +1,27 @@
+FROM ubuntu:18.04
+
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+
+ENV DEBIAN_FRONTEND noninteractive \
+    AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+    AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+
+RUN apt-get update && apt-get install aptly awscli binutils curl gnupg2 silversearcher-ag tree -y
+
+WORKDIR /root
+
+COPY . .
+
+RUN aws s3 sync s3://algorand-internal/packages packages && \
+    aptly repo create -distribution=stable -architectures=amd64 -component=main -comment=mainnet algorand && \
+    aptly repo create -distribution=beta -architectures=amd64 -component=main -comment=betanet algorand-beta && \
+    aptly repo create -distribution=indexer -architectures=amd64 -component=main -comment=indexer algorand-indexer && \
+    aptly repo create -distribution=indexer-beta -architectures=amd64 -component=main -comment=indexer-beta algorand-indexer-beta && \
+    aptly repo add algorand packages/deb/stable/*.deb && \
+    aptly repo add algorand-beta packages/deb/beta/*.deb && \
+    aptly repo add algorand-indexer packages/deb/indexer/stable/*.deb && \
+    aptly repo add algorand-indexer-beta packages/deb/indexer/beta/*.deb
+
+CMD ["/bin/bash"]
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -63,3 +63,6 @@ assets

		# test binaries
		*.test

		index.html
Original file line number	Diff line number	Diff line change
Expand Up		@@ -67,3 +67,4 @@ jobs:
		package-docker:
		tasks:
		- docker.Make.docker