Move artifact publishing out of yaml files (digital-asset#9071)

* Move artifact publishing out of yaml files The current publishing process pretty much hardcodes the set of artifacts we publish in the yaml config. This is a problem because we always release from `main` so the yaml files are always identical. However, we will add new artifacts over time and this starts falling apart. This PR changes this such that the process described in the yaml files is very generic and just uploads and downloads everything in a directory whereas the details are handled in bash scripts that will come from the respective release branch and are therefore version-dependent. As usual for these type of changes, I don’t have a great way to test this. I did do some due diligence to test that at least the artifacts are published correctly and I can download them but I can’t test the actual publishing. changelog_begin changelog_end * Update ci/copy-unix-release-artifacts.sh Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com> * Update ci/copy-windows-release-artifacts.sh Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com> * Update ci/publish-artifactory.sh Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com> Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com>
kgeonw · Mar 11, 2021 · 979e12f · 979e12f
1 parent 0e4af74
commit 979e12f
Show file tree

Hide file tree

Showing 8 changed files with 126 additions and 210 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -51,17 +51,6 @@ jobs:
   pool:
     vmImage: "Ubuntu-16.04"
   variables:
-    linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ]
-    macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ]
-    windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ]
-    windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ]
-    protos-zip: $[ dependencies.Linux.outputs['publish.protos-zip'] ]
-    daml-on-sql: $[ dependencies.Linux.outputs['publish.daml-on-sql'] ]
-    json-api: $[ dependencies.Linux.outputs['publish.json-api'] ]
-    script-runner: $[ dependencies.Linux.outputs['publish.script-runner'] ]
-    trigger-runner: $[ dependencies.Linux.outputs['publish.trigger-runner'] ]
-    trigger-service: $[ dependencies.Linux.outputs['publish.trigger-service'] ]
-    oauth2-middleware: $[ dependencies.Linux.outputs['publish.oauth2-middleware'] ]
     release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
     release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
     trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
@@ -78,65 +67,25 @@ jobs:
         source $(bash-lib)
         if git tag v$(release_tag) $(release_sha); then
           git push origin v$(release_tag)
-          mkdir $(Build.StagingDirectory)/release
-          mkdir $(Build.StagingDirectory)/artifactory
+          mkdir $(Build.StagingDirectory)/release-artifacts
         else
           setvar skip-github TRUE
         fi
     - task: DownloadPipelineArtifact@0
       inputs:
-        artifactName: $(linux-tarball)
-        targetPath: $(Build.StagingDirectory)/release
+        artifactName: linux-release
+        targetPath: $(Build.StagingDirectory)/release-artifacts
       condition: not(eq(variables['skip-github'], 'TRUE'))
     - task: DownloadPipelineArtifact@0
       inputs:
-        artifactName: $(macos-tarball)
-        targetPath: $(Build.StagingDirectory)/release
+        artifactName: macos-release
+        targetPath: $(Build.StagingDirectory)/release-artifacts
       condition: not(eq(variables['skip-github'], 'TRUE'))
     - task: DownloadPipelineArtifact@0
       inputs:
-        artifactName: $(windows-tarball)
-        targetPath: $(Build.StagingDirectory)/release
+        artifactName: windows-release
+        targetPath: $(Build.StagingDirectory)/release-artifacts
       condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(windows-installer)
-        targetPath: $(Build.StagingDirectory)/release
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(protos-zip)
-        targetPath: $(Build.StagingDirectory)/release
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(daml-on-sql)
-        targetPath: $(Build.StagingDirectory)/release
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(json-api)
-        targetPath: $(Build.StagingDirectory)/release
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(trigger-service)
-        targetPath: $(Build.StagingDirectory)/release
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(oauth2-middleware)
-        targetPath: $(Build.StagingDirectory)/release
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(trigger-runner)
-        targetPath: $(Build.StagingDirectory)/artifactory
-      condition: not(eq(variables['skip-github'], 'TRUE'))
-    - task: DownloadPipelineArtifact@0
-      inputs:
-        artifactName: $(script-runner)
-        targetPath: $(Build.StagingDirectory)/artifactory
       condition: not(eq(variables['skip-github'], 'TRUE'))
     - bash: |
         set -euo pipefail
@@ -148,7 +97,7 @@ jobs:
         trap cleanup EXIT
         echo "$GPG_KEY" | base64 -d > $KEY_FILE
         gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
-        cd $(Build.StagingDirectory)/release
+        cd $(Build.StagingDirectory)/release-artifacts/github
         sha256sum $(find . -type f | sort) > sha256sums
         # Note: relies on our release artifacts not having spaces in their
         # names. Creates a ${f}.asc with the signature for each $f.
@@ -169,7 +118,7 @@ jobs:
         target: '$(release_sha)'
         tagSource: 'manual'
         tag: 'v$(release_tag)'
-        assets: $(Build.StagingDirectory)/release/*
+        assets: $(Build.StagingDirectory)/release-artifacts/github/*
         assetUploadMode: 'replace'
         title: '$(release_tag)'
         addChangeLog: false
@@ -179,27 +128,7 @@ jobs:
       condition: not(eq(variables['skip-github'], 'TRUE'))
     - bash: |
         set -euo pipefail
-
-        push() {
-          local file repository local_path md5 sha1
-          repository=$1
-          file=$2
-          local_path=$(Build.StagingDirectory)/artifactory/${file}
-          md5=$(md5sum ${local_path} | awk '{print $1}')
-          sha1=$(sha1sum ${local_path} | awk '{print $1}')
-          curl -f \
-               -u "$AUTH" \
-               -H "X-Checksum-MD5:${md5}" \
-               -H "X-Checksum-SHA1:${sha1}" \
-               -X PUT \
-               -T ${local_path} \
-               https://digitalasset.jfrog.io/artifactory/${repository}/$(release_tag)/${file}
-        }
-
-        push daml-trigger-runner $(trigger-runner)
-        push daml-trigger-runner $(trigger-runner).asc
-        push daml-script-runner $(script-runner)
-        push daml-script-runner $(script-runner).asc
+        ./ci/publish-artifactory.sh $(Build.StagingDirectory) $(release_tag)
       env:
         AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
       condition: not(eq(variables['skip-github'], 'TRUE'))

diff --git a/build.ps1 b/build.ps1
@@ -14,6 +14,8 @@ if (!(Test-Path .\.bazelrc.local)) {
 
 $ARTIFACT_DIRS = if ("$env:BUILD_ARTIFACTSTAGINGDIRECTORY") { $env:BUILD_ARTIFACTSTAGINGDIRECTORY } else { Get-Location }
 
+mkdir -p ${ARTIFACT_DIRS}/logs
+
 # If a previous build was forcefully terminated, then stack's lock file might
 # not have been cleaned up properly leading to errors of the form
 #
@@ -53,8 +55,8 @@ bazel shutdown
 # It isn’t clear where exactly those errors are coming from.
 bazel fetch @nodejs_dev_env//...
 
-bazel build `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/build_execution_windows.log //...
+bazel build `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/logs/build_execution_windows.log //...
 
 bazel shutdown
 
-bazel test `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/test_execution_windows.log //...
+bazel test `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/logs/test_execution_windows.log //...
diff --git a/ci/build-unix.yml b/ci/build-unix.yml
@@ -68,50 +68,9 @@ steps:
       var_name: bash-lib
   - bash: |
       set -euo pipefail
-      source $(bash-lib)
       eval "$(./dev-env/bin/dade-assist)"
-
-      TARBALL=daml-sdk-${{parameters.release_tag}}-${{parameters.name}}.tar.gz
-      cp bazel-bin/release/sdk-release-tarball.tar.gz $(Build.StagingDirectory)/$TARBALL
-      setvar tarball "$TARBALL"
-
-      PROTOS_ZIP=protobufs-${{parameters.release_tag}}.zip
-      cp bazel-bin/release/protobufs.zip $(Build.StagingDirectory)/$PROTOS_ZIP
-      setvar protos-zip "$PROTOS_ZIP"
-
-      DAML_ON_SQL=daml-on-sql-${{parameters.release_tag}}.jar
-      ## Not built by default
-      bazel build //ledger/daml-on-sql:daml-on-sql-binary_deploy.jar
-      cp bazel-bin/ledger/daml-on-sql/daml-on-sql-binary_deploy.jar $(Build.StagingDirectory)/$DAML_ON_SQL
-      setvar daml-on-sql "$DAML_ON_SQL"
-
-      JSON_API=http-json-${{parameters.release_tag}}.jar
-      ## Not built by default
-      bazel build //ledger-service/http-json:http-json-binary_deploy.jar
-      cp bazel-bin/ledger-service/http-json/http-json-binary_deploy.jar $(Build.StagingDirectory)/$JSON_API
-      setvar json-api "$JSON_API"
-
-      TRIGGER=daml-trigger-runner-${{parameters.release_tag}}.jar
-      bazel build //triggers/runner:trigger-runner_deploy.jar
-      cp bazel-bin/triggers/runner/trigger-runner_deploy.jar $(Build.StagingDirectory)/$TRIGGER
-      setvar trigger-runner "$TRIGGER"
-
-      TRIGGER_SERVICE=trigger-service-${{parameters.release_tag}}.jar
-      ## Not built by default
-      bazel build //triggers/service:trigger-service-binary_deploy.jar
-      cp bazel-bin/triggers/service/trigger-service-binary_deploy.jar $(Build.StagingDirectory)/$TRIGGER_SERVICE
-      setvar trigger-service "$TRIGGER_SERVICE"
-
-      OAUTH2_MIDDLEWARE=oauth2-middleware-${{parameters.release_tag}}.jar
-      ## Not built by default
-      bazel build //triggers/service/auth:oauth2-middleware-binary_deploy.jar
-      cp bazel-bin/triggers/service/auth/oauth2-middleware-binary_deploy.jar $(Build.StagingDirectory)/$OAUTH2_MIDDLEWARE
-      setvar oauth2-middleware "$OAUTH2_MIDDLEWARE"
-
-      SCRIPT=daml-script-${{parameters.release_tag}}.jar
-      bazel build //daml-script/runner:script-runner_deploy.jar
-      cp bazel-bin/daml-script/runner/script-runner_deploy.jar $(Build.StagingDirectory)/$SCRIPT
-      setvar script-runner "$SCRIPT"
+      mkdir -p $(Build.StagingDirectory)/release
+      ./ci/copy-unix-release-artifacts.sh ${{parameters.release_tag}} ${{parameters.name}} $(Build.StagingDirectory)/release
     env:
       DAML_SDK_RELEASE_VERSION: ${{parameters.release_tag}}
     name: publish
@@ -120,64 +79,8 @@ steps:
                    eq(variables['Build.SourceBranchName'], 'main'))
   - task: PublishPipelineArtifact@0
     inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.tarball)
-      artifactName: $(publish.tarball)
+      targetPath: $(Build.StagingDirectory)/release
+      artifactName: ${{parameters.name}}-release
     condition: and(succeeded(),
                    eq(${{parameters.is_release}}, 'true'),
                    eq(variables['Build.SourceBranchName'], 'main'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.protos-zip)
-      artifactName: $(publish.protos-zip)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.daml-on-sql)
-      artifactName: $(publish.daml-on-sql)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.json-api)
-      artifactName: $(publish.json-api)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.trigger-runner)
-      artifactName: $(publish.trigger-runner)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.trigger-service)
-      artifactName: $(publish.trigger-service)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.oauth2-middleware)
-      artifactName: $(publish.oauth2-middleware)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
-  - task: PublishPipelineArtifact@0
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.script-runner)
-      artifactName: $(publish.script-runner)
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'),
-                   eq('${{parameters.name}}', 'linux'))
diff --git a/ci/build-windows.yml b/ci/build-windows.yml
@@ -34,22 +34,8 @@ steps:
 
   - bash: |
       set -euo pipefail
-      source $(bash-lib)
-      INSTALLER=daml-sdk-${{parameters.release_tag}}-windows.exe
-      mv "bazel-bin/release/windows-installer/daml-sdk-installer.exe" "$(Build.StagingDirectory)/$INSTALLER"
-      chmod +wx "$(Build.StagingDirectory)/$INSTALLER"
-      cleanup () {
-          rm -f signing_key.pfx
-      }
-      trap cleanup EXIT
-      echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
-      MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
-      rm signing_key.pfx
-      trap - EXIT
-      setvar installer "$INSTALLER"
-      TARBALL=daml-sdk-${{parameters.release_tag}}-windows.tar.gz
-      cp bazel-bin/release/sdk-release-tarball.tar.gz '$(Build.StagingDirectory)'/$TARBALL
-      setvar tarball "$TARBALL"
+      mkdir -p '$(Build.StagingDirectory)'/release
+      ./ci/copy-windows-release-artifacts.sh ${{parameters.release_tag}} '$(Build.StagingDirectory)'/release
     name: publish
     env:
       SIGNING_KEY: $(microsoft-code-signing)
@@ -62,12 +48,5 @@ steps:
                    eq(${{parameters.is_release}}, 'true'),
                    eq(variables['Build.SourceBranchName'], 'main'))
     inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.installer)
-      artifactName: $(publish.installer)
-  - task: PublishPipelineArtifact@0
-    condition: and(succeeded(),
-                   eq(${{parameters.is_release}}, 'true'),
-                   eq(variables['Build.SourceBranchName'], 'main'))
-    inputs:
-      targetPath: $(Build.StagingDirectory)/$(publish.tarball)
-      artifactName: $(publish.tarball)
+      targetPath: $(Build.StagingDirectory)/release
+      artifactName: windows-release
diff --git a/ci/build.yml b/ci/build.yml
@@ -174,7 +174,7 @@ jobs:
     - task: PublishBuildArtifacts@1
       condition: succeededOrFailed()
       inputs:
-        pathtoPublish: '$(Build.StagingDirectory)'
+        pathtoPublish: '$(Build.StagingDirectory)/logs'
         artifactName: 'Bazel Logs'
     - template: tell-slack-failed.yml
       parameters:

diff --git a/ci/copy-unix-release-artifacts.sh b/ci/copy-unix-release-artifacts.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+set -euo pipefail
+
+RELEASE_TAG=$1
+NAME=$2
+OUTPUT_DIR=$3
+
+mkdir -p $OUTPUT_DIR/github
+mkdir -p $OUTPUT_DIR/artifactory
+
+
+TARBALL=daml-sdk-$RELEASE_TAG-$NAME.tar.gz
+cp bazel-bin/release/sdk-release-tarball.tar.gz $OUTPUT_DIR/github/$TARBALL
+
+# Platform independent artifacts are only built on Linux.
+if [[ "$NAME" == "linux" ]]; then
+    PROTOS_ZIP=protobufs-$RELEASE_TAG.zip
+    cp bazel-bin/release/protobufs.zip $OUTPUT_DIR/github/$PROTOS_ZIP
+
+    DAML_ON_SQL=daml-on-sql-$RELEASE_TAG.jar
+    bazel build //ledger/daml-on-sql:daml-on-sql-binary_deploy.jar
+    cp bazel-bin/ledger/daml-on-sql/daml-on-sql-binary_deploy.jar $OUTPUT_DIR/github/$DAML_ON_SQL
+
+    JSON_API=http-json-$RELEASE_TAG.jar
+    bazel build //ledger-service/http-json:http-json-binary_deploy.jar
+    cp bazel-bin/ledger-service/http-json/http-json-binary_deploy.jar $OUTPUT_DIR/github/$JSON_API
+
+    TRIGGER_SERVICE=trigger-service-$RELEASE_TAG.jar
+    bazel build //triggers/service:trigger-service-binary_deploy.jar
+    cp bazel-bin/triggers/service/trigger-service-binary_deploy.jar $OUTPUT_DIR/github/$TRIGGER_SERVICE
+
+    OAUTH2_MIDDLEWARE=oauth2-middleware-$RELEASE_TAG.jar
+    bazel build //triggers/service/auth:oauth2-middleware-binary_deploy.jar
+    cp bazel-bin/triggers/service/auth/oauth2-middleware-binary_deploy.jar $OUTPUT_DIR/github/$OAUTH2_MIDDLEWARE
+
+
+    TRIGGER=daml-trigger-runner-$RELEASE_TAG.jar
+    bazel build //triggers/runner:trigger-runner_deploy.jar
+    cp bazel-bin/triggers/runner/trigger-runner_deploy.jar $OUTPUT_DIR/artifactory/$TRIGGER
+
+    SCRIPT=daml-script-$RELEASE_TAG.jar
+    bazel build //daml-script/runner:script-runner_deploy.jar
+    cp bazel-bin/daml-script/runner/script-runner_deploy.jar $OUTPUT_DIR/artifactory/$SCRIPT
+fi
diff --git a/ci/copy-windows-release-artifacts.sh b/ci/copy-windows-release-artifacts.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+set -eou pipefail
+
+RELEASE_TAG=$1
+OUTPUT_DIR=$2
+
+mkdir -p $OUTPUT_DIR/github
+INSTALLER=daml-sdk-$RELEASE_TAG-windows.exe
+mv "bazel-bin/release/windows-installer/daml-sdk-installer.exe" "$OUTPUT_DIR/github/$INSTALLER"
+chmod +wx "$OUTPUT_DIR/github/$INSTALLER"
+cleanup () {
+    rm -f signing_key.pfx
+}
+trap cleanup EXIT
+echo "$SIGNING_KEY" | base64 -d > signing_key.pfx
+MSYS_NO_PATHCONV=1 signtool.exe sign '/f' signing_key.pfx '/fd' sha256 '/tr' "http://timestamp.digicert.com" '/v' "$(Build.StagingDirectory)/$INSTALLER"
+rm signing_key.pfx
+trap - EXIT
+TARBALL=daml-sdk-$RELEASE_TAG-windows.tar.gz
+cp bazel-bin/release/sdk-release-tarball.tar.gz "$OUTPUT_DIR/github/$TARBALL"