automated ghc-lib build

This PR aims at automating the build of ghc-lib. The current process still has a few manual steps; it needs to be updated because Bintray is going away, so this seemed like a good opportunity to fully automate it. This works like the "patch bazel on Windows" jobs: the filename will contain a hash of the `ci/da-ghc-lib` folder, and the job will run only if the corresponding filename does not yet exist on the GCS bucket. PRs aiming at changing the ghc-lib version will need to run twice: once to create the artifacts, and once to change the `stack-snapshot.yaml` file to match. CHANGELOG_BEGIN CHANGELOG_END
digital-asset · Jun 2, 2020 · 73730af · 73730af
1 parent 256d48a
commit 73730af
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 1 deletion.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -82,8 +82,13 @@ jobs:
       - checkout: self
       - bash: ci/check-changelog.sh $(fork_sha)
 
+  - template: ci/da-ghc-lib/compile.yml
+    parameters:
+      final_job_name: da_ghc_lib
+
   - job: Linux
     dependsOn:
+      - da_ghc_lib
       - check_for_release
     variables:
       release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
@@ -121,6 +126,7 @@ jobs:
 
   - job: macOS
     dependsOn:
+      - da_ghc_lib
       - check_for_release
     timeoutInMinutes: 360
     pool:
@@ -155,6 +161,7 @@ jobs:
 
   - job: Windows
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - patch_bazel_windows
     variables:
@@ -191,6 +198,7 @@ jobs:
 
   - job: compatibility_ts_libs
     dependsOn:
+      - da_ghc_lib
       - check_for_release
     timeoutInMinutes: 60
     pool:
@@ -205,6 +213,7 @@ jobs:
 
   - job: compatibility_linux
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - compatibility_ts_libs
     timeoutInMinutes: 60
@@ -222,6 +231,7 @@ jobs:
 
   - job: compatibility_macos
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - compatibility_ts_libs
     timeoutInMinutes: 60
@@ -238,6 +248,7 @@ jobs:
 
   - job: compatibility_windows
     dependsOn:
+      - da_ghc_lib
       - check_for_release
       - compatibility_ts_libs
       - patch_bazel_windows
@@ -470,7 +481,9 @@ jobs:
       - template: ci/report-end.yml
 
   - job: write_ledger_dump
-    dependsOn: [ "check_for_release" ]
+    dependsOn:
+      - check_for_release
+      - da_ghc_lib
     pool:
       vmImage: "Ubuntu-16.04"
     condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),

diff --git a/ci/da-ghc-lib/compile.yml b/ci/da-ghc-lib/compile.yml
@@ -0,0 +1,74 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+parameters:
+- name: final_job_name
+
+jobs:
+
+- job: ${{ parameters.final_job_name }}
+  pool:
+    name: linux-pool
+  variables:
+    base-sha: '9c787d4d24f2b515934c8503ee2bbd7cfac4da20'
+    patches: '4650347db9a7e8c65831ab83b80da43b28c3deba 833ca63be2ab14871874ccb697'
+    ghc-lib-sha: '952911541233fa8a5bd94486bc46e03688b23bdd'
+    flavor: 'ghc-8.8.1'
+  steps:
+  - checkout: self
+  - bash: |
+      set -euo pipefail
+
+      eval "$(dev-env/bin/dade-assist)"
+
+      CACHE_KEY="$(find ci/da-ghc-lib -type f -print0 | sort -z | xargs -r0 md5sum | md5sum | awk '{print $1}')"
+      LIB="da-ghc-lib/ghc-lib-$CACHE_KEY.tar.gz"
+      PARSER="da-ghc-lib/ghc-lib-parser-$CACHE_KEY.tar.gz"
+      URL_BASE="https://daml-binaries.da-ext.net"
+      if [ "200" = "$(curl -Is "$URL_BASE/$LIB" | head -1 | awk '{print $2}')" ] \
+        && [ "200" = "$(curl -Is "$URL_BASE/$PARSER" | head -1 | awk '{print $2}')" ]; then
+          echo "Both artifacts exist; no need to run."
+          exit 0
+      fi
+
+      DIR="$(pwd)"
+      WORKDIR="$(mktemp -d -p "$(pwd)")"
+      cleanup() {
+          rm -rf "$WORKDIR"
+      }
+      trap cleanup EXIT
+
+      cd $WORKDIR
+
+      git clone https://github.com/digital-asset/ghc-lib.git
+      cd ghc-lib
+      git checkout $(ghc-lib-sha)
+      stack runhaskell --package extra \
+                       --package optparse-applicative \
+                       CI.hs \
+                       -- \
+                         --ghc-flavor=da \
+                         --da-base-sha=$(base-sha) \
+                         --da-patches="$(patches)" \
+                         --da-gen-flavor=$(flavor) \
+            | tee log
+      VERSION=$(cat log | tail -1)
+
+      GCS_KEY=$(mktemp)
+      cleanup() {
+          rm -rf $GCS_KEY
+          rm -rf "$WORKDIR"
+      }
+      trap cleanup EXIT
+      # This will break on external PRs.
+      echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
+      gcloud auth activate-service-account --key-file=$GCS_KEY
+      BOTO_CONFIG=/dev/null gsutil cp "ghc-lib-${VERSION}.tar.gz" "gs://daml-binaries/$LIB"
+      BOTO_CONFIG=/dev/null gsutil cp "ghc-lib-parser-${VERSION}.tar.gz" "gs://daml-binaries/$PARSER"
+      echo "Please add(/replace) the following lines to stack-snapshot.yaml:"
+      echo "  - archive: $URL_BASE/$LIB"
+      echo "    sha256: \"$(sha256sum ghc-lib-${VERSION}.tar.gz | awk '{print $1}')\""
+      echo "  - archive: $URL_BASE/$PARSER"
+      echo "    sha256: \"$(sha256sum ghc-lib-parser-${VERSION}.tar.gz | awk '{print $1}')\""
+    env:
+      GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
diff --git a/dev-env/bin/stack b/dev-env/bin/stack
@@ -0,0 +1 @@
+../lib/dade-exec-nix-tool
diff --git a/nix/default.nix b/nix/default.nix
@@ -45,6 +45,7 @@ in rec {
     ghcid           = pkgs.haskellPackages.ghcid;
     hlint           = bazel_dependencies.ghcStaticPkgs.hlint;
     ghci            = bazel_dependencies.ghcStatic;
+    stack           = pkgs.stack;
 
     # Hazel’s configure step currently searches for the C compiler in
     # PATH instead of taking it from our cc toolchain so we have to add