From 0e9cb10f7d2ed2e87b53ad60bd8141f0340f6d52 Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Tue, 27 Jun 2023 11:00:24 +0200 Subject: [PATCH] optionally get canton EE (#17039) --- .bazelrc | 2 + .envrc | 4 ++ WORKSPACE | 12 ++++++ bazel_tools/build_environment.bzl | 4 +- build.ps1 | 4 +- build.sh | 4 ++ canton/BUILD.bazel | 63 +++++++++++++++++++++++++++++ canton/README.md | 21 ++++++++++ ci/build-unix.yml | 2 + ci/build-windows.yml | 2 + ci/cron/daily-compat.yml | 29 ++++++++----- compatibility/.bazelrc | 2 + compiler/daml-extension/BUILD.bazel | 2 +- 13 files changed, 139 insertions(+), 12 deletions(-) diff --git a/.bazelrc b/.bazelrc index 80993ea90035..aac676a37800 100644 --- a/.bazelrc +++ b/.bazelrc @@ -137,3 +137,5 @@ test:windows --test_env=ComSpec build -c opt try-import %workspace%/.bazelrc.local +build --build_tag_filters=-canton-ee +test --test_tag_filters=-canton-ee diff --git a/.envrc b/.envrc index 5677ae2abc60..8062253116b9 100644 --- a/.envrc +++ b/.envrc @@ -6,6 +6,10 @@ DADE_PRE_COMMIT_HOOK_TYPE=pre-push # Load private overrides [[ -f .envrc.private ]] && source_env .envrc.private +if [ -n "${ARTIFACTORY_USERNAME:-}" ] && [ -n "${ARTIFACTORY_PASSWORD:-}" ]; then + export ARTIFACTORY_AUTH=$(echo -n "$ARTIFACTORY_USERNAME:$ARTIFACTORY_PASSWORD" | base64 -w0) +fi + # always start from a clean slate to ensure that the install phase produces the scenario described by `DADE_NO_PRE_COMMIT` and `DADE_PRE_COMMIT_HOOK_TYPE` # this is a bit unfortunate but it doesn't look like pre-commit currently supports uninstalling all hooks in one go as of version 2.20.0 HOOK_TYPES=(pre-commit pre-merge-commit pre-push prepare-commit-msg commit-msg post-checkout post-commit post-merge post-rewrite) diff --git a/WORKSPACE b/WORKSPACE index 0475ad6e00ad..f4eded56b0ea 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -118,11 +118,23 @@ nixpkgs_python_configure(repository = "@nixpkgs") if not is_windows else None nixpkgs_package( name = "curl_nix", attribute_path = "curl", + fail_not_supported = False, nix_file = "//nix:bazel.nix", nix_file_deps = common_nix_file_deps, repositories = dev_env_nix_repos, ) +dev_env_tool( + name = "curl_dev_env", + nix_include = ["bin/curl"], + nix_label = "@curl_nix", + nix_paths = ["bin/curl"], + tools = ["curl"], + win_include = ["bin/curl.exe"], + win_paths = ["bin/curl.exe"], + win_tool = "curl", +) + # Sysctl system dependency nixpkgs_package( name = "sysctl_nix", diff --git a/bazel_tools/build_environment.bzl b/bazel_tools/build_environment.bzl index 524e80b0c137..b3d7778d6f4b 100644 --- a/bazel_tools/build_environment.bzl +++ b/bazel_tools/build_environment.bzl @@ -24,11 +24,13 @@ npm_version = "{NPM_VERSION}" mvn_version = "{MVN_VERSION}" ghc_version = "{GHC_VERSION}" sdk_version = "{SDK_VERSION}" +artif_auth = "{artif_auth}" """.format( SDK_VERSION = semver, NPM_VERSION = semver, MVN_VERSION = semver, GHC_VERSION = ghc, + artif_auth = ctx.os.environ.get("ARTIFACTORY_AUTH", default = ""), ), executable = False, ) @@ -36,7 +38,7 @@ sdk_version = "{SDK_VERSION}" build_environment = repository_rule( # Tell Bazel that this rule will produce different results if any of the # env vars in the list has changed. - environ = ["DAML_SDK_RELEASE_VERSION"], + environ = ["DAML_SDK_RELEASE_VERSION", "ARTIFACTORY_AUTH"], implementation = _impl, attrs = {}, ) diff --git a/build.ps1 b/build.ps1 index 50899814588e..fde541ce7984 100644 --- a/build.ps1 +++ b/build.ps1 @@ -36,6 +36,8 @@ if (Test-Path -Path $env:appdata\stack\pantry\hackage\hackage-security-lock) { Remove-Item -ErrorAction Continue -Force -Recurse -Path $env:appdata\stack } +$env:ARTIFACTORY_AUTH = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("$env:ARTIFACTORY_USERNAME" + ":" + "$env:ARTIFACTORY_PASSWORD")) + function bazel() { Write-Output ">> bazel $args" $global:lastexitcode = 0 @@ -74,7 +76,7 @@ if ($env:SKIP_TESTS -ceq "False") { | Out-File -Encoding UTF8 -NoNewline scala-test-suite-name-map.json $tag_filter = "-dev-canton-test" - + bazel test //... ` `-`-build_tag_filters "$tag_filter" ` `-`-test_tag_filters "$tag_filter" ` diff --git a/build.sh b/build.sh index 572d60a20757..dbb3b9b943f7 100755 --- a/build.sh +++ b/build.sh @@ -39,6 +39,10 @@ else bazel=bazel fi +if [ -n "${ARTIFACTORY_USERNAME:-}" ] && [ -n "${ARTIFACTORY_PASSWORD:-}" ]; then + export ARTIFACTORY_AUTH=$(echo -n "$ARTIFACTORY_USERNAME:$ARTIFACTORY_PASSWORD" | base64 -w0) +fi + # Bazel test only builds targets that are dependencies of a test suite so do a full build first. $bazel build //... \ --build_tag_filters "${tag_filter:1}" \ diff --git a/canton/BUILD.bazel b/canton/BUILD.bazel index e692b4d05109..13896a108c30 100644 --- a/canton/BUILD.bazel +++ b/canton/BUILD.bazel @@ -7,6 +7,8 @@ load( "lf_scalacopts_stricter", ) load("//:canton_dep.bzl", "canton") +load("@build_environment//:configuration.bzl", "artif_auth") +load("@os_info//:os_info.bzl", "is_windows") java_binary( name = "canton", @@ -50,3 +52,64 @@ java_import( jars = [":canton-lib.jar"], visibility = ["//visibility:public"], ) + +java_import( + name = "lib-ee", + jars = [":canton-lib-ee.jar"], + tags = ["canton-ee"], + visibility = ["//visibility:public"], +) + +genrule( + name = "canton-lib-ee", + # We don't use the else branch but we need a resolvable value + srcs = [":lib/canton.jar"] if canton["local"] else ["@canton//:jar"], + outs = ["canton-lib-ee.jar"], + cmd = """ +set -euo pipefail + +if [ "{local}" = "true" ]; then + cp $(location {src}) $@ + exit 0 +fi + +CANTON_ENTERPRISE_VERSION=2.7.0-snapshot.20230620.10605.0.vb9879533 +CANTON_ENTERPRISE_SHA=6a62452cb78b37a4b7c37efe22839ab8b19963e90ad71be6a7a2cfdbe0ca3833 + +url=https://digitalasset.jfrog.io/artifactory/canton-enterprise/canton-enterprise-$$CANTON_ENTERPRISE_VERSION.tar.gz + +curl=$(location {curl}) + +tmp=$$(mktemp) +$$curl --location \ + --fail \ + --insecure \ + --silent \ + -H "Authorization: Basic {auth}" \ + $$url \ + > $$tmp + +computed_sha=$$(sha256sum $$tmp | awk '{{print $$1}}') +if [ "$$CANTON_ENTERPRISE_SHA" != "$$computed_sha" ]; then + echo "Wrong sha for canton enterprise jar." >&2 + echo "Trying to download from: $$url" >&2 + echo "Got: $$computed_sha" >&2 + echo "Expected: $$CANTON_ENTERPRISE_SHA" >&2 + exit 1 +fi + +tar xzf $$tmp + +cp **/lib/*.jar $@ +""".format( + auth = artif_auth, + curl = "@curl_dev_env//:bin/curl" if not is_windows else "@curl_dev_env//:bin/curl.exe", + local = "true" if canton["local"] else "", + src = ":lib/canton.jar" if canton["local"] else "@canton//:jar", # not used in else case but still must resolve + ), + tags = ["canton-ee"], + tools = [ + "@curl_dev_env//:bin/curl" if not is_windows else "@curl_dev_env//:bin/curl.exe", + ], + visibility = ["//visibility:public"], +) diff --git a/canton/README.md b/canton/README.md index 676153ec5494..2ca4ee2d2af1 100644 --- a/canton/README.md +++ b/canton/README.md @@ -53,3 +53,24 @@ situations, the daily canton bump job will reset the daml repo to depend on the snapshot (i.e. if the tip of the main branch has an `arbitrary_canton_sha` file, the daily job will delete it as part of the canton bump PR, and likely it will set the `local` attibute back to `False`). + +## Enterprise Edition + +Some situations may require running Canton Enterprise Edition, but this is an +open-source repository so we cannot assume every contributor will have a Canton +EE license key. + +Tests that require Canton EE **must** be tagged with `"canton-ee"`, which is +disabled by default through `.bazelrc`. To run those tests locally, either +explicitly target them or add `--build_tag_filters=` or `--test_tag_filters=` +as appropriate (yes, these are the full options: by setting the "running" +filters to empty for the current run, you overwrite the `-canton-ee` set in +`.bazelrc` which excludes the Canton EE tests, thereby removing the exclusion +and including the tests). + +Those tests are run on CI. + +If you're using a local build of canton (setting `local` to `True` per above) +_and_ you are explicitly overwriting the `*_tag_filters` to run the Canton EE +tests, they will be run using your provided `canton.jar` (which therefore needs +to be an EE jar at that point). diff --git a/ci/build-unix.yml b/ci/build-unix.yml index 36b01c183c77..dc3eaf2c92a8 100644 --- a/ci/build-unix.yml +++ b/ci/build-unix.yml @@ -37,6 +37,8 @@ steps: env: DAML_SDK_RELEASE_VERSION: ${{parameters.release_tag}} DAML_SCALA_VERSION: ${{parameters.scala_version}} + ARTIFACTORY_USERNAME: $(ARTIFACTORY_USERNAME) + ARTIFACTORY_PASSWORD: $(ARTIFACTORY_PASSWORD) - template: publish-platform-independence-dar.yml parameters: diff --git a/ci/build-windows.yml b/ci/build-windows.yml index e387d5d4b015..47b349bd29e4 100644 --- a/ci/build-windows.yml +++ b/ci/build-windows.yml @@ -22,6 +22,8 @@ steps: displayName: 'Build' env: DAML_SDK_RELEASE_VERSION: ${{parameters.release_tag}} + ARTIFACTORY_USERNAME: $(ARTIFACTORY_USERNAME) + ARTIFACTORY_PASSWORD: $(ARTIFACTORY_PASSWORD) # On release, we want to skip testing because: # - Caching on Windows is keyed on full path, and full path is somewhat # random (for a given machine, each pipeline has a its own workdir, but diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml index 5563b05b6cf1..2b790586f37b 100644 --- a/ci/cron/daily-compat.yml +++ b/ci/cron/daily-compat.yml @@ -331,6 +331,17 @@ jobs: git fetch git checkout origin/main + url_sha() ( + url="$1" + curl -u $AUTH \ + --fail \ + --location \ + --silent \ + "$url" \ + | sha256sum \ + | awk '{print $1}') + ) + canton_version=$(curl -u $AUTH \ --fail \ --location \ @@ -342,13 +353,7 @@ jobs: | sort -V \ | tail -1) canton_url="https://digitalasset.jfrog.io/artifactory/assembly/canton/$canton_version/canton-open-source-$canton_version.tar.gz" - canton_sha=$(curl -u $AUTH \ - --fail \ - --location \ - --silent \ - "$canton_url" \ - | sha256sum \ - | awk '{print $1}') + canton_sha=$(url_sha "$canton_url") sed -i 's|SKIP_DEV_CANTON_TESTS=.*|SKIP_DEV_CANTON_TESTS=false|' build.sh sed -e 's/^/# /' COPY > canton_dep.bzl @@ -361,15 +366,21 @@ jobs: } EOF + ee_canton_url="https://digitalasset.jfrog.io/artifactory/assembly/canton/$canton_version/canton-enterprise-$canton_version.tar.gz" + ee_canton_sha=$(url_sha "$ee_canton_url") + + sed -i "s|CANTON_ENTERPRISE_VERSION=.*|CANTON_ENTERPRISE_VERSION=$canton_version|" canton/BUILD.bazel + sed -i "s|CANTON_ENTERPRISE_SHA=.*|CANTON_ENTERPRISE_SHA=$ee_canton|" canton/BUILD.bazel + rm -f arbitrary_canton_sha branch="canton-update-$canton_version" - if git diff --exit-code -- canton_dep.bzl build.sh arbitrary_canton_sha; then + if git diff --exit-code -- canton_dep.bzl build.sh arbitrary_canton_sha canton/BUILD.bazel; then echo "Already up-to-date with latest Canton snapshot." else if [ "main" = "$(Build.SourceBranchName)" ]; then - git add canton_dep.bzl build.sh + git add canton_dep.bzl build.sh canton/BUILD.bazel git ls-files | grep arbitrary_canton_sha >&2 && git rm -f arbitrary_canton_sha open_pr "$branch" "bump canton to $canton_version" az extension add --name azure-devops diff --git a/compatibility/.bazelrc b/compatibility/.bazelrc index 80993ea90035..aac676a37800 100644 --- a/compatibility/.bazelrc +++ b/compatibility/.bazelrc @@ -137,3 +137,5 @@ test:windows --test_env=ComSpec build -c opt try-import %workspace%/.bazelrc.local +build --build_tag_filters=-canton-ee +test --test_tag_filters=-canton-ee diff --git a/compiler/daml-extension/BUILD.bazel b/compiler/daml-extension/BUILD.bazel index 864112b43385..7dba8a88a63b 100644 --- a/compiler/daml-extension/BUILD.bazel +++ b/compiler/daml-extension/BUILD.bazel @@ -74,7 +74,7 @@ genrule( ) genrule( - name = "webview-stylesheet.css", + name = "webview-stylesheet", srcs = ["src/webview.css"], outs = ["webview-stylesheet.css"], cmd = """