Skip to content

Commit

Permalink
Upgrade to Bazel 5 (again) (digital-asset#17319)
Browse files Browse the repository at this point in the history 
* Update to rules_haskell v0.16
* Update comments re bazel patches
* clean up bazel overrides
* Upgrade to Bazel 5.2.0
* Remove '--distinct_host_configuration=false'
* Update buildifier to 6.3.2
* Suffix macos and ubuntu caches with yyyymm
* bump windows cache to v14
* [REVERTME] bump linux/macos/darwin timeout to 4h
  • Loading branch information
@akrmn
akrmn authored Sep 1, 2023
1 parent aa83088 commit 6ae0e01
Show file tree
Hide file tree
Showing 27 changed files with 117 additions and 274 deletions.
28 changes: 6 additions & 22 deletions .bazelrc
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Bazel distributed cache, can be temporarily disabled by passing the following
# flag: --noremote_accept_cached
build:darwin --remote_http_cache=https://bazel-cache.da-ext.net
build:linux --remote_http_cache=https://bazel-cache.da-ext.net/ubuntu_20_04
build:darwin --remote_cache=https://bazel-cache.da-ext.net/macos/202308
build:linux --remote_cache=https://bazel-cache.da-ext.net/ubuntu/202308
# Don't push local build results to the remote cache.
build --remote_upload_local_results=false
# Do still push local build results to the local disk cache.
Expand Down Expand Up @@ -62,26 +62,10 @@ test --strategy=Scalac=worker
# https://github.com/grpc/grpc/pull/13929/files
build --copt -DGRPC_BAZEL_BUILD

# Bazel defaults to JDK9. This makes sure we run Nix provided JDK, and
# avoid dreaded "Unrecognized VM option 'CompactStrings'" error
build:linux --javabase=@nixpkgs_java_runtime//:runtime
build:linux --host_javabase=@nixpkgs_java_runtime//:runtime
build:linux --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:linux --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:darwin --javabase=@nixpkgs_java_runtime//:runtime
build:darwin --host_javabase=@nixpkgs_java_runtime//:runtime
build:darwin --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:darwin --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:windows --javabase=@dadew_java_runtime//:runtime
build:windows --host_javabase=@dadew_java_runtime//:runtime
build:windows --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:windows --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11

# Do not use a distinct configuration for "host", that is, binaries used
# at build time should be the same as release binaries.
# Without this, we will end up building everything twice as we need "damlc"
# at build time to bootstrap itself (for compiling ghc-prim).
build --distinct_host_configuration=false
build --java_language_version=11
build --java_runtime_version=nixpkgs_java_11
build --tool_java_runtime_version=nixpkgs_java_11
build --tool_java_language_version=11

# Propagate locales and the java home to the test environment.
build --test_env=LANG=en_US.utf8 --test_env=LOCALE_ARCHIVE --test_env=JAVA_HOME
Expand Down
2 changes: 1 addition & 1 deletion BAZEL.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -912,7 +912,7 @@ be defined using `da_scala_test`. It is preferable to always use
Scala benchmarks based on the JMH toolkit can be defined using the
`scala_benchmark_jmh` macro provided by `rules_scala`. It supports a restricted
subset of the attributes of `da_scala_binary`, namely: `name`, `deps`, `srcs`,
`scalacopts`, `resources` and `resource_jars`.
`scalacopts` and `resources`.
The end result of building the benchmark is a Scala binary of the same name,
which can be executed with `bazel run`.
Expand Down
6 changes: 5 additions & 1 deletion WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ load("//bazel_tools/dev_env_tool:dev_env_tool.bzl", "dadew", "dev_env_tool")
load(
"@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl",
"nixpkgs_cc_configure",
"nixpkgs_java_configure",
"nixpkgs_local_repository",
"nixpkgs_package",
"nixpkgs_python_configure",
Expand Down Expand Up @@ -561,7 +562,7 @@ load("@rules_haskell//tools:repositories.bzl", "rules_haskell_worker_dependencie
# Call this after `daml_haskell_deps` to ensure that the right `stack` is used.
rules_haskell_worker_dependencies()

load("//bazel_tools:java.bzl", "dadew_java_configure", "nixpkgs_java_configure")
load("//bazel_tools:java.bzl", "dadew_java_configure")

dadew_java_configure(
name = "dadew_java_runtime",
Expand All @@ -573,6 +574,9 @@ nixpkgs_java_configure(
nix_file = "//nix:bazel.nix",
nix_file_deps = common_nix_file_deps,
repositories = dev_env_nix_repos,
toolchain = True,
toolchain_name = "nixpkgs_java",
toolchain_version = "11",
) if not is_windows else None

# rules_go used here to compile a wrapper around the protoc-gen-scala plugin
Expand Down
143 changes: 21 additions & 122 deletions bazel_tools/java.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,125 +8,6 @@ load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_package")
load("//bazel_tools:pkg.bzl", "pkg_empty_zip")
load("//bazel_tools/dev_env_tool:dev_env_tool.bzl", "dadew_tool_home", "dadew_where")

_java_nix_file_content = """
let
pkgs = import <nixpkgs> { config = {}; overlays = []; };
in
{ attrPath
, attrSet
, filePath
}:
let
javaHome =
if attrSet == null then
pkgs.lib.attrByPath (pkgs.lib.splitString "." attrPath) null pkgs
else
pkgs.lib.attrByPath (pkgs.lib.splitString "." attrPath) null attrSet
;
javaHomePath =
if filePath == "" then
"${javaHome}"
else
"${javaHome}/${filePath}"
;
in
assert javaHome != null;
pkgs.runCommand "bazel-nixpkgs-java-runtime"
{ executable = false;
# Pointless to do this on a remote machine.
preferLocalBuild = true;
allowSubstitutes = false;
}
''
n=$out/BUILD.bazel
mkdir -p "$(dirname "$n")"
cat >>$n <<EOF
load("@rules_java//java:defs.bzl", "java_runtime")
java_runtime(
name = "runtime",
java_home = r"${javaHomePath}",
visibility = ["//visibility:public"],
)
EOF
''
"""

def nixpkgs_java_configure(
name = "nixpkgs_java_runtime",
attribute_path = None,
java_home_path = "",
repository = None,
repositories = {},
nix_file = None,
nix_file_content = "",
nix_file_deps = None,
nixopts = [],
fail_not_supported = True,
quiet = False):
"""Define a Java runtime provided by nixpkgs.
Creates a `nixpkgs_package` for a `java_runtime` instance.
Args:
name: The name-prefix for the created external repositories.
attribute_path: string, The nixpkgs attribute path for `jdk.home`.
java_home_path: optional, string, The path to `JAVA_HOME` within the package.
repository: See [`nixpkgs_package`](#nixpkgs_package-repository).
repositories: See [`nixpkgs_package`](#nixpkgs_package-repositories).
nix_file: optional, Label, Obtain the runtime from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`.
nix_file_content: optional, string, Obtain the runtime from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`.
nix_file_deps: See [`nixpkgs_package`](#nixpkgs_package-nix_file_deps).
nixopts: See [`nixpkgs_package`](#nixpkgs_package-nixopts).
fail_not_supported: See [`nixpkgs_package`](#nixpkgs_package-fail_not_supported).
quiet: See [`nixpkgs_package`](#nixpkgs_package-quiet).
"""
if attribute_path == None:
fail("'attribute_path' is required.", "attribute_path")

nix_expr = None
if nix_file and nix_file_content:
fail("Cannot specify both 'nix_file' and 'nix_file_content'.")
elif nix_file:
nix_expr = "import $(location {}) {{}}".format(nix_file)
nix_file_deps = depset(direct = [nix_file] + nix_file_deps).to_list()
elif nix_file_content:
nix_expr = nix_file_content
else:
nix_expr = "null"

nixopts = list(nixopts)
nixopts.extend([
"--argstr",
"attrPath",
attribute_path,
"--arg",
"attrSet",
nix_expr,
"--argstr",
"filePath",
java_home_path,
])

kwargs = dict(
repository = repository,
repositories = repositories,
nix_file_deps = nix_file_deps,
nixopts = nixopts,
fail_not_supported = fail_not_supported,
quiet = quiet,
)
java_runtime = "@%s//:runtime" % name
nixpkgs_package(
name = name,
nix_file_content = _java_nix_file_content,
**kwargs
)

def _dadew_java_configure_impl(repository_ctx):
ps = repository_ctx.which("powershell")
dadew = dadew_where(repository_ctx, ps)
Expand All @@ -138,11 +19,24 @@ java_runtime(
java_home = r"{java_home}",
visibility = ["//visibility:public"],
)
toolchain(
name = "toolchain",
toolchain = ":runtime",
toolchain_type = "@bazel_tools//tools/jdk:runtime_toolchain_type",
exec_compatible_with = [
"@platforms//cpu:x86_64",
"@platforms//os:windows",
],
target_compatible_with = [
"@platforms//cpu:x86_64",
"@platforms//os:windows",
],
)
""".format(
java_home = java_home.replace("\\", "/"),
))

dadew_java_configure = repository_rule(
_dadew_java_configure = repository_rule(
implementation = _dadew_java_configure_impl,
attrs = {
"dadew_path": attr.string(
Expand All @@ -159,13 +53,19 @@ Creates a `java_runtime` that uses the JDK installed by dadew.
""",
)

def dadew_java_configure(name, dadew_path):
_dadew_java_configure(
name = name,
dadew_path = dadew_path,
)
native.register_toolchains("@{}//:toolchain".format(name))

def da_java_library(
name,
deps,
srcs,
data = [],
resources = [],
resource_jars = [],
resource_strip_prefix = None,
tags = [],
visibility = None,
Expand All @@ -182,7 +82,6 @@ def da_java_library(
srcs = srcs,
data = data,
resources = resources,
resource_jars = resource_jars,
resource_strip_prefix = resource_strip_prefix,
tags = tags,
visibility = visibility,
Expand Down
2 changes: 1 addition & 1 deletion bazel_tools/packaging/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ sh_binary(
name = "package-app",
srcs = ["package-app.sh"],
data = [
"//bazel_tools/sh:mktgz",
"@gzip_dev_env//:gzip",
"@tar_dev_env//:tar",
"//bazel_tools/sh:mktgz",
] + (["@patchelf_nix//:bin/patchelf"] if not is_windows else []),
visibility = ["//visibility:public"],
deps = ["@bazel_tools//tools/bash/runfiles"],
Expand Down
4 changes: 2 additions & 2 deletions canton/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,11 +96,11 @@ tar xzf $$tmp
cp canton-*/lib/*.jar $@
""".format(
artif_user = artif_user,
src = ":lib/canton-ee.jar" if canton["local"] else "@canton//:jar", # not used in else case but still must resolve
artif_pass = artif_pass,
artif_user = artif_user,
curl = "@curl_dev_env//:bin/curl" if not is_windows else "@curl_dev_env//:bin/curl.exe",
local = "true" if canton["local"] else "",
src = ":lib/canton-ee.jar" if canton["local"] else "@canton//:jar", # not used in else case but still must resolve
),
tags = ["canton-ee"],
tools = [
Expand Down
10 changes: 5 additions & 5 deletions ci/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
timeoutInMinutes: 180
timeoutInMinutes: 240
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
Expand Down Expand Up @@ -103,7 +103,7 @@ jobs:
value: $[ and(eq(variables.is_release, 'true'),
eq(variables['Build.SourceBranchName'], 'main')) ]
- template: job-variables.yml
timeoutInMinutes: 180
timeoutInMinutes: 240
pool:
name: 'windows-pool'
demands: assignment -equals default
Expand Down Expand Up @@ -157,7 +157,7 @@ jobs:
- job: Linux_oracle
dependsOn:
- build_canton
timeoutInMinutes: 180
timeoutInMinutes: 240
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
Expand Down Expand Up @@ -301,7 +301,7 @@ jobs:
- check_for_release
condition: and(succeeded(),
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
timeoutInMinutes: 180
timeoutInMinutes: 240
pool:
name: ubuntu_20_04
demands: assignment -equals default
Expand All @@ -318,7 +318,7 @@ jobs:
- check_for_release
- compatibility_ts_libs
- build_canton
timeoutInMinutes: 180
timeoutInMinutes: 240
pool:
name: ubuntu_20_04
demands: assignment -equals default
Expand Down
6 changes: 3 additions & 3 deletions ci/configure-bazel.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ if is_windows; then
SUFFIX="${SUFFIX:0:12}"
echo "Platform suffix: $SUFFIX"
# We include an extra version at the end that we can bump manually.
CACHE_SUFFIX="$SUFFIX-v13"
CACHE_SUFFIX="$SUFFIX-v14"
CACHE_URL="$CACHE_URL/$CACHE_SUFFIX"
echo "build:windows-ci --remote_http_cache=https://bazel-cache.da-ext.net/$CACHE_SUFFIX" >> .bazelrc.local
echo "build:windows-ci --remote_cache=https://bazel-cache.da-ext.net/$CACHE_SUFFIX" >> .bazelrc.local
fi

# sets up write access to the shared remote cache if the branch is not a fork
Expand All @@ -86,5 +86,5 @@ if [[ "${IS_FORK}" = False ]]; then
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > "$GOOGLE_APPLICATION_CREDENTIALS"
unset GOOGLE_APPLICATION_CREDENTIALS_CONTENT
export GOOGLE_APPLICATION_CREDENTIALS
echo "build --remote_http_cache=$CACHE_URL --remote_upload_local_results=true --google_credentials=${GOOGLE_APPLICATION_CREDENTIALS}" >> .bazelrc.local
echo "build --remote_cache=$CACHE_URL --remote_upload_local_results=true --google_credentials=${GOOGLE_APPLICATION_CREDENTIALS}" >> .bazelrc.local
fi
2 changes: 1 addition & 1 deletion ci/macOS.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
dependsOn:
- check_for_release
- build_canton
timeoutInMinutes: 180
timeoutInMinutes: 240
pool:
name: macOS-pool
demands: assignment -equals ${{parameters.assignment}}
Expand Down
28 changes: 6 additions & 22 deletions compatibility/.bazelrc
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Bazel distributed cache, can be temporarily disabled by passing the following
# flag: --noremote_accept_cached
build:darwin --remote_http_cache=https://bazel-cache.da-ext.net
build:linux --remote_http_cache=https://bazel-cache.da-ext.net/ubuntu_20_04
build:darwin --remote_cache=https://bazel-cache.da-ext.net/macos/202308
build:linux --remote_cache=https://bazel-cache.da-ext.net/ubuntu/202308
# Don't push local build results to the remote cache.
build --remote_upload_local_results=false
# Do still push local build results to the local disk cache.
Expand Down Expand Up @@ -62,26 +62,10 @@ test --strategy=Scalac=worker
# https://github.com/grpc/grpc/pull/13929/files
build --copt -DGRPC_BAZEL_BUILD

# Bazel defaults to JDK9. This makes sure we run Nix provided JDK, and
# avoid dreaded "Unrecognized VM option 'CompactStrings'" error
build:linux --javabase=@nixpkgs_java_runtime//:runtime
build:linux --host_javabase=@nixpkgs_java_runtime//:runtime
build:linux --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:linux --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:darwin --javabase=@nixpkgs_java_runtime//:runtime
build:darwin --host_javabase=@nixpkgs_java_runtime//:runtime
build:darwin --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:darwin --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:windows --javabase=@dadew_java_runtime//:runtime
build:windows --host_javabase=@dadew_java_runtime//:runtime
build:windows --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
build:windows --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11

# Do not use a distinct configuration for "host", that is, binaries used
# at build time should be the same as release binaries.
# Without this, we will end up building everything twice as we need "damlc"
# at build time to bootstrap itself (for compiling ghc-prim).
build --distinct_host_configuration=false
build --java_language_version=11
build --java_runtime_version=nixpkgs_java_11
build --tool_java_runtime_version=nixpkgs_java_11
build --tool_java_language_version=11

# Propagate locales and the java home to the test environment.
build --test_env=LANG=en_US.utf8 --test_env=LOCALE_ARCHIVE --test_env=JAVA_HOME
Expand Down
Loading

0 comments on commit 6ae0e01

Please sign in to comment.