Skip to content

Commit

Permalink
Add a Scala 2.13 build pipeline (#8271)
Browse files Browse the repository at this point in the history 
* Add a Scala 2.13 build pipeline

This adds initial support for multiple Scala versions controlled via
the DAML_SCALA_VERSION env var and a CI job to make sure we don’t
regress. For now we only test //libs-scala/ports/... which seemed like
the easiest starting point I could find. We can incrementally expand
that over time.

changelog_begin
changelog_end

* Document pinning

changelog_begin
changelog_end

* Address review comments

changelog_begin
changelog_end
  • Loading branch information
@cocreature
cocreature authored Dec 14, 2020
1 parent ae28cf4 commit d24ef94
Show file tree
Hide file tree
Showing 13 changed files with 15,885 additions and 96 deletions.
5 changes: 5 additions & 0 deletions .bazelrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ build:darwin --repository_cache=.bazel-cache/repo
fetch:darwin --repository_cache=.bazel-cache/repo
sync:darwin --repository_cache=.bazel-cache/repo

build:scala_2_13 --repo_env=DAML_SCALA_VERSION=2.13.3
fetch:scala_2_13 --repo_env=DAML_SCALA_VERSION=2.13.3
query:scala_2_13 --repo_env=DAML_SCALA_VERSION=2.13.3
sync:scala_2_13 --repo_env=DAML_SCALA_VERSION=2.13.3

# Improve remote cache hit rate by exluding environment variables from the
# sandbox that are not whitelisted using --action_env.
# This will become the default in a future Bazel release.
Expand Down
13 changes: 8 additions & 5 deletions BAZEL-JVM.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,14 @@ symbols are replaced by `_` characters. E.g. the artifact
`org.scalaz:scalaz-core_2.12:7.2.24` is available as
`@maven//:org_scalaz_scalaz_core_2_12`.

Adding, changing, or removing a Maven dependency requires two steps: First, you
need to modify the `artifacts` attribute to `maven_install`, second, you need
to execute `bazel run @unpinned_maven//:pin` to update `maven_install.json`.
You should also run `@unpinned_maven//:pin` if you change other attributes to
`maven_install`.
Adding, changing, or removing a Maven dependency requires two steps:
First, you need to modify the `artifacts` attribute to
`maven_install`, second, you need to execute `bazel run
@unpinned_maven//:pin` to update `maven_install.json`. You should
also run `@unpinned_maven//:pin` if you change other attributes to
`maven_install`. You need to run this once per Scala version with the
default being 2.12 so run it again for 2.13:
``DAML_SCALA_VERSION=2.13.3 bazel run @unpinned_maven//:pin``.

Refer to the [`rules_jvm_external` documentation][rules_jvm_external] for
further information.
Expand Down
4 changes: 3 additions & 1 deletion BAZEL.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -989,7 +989,9 @@ resolution and import the required artifacts into the Bazel build.
The resolved versions are pinned in the file `maven_install.json`. Execute
`bazel run @unpinned_maven//:pin` when you wish to update or add a new
dependency. See [`rules_jvm_external`][rules_jvm_external] for details.
dependency. You also need to run this a second time to update the deps for 2.13
`DAML_SCALA_VERSION=2.13.3 bazel run @unpinned_maven//:pin`.
See [`rules_jvm_external`][rules_jvm_external] for details.
[rules_jvm_external]: https://github.com/bazelbuild/rules_jvm_external#updating-maven_installjson
Expand Down
34 changes: 21 additions & 13 deletions WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ load("//bazel_tools:build_environment.bzl", "build_environment")

build_environment(name = "build_environment")

load("//bazel_tools:scala_version.bzl", "scala_version")

scala_version(name = "scala_version")

load("@scala_version//:index.bzl", "scala_major_version", "scala_version")

dadew(name = "dadew")

load("@os_info//:os_info.bzl", "is_darwin", "is_linux", "is_windows")
Expand Down Expand Up @@ -698,7 +704,7 @@ pinned_maven_install()

load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")

scala_config("2.12.12")
scala_config(scala_version)

load(
"@io_bazel_rules_scala//scala:scala.bzl",
Expand All @@ -710,19 +716,21 @@ scala_repositories(
fetch_sources = True,
overriden_artifacts =
{
"io_bazel_rules_scala_scala_compiler": {
"artifact": "org.scala-lang:scala-compiler:2.12.12",
"sha256": "9dfa682ad7c2859cdcf6a31b9734c8f1ee38e7e391aeafaef91967b6ce819b6b",
},
"io_bazel_rules_scala_scala_library": {
"artifact": "org.scala-lang:scala-library:2.12.12",
"sha256": "1673ffe8792021f704caddfe92067ed1ec75229907f84380ad68fe621358c925",
},
"io_bazel_rules_scala_scala_reflect": {
"artifact": "org.scala-lang:scala-reflect:2.12.12",
"sha256": "3c502791757c0c8208f00033d8c4d778ed446efa6f49a6f89b59c6f92b347774",
"2.12": {
"io_bazel_rules_scala_scala_compiler": {
"artifact": "org.scala-lang:scala-compiler:2.12.12",
"sha256": "9dfa682ad7c2859cdcf6a31b9734c8f1ee38e7e391aeafaef91967b6ce819b6b",
},
"io_bazel_rules_scala_scala_library": {
"artifact": "org.scala-lang:scala-library:2.12.12",
"sha256": "1673ffe8792021f704caddfe92067ed1ec75229907f84380ad68fe621358c925",
},
"io_bazel_rules_scala_scala_reflect": {
"artifact": "org.scala-lang:scala-reflect:2.12.12",
"sha256": "3c502791757c0c8208f00033d8c4d778ed446efa6f49a6f89b59c6f92b347774",
},
},
},
}.get(scala_major_version, {}),
)

load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")
Expand Down
56 changes: 56 additions & 0 deletions azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,57 @@ jobs:
trigger_sha: '$(trigger_sha)'
- template: ci/report-end.yml

- job: Linux_scala_2_13
dependsOn:
- da_ghc_lib
- check_for_release
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
timeoutInMinutes: 360
pool:
name: 'linux-pool'
demands: assignment -equals default
steps:
- template: ci/report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: eq(variables.is_release, 'true')
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- bash: |
source dev-env/lib/ensure-nix
ci/dev-env-push.py
displayName: 'Push Developer Environment build results'
condition: eq(variables['System.PullRequest.IsFork'], 'False')
env:
# to upload to the Nix cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel test --config scala_2_13 //libs-scala/ports/...
displayName: 'Build'
env:
DAML_SDK_RELEASE_VERSION: $(release_tag)
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: ci/report-end.yml

- job: compatibility_ts_libs
dependsOn:
- da_ghc_lib
Expand Down Expand Up @@ -621,6 +672,7 @@ jobs:
- compatibility_macos
- compatibility_linux
- compatibility_windows
- Linux_scala_2_13
pool:
name: "linux-pool"
demands: assignment -equals default
Expand All @@ -637,6 +689,10 @@ jobs:
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
Linux_scala_2_13.start: $[ dependencies.Linux_scala_2_13.outputs['start.time'] ]
Linux_scala_2_13.machine: $[ dependencies.Linux_scala_2_13.outputs['start.machine'] ]
Linux_scala_2_13.end: $[ dependencies.Linux_scala_2_13.outputs['end.time'] ]
Linux_scala_2_13.status: $[ dependencies.Linux_scala_2_13.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
Expand Down
108 changes: 57 additions & 51 deletions bazel-java-deps.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,52 +7,59 @@

load("@rules_jvm_external//:defs.bzl", "maven_install")
load("@rules_jvm_external//:specs.bzl", "maven")
load("@scala_version//:index.bzl", "scala_major_version", "scala_version")

version_specific = {
"2.12": [
"ai.x:diff_2.12:2.0.1",
"org.gnieh:diffson-spray-json_2.12:3.1.1",
],
}

def install_java_deps():
maven_install(
artifacts = [
"ai.x:diff_2.12:2.0.1",
artifacts = version_specific.get(scala_major_version, []) + [
"ch.qos.logback:logback-classic:1.2.3",
"ch.qos.logback:logback-core:1.2.3",
"com.auth0:java-jwt:3.10.3",
"com.auth0:jwks-rsa:0.11.0",
"com.chuusai:shapeless_2.12:2.3.3",
"com.chuusai:shapeless_{}:2.3.3".format(scala_major_version),
"com.github.ben-manes.caffeine:caffeine:2.8.0",
"com.github.ghik:silencer-lib_2.12.12:1.7.1",
"com.github.ghik:silencer-plugin_2.12.12:1.7.1",
"com.github.ghik:silencer-lib_{}:1.7.1".format(scala_version),
"com.github.ghik:silencer-plugin_{}:1.7.1".format(scala_version),
"com.github.pureconfig:pureconfig_2.12:0.14.0",
"com.github.pureconfig:pureconfig-core_2.12:0.14.0",
"com.github.pureconfig:pureconfig-generic_2.12:0.14.0",
maven.artifact("com.github.pureconfig", "pureconfig-macros_2.12", "0.14.0", neverlink = True),
"com.github.scopt:scopt_2.12:3.7.1",
"com.github.scopt:scopt_{}:3.7.1".format(scala_major_version),
"com.google.code.findbugs:jsr305:3.0.2",
"com.google.code.gson:gson:2.8.2",
"com.google.guava:guava:29.0-jre",
"com.h2database:h2:1.4.200",
"com.lihaoyi:pprint_2.12:0.6.0",
"com.lihaoyi:pprint_{}:0.6.0".format(scala_major_version),
"commons-io:commons-io:2.5",
"com.sparkjava:spark-core:2.9.1",
"com.squareup:javapoet:1.11.1",
"com.storm-enroute:scalameter_2.12:0.19",
"com.storm-enroute:scalameter-core_2.12:0.19",
"com.typesafe.akka:akka-actor_2.12:2.6.10",
"com.typesafe.akka:akka-actor-typed_2.12:2.6.10",
"com.typesafe.akka:akka-http_2.12:10.2.1",
"com.typesafe.akka:akka-http-spray-json_2.12:10.2.1",
"com.typesafe.akka:akka-http-testkit_2.12:10.2.1",
"com.typesafe.akka:akka-slf4j_2.12:2.6.10",
"com.typesafe.akka:akka-stream_2.12:2.6.10",
"com.typesafe.akka:akka-stream-testkit_2.12:2.6.10",
"com.typesafe.akka:akka-testkit_2.12:2.6.10",
"org.playframework.anorm:anorm_2.12:2.6.8",
"org.playframework.anorm:anorm-akka_2.12:2.6.8",
"com.typesafe.scala-logging:scala-logging_2.12:3.9.2",
"com.storm-enroute:scalameter_{}:0.19".format(scala_major_version),
"com.storm-enroute:scalameter-core_{}:0.19".format(scala_major_version),
"com.typesafe.akka:akka-actor_{}:2.6.10".format(scala_major_version),
"com.typesafe.akka:akka-actor-typed_{}:2.6.10".format(scala_major_version),
"com.typesafe.akka:akka-http_{}:10.2.1".format(scala_major_version),
"com.typesafe.akka:akka-http-spray-json_{}:10.2.1".format(scala_major_version),
"com.typesafe.akka:akka-http-testkit_{}:10.2.1".format(scala_major_version),
"com.typesafe.akka:akka-slf4j_{}:2.6.10".format(scala_major_version),
"com.typesafe.akka:akka-stream_{}:2.6.10".format(scala_major_version),
"com.typesafe.akka:akka-stream-testkit_{}:2.6.10".format(scala_major_version),
"com.typesafe.akka:akka-testkit_{}:2.6.10".format(scala_major_version),
"org.playframework.anorm:anorm_{}:2.6.8".format(scala_major_version),
"org.playframework.anorm:anorm-akka_{}:2.6.8".format(scala_major_version),
"com.typesafe.scala-logging:scala-logging_{}:3.9.2".format(scala_major_version),
"com.zaxxer:HikariCP:3.2.0",
"eu.rekawek.toxiproxy:toxiproxy-java:2.1.3",
"io.circe:circe-core_2.12:0.13.0",
"io.circe:circe-generic_2.12:0.13.0",
"io.circe:circe-parser_2.12:0.13.0",
"io.circe:circe-yaml_2.12:0.13.0",
"io.circe:circe-core_{}:0.13.0".format(scala_major_version),
"io.circe:circe-generic_{}:0.13.0".format(scala_major_version),
"io.circe:circe-parser_{}:0.13.0".format(scala_major_version),
"io.circe:circe-yaml_{}:0.13.0".format(scala_major_version),
"io.dropwizard.metrics:metrics-core:4.1.2",
"io.dropwizard.metrics:metrics-graphite:4.1.2",
"io.dropwizard.metrics:metrics-jmx:4.1.2",
Expand Down Expand Up @@ -91,14 +98,14 @@ def install_java_deps():
# protobuf
"com.google.protobuf:protobuf-java:3.11.0",
#scalapb
"com.thesamet.scalapb:compilerplugin_2.12:0.9.0",
"com.thesamet.scalapb:lenses_2.12:0.9.0",
"com.thesamet.scalapb:protoc-bridge_2.12:0.7.8",
"com.thesamet.scalapb:scalapb-runtime_2.12:0.9.0",
"com.thesamet.scalapb:scalapb-runtime-grpc_2.12:0.9.0",
"com.thesamet.scalapb:compilerplugin_{}:0.9.0".format(scala_major_version),
"com.thesamet.scalapb:lenses_{}:0.9.0".format(scala_major_version),
"com.thesamet.scalapb:protoc-bridge_{}:0.7.8".format(scala_major_version),
"com.thesamet.scalapb:scalapb-runtime_{}:0.9.0".format(scala_major_version),
"com.thesamet.scalapb:scalapb-runtime-grpc_{}:0.9.0".format(scala_major_version),
# ---- end of grpc-protobuf-netty block
"io.reactivex.rxjava2:rxjava:2.2.1",
"io.spray:spray-json_2.12:1.3.5",
"io.spray:spray-json_{}:1.3.5".format(scala_major_version),
"io.zipkin.brave:brave:4.6.0",
"io.zipkin.reporter:zipkin-sender-okhttp3:1.0.4",
"javax.annotation:javax.annotation-api:1.2",
Expand All @@ -112,7 +119,6 @@ def install_java_deps():
"org.checkerframework:checker:2.5.4",
"org.flywaydb:flyway-core:6.5.0",
"org.freemarker:freemarker-gae:2.3.28",
"org.gnieh:diffson-spray-json_2.12:3.1.1",
"org.jline:jline:3.7.1",
"org.jline:jline-reader:3.7.1",
"org.junit.jupiter:junit-jupiter-api:5.0.0",
Expand All @@ -121,32 +127,32 @@ def install_java_deps():
"org.junit.platform:junit-platform-runner:1.0.0",
"org.mockito:mockito-core:3.6.28",
"org.mockito:mockito-inline:3.6.28",
"org.mockito:mockito-scala_2.12:1.16.3",
"org.mockito:mockito-scala_{}:1.16.3".format(scala_major_version),
"org.pcollections:pcollections:2.1.3",
"org.postgresql:postgresql:42.2.18",
"org.reactivestreams:reactive-streams:1.0.2",
"org.reactivestreams:reactive-streams-tck:1.0.2",
"org.sangria-graphql:sangria_2.12:2.0.1",
"org.sangria-graphql:sangria-spray-json_2.12:1.0.2",
"org.scalacheck:scalacheck_2.12:1.14.0",
"org.scala-lang.modules:scala-collection-compat_2.12:2.1.6",
"org.scala-lang.modules:scala-java8-compat_2.12:0.9.0",
"org.sangria-graphql:sangria_{}:2.0.1".format(scala_major_version),
"org.sangria-graphql:sangria-spray-json_{}:1.0.2".format(scala_major_version),
"org.scalacheck:scalacheck_{}:1.14.0".format(scala_major_version),
"org.scala-lang.modules:scala-collection-compat_{}:2.1.6".format(scala_major_version),
"org.scala-lang.modules:scala-java8-compat_{}:0.9.0".format(scala_major_version),
"org.scala-sbt:sbt:1.1.4",
"org.scalactic:scalactic_2.12:3.1.2",
"org.scalatest:scalatest_2.12:3.1.2",
"org.scalatestplus:scalacheck-1-14_2.12:3.1.4.0",
"org.scalatestplus:selenium-3-141_2.12:3.1.3.0",
"org.scalatestplus:testng-6-7_2.12:3.1.4.0",
"org.scalaz:scalaz-core_2.12:7.2.30",
"org.scalaz:scalaz-scalacheck-binding_2.12:7.2.30-scalacheck-1.14",
"org.scalactic:scalactic_{}:3.1.2".format(scala_major_version),
"org.scalatest:scalatest_{}:3.1.2".format(scala_major_version),
"org.scalatestplus:scalacheck-1-14_{}:3.1.4.0".format(scala_major_version),
"org.scalatestplus:selenium-3-141_{}:3.1.3.0".format(scala_major_version),
"org.scalatestplus:testng-6-7_{}:3.1.4.0".format(scala_major_version),
"org.scalaz:scalaz-core_{}:7.2.30".format(scala_major_version),
"org.scalaz:scalaz-scalacheck-binding_{}:7.2.30-scalacheck-1.14".format(scala_major_version),
"org.seleniumhq.selenium:selenium-java:3.12.0",
"org.slf4j:slf4j-api:1.7.26",
"org.slf4j:slf4j-simple:1.7.26",
"org.typelevel:kind-projector_2.12.12:0.11.2",
"org.tpolecat:doobie-core_2.12:0.9.2",
"org.tpolecat:doobie-postgres_2.12:0.9.2",
"org.typelevel:paiges-core_2.12:0.3.2",
"org.wartremover:wartremover_2.12.12:2.4.10",
"org.typelevel:kind-projector_{}:0.11.2".format(scala_version),
"org.tpolecat:doobie-core_{}:0.9.2".format(scala_major_version),
"org.tpolecat:doobie-postgres_{}:0.9.2".format(scala_major_version),
"org.typelevel:paiges-core_{}:0.3.2".format(scala_major_version),
"org.wartremover:wartremover_{}:2.4.10".format(scala_version),
"org.xerial:sqlite-jdbc:3.30.1",
# gatling dependencies
"io.gatling:gatling-app:3.3.1",
Expand All @@ -161,7 +167,7 @@ def install_java_deps():
"com.fasterxml.jackson.core:jackson-databind:2.12.0",
],
fetch_sources = True,
maven_install_json = "@com_github_digital_asset_daml//:maven_install.json",
maven_install_json = "@com_github_digital_asset_daml//:maven_install_{}.json".format(scala_major_version),
override_targets = {
# Replacements for core Scala libraries.
# These libraries must be provided by the Scala toolchain.
Expand Down
Loading

0 comments on commit d24ef94

Please sign in to comment.