Skip to content

Commit

Permalink
Reduce SDK tarball size by 70% by deduplicating Scala dependencies. (d…
Browse files Browse the repository at this point in the history 
…igital-asset#3678)

* Start moving all the jars into a single jar.

* Fold navigator into daml-sdk jar

* include sandbox

* Remove unnecessary compileDeps.

CHANGELOG_BEGIN

- [DAML SDK] Reduced the size of the DAML SDK by about
60% uncompressed, 70% compressed, by deduplicating Scala
dependencies.

CHANGELOG_END

* update copyright header

* buildifier fix
  • Loading branch information
associahedron authored Nov 29, 2019
1 parent 9f18a83 commit 2713d69
Show file tree
Hide file tree
Showing 11 changed files with 182 additions and 49 deletions.
25 changes: 11 additions & 14 deletions daml-assistant/daml-helper/src/DA/Daml/Helper/Run.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,7 @@ navigatorURL (NavigatorPort p) = "http://localhost:" <> show p

withSandbox :: SandboxPort -> [String] -> (Process () () () -> IO a) -> IO a
withSandbox (SandboxPort port) args a = do
withJar sandboxPath [] (["--port", show port] ++ args) $ \ph -> do
withJar damlSdkJar [] (["sandbox", "--port", show port] ++ args) $ \ph -> do
putStrLn "Waiting for sandbox to start: "
-- TODO We need to figure out what a sane timeout for this step.
waitForConnectionOnPort (putStr "." *> threadDelay 500000) port
Expand All @@ -698,18 +698,24 @@ withNavigator (SandboxPort sandboxPort) navigatorPort args a = do
, navigatorPortNavigatorArgs navigatorPort
, args
]
withJar navigatorPath [] navigatorArgs $ \ph -> do
withJar damlSdkJar [] ("navigator":navigatorArgs) $ \ph -> do
putStrLn "Waiting for navigator to start: "
-- TODO We need to figure out a sane timeout for this step.
waitForHttpServer (putStr "." *> threadDelay 500000) (navigatorURL navigatorPort) []
a ph

damlSdkJarFolder :: FilePath
damlSdkJarFolder = "daml-sdk"

damlSdkJar :: FilePath
damlSdkJar = damlSdkJarFolder </> "daml-sdk.jar"

withJsonApi :: SandboxPort -> JsonApiPort -> [String] -> (Process () () () -> IO a) -> IO a
withJsonApi (SandboxPort sandboxPort) (JsonApiPort jsonApiPort) args a = do
logbackArg <- getLogbackArg ("json-api" </> "json-api-logback.xml")
logbackArg <- getLogbackArg (damlSdkJarFolder </> "json-api-logback.xml")
let jsonApiArgs =
["--ledger-host", "localhost", "--ledger-port", show sandboxPort, "--http-port", show jsonApiPort] <> args
withJar jsonApiPath [logbackArg] jsonApiArgs $ \ph -> do
withJar damlSdkJar [logbackArg] ("json-api":jsonApiArgs) $ \ph -> do
putStrLn "Waiting for JSON API to start: "
-- For now, we have a dummy authorization header here to wait for startup since we cannot get a 200
-- response otherwise. We probably want to add some method to detect successful startup without
Expand Down Expand Up @@ -897,7 +903,7 @@ runLedgerNavigator flags remainingArguments = do
writeFileUTF8 navigatorConfPath (T.unpack $ navigatorConfig partyDetails)
unsetEnv "DAML_PROJECT" -- necessary to prevent config contamination
withCurrentDirectory confDir $ do
withJar navigatorPath [] navigatorArgs $ \ph -> do
withJar damlSdkJar [] ("navigator":navigatorArgs) $ \ph -> do
exitCode <- waitExitCode ph
exitWith exitCode

Expand Down Expand Up @@ -1000,12 +1006,3 @@ waitForHttpServer sleep url headers = do
_ -> sleep *> pure Nothing
where isIOException e = isJust (fromException e :: Maybe IOException)
isHttpException e = isJust (fromException e :: Maybe HTTP.HttpException)

sandboxPath :: FilePath
sandboxPath = "sandbox/sandbox.jar"

navigatorPath :: FilePath
navigatorPath = "navigator/navigator.jar"

jsonApiPath :: FilePath
jsonApiPath = "json-api/json-api.jar"
93 changes: 93 additions & 0 deletions daml-assistant/daml-sdk/BUILD.bazel
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# Copyright (c) 2019 The DAML Authors. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

load(
"//bazel_tools:scala.bzl",
"da_scala_binary",
"da_scala_library",
)

compileDeps = [

# EXTRACTOR DEPS
"@maven//:io_grpc_grpc_services", # this one has to go first
"@maven//:io_netty_netty_handler",
"@maven//:io_netty_netty_tcnative_boringssl_static",
"@maven//:com_chuusai_shapeless_2_12",
"@maven//:com_lihaoyi_fansi_2_12",
"@maven//:org_spire_math_kind_projector_2_12",
"@maven//:org_typelevel_cats_core_2_12",
"@maven//:org_typelevel_cats_effect_2_12",
"@maven//:org_typelevel_cats_free_2_12",
"@maven//:org_typelevel_cats_kernel_2_12",
"@maven//:com_github_scopt_scopt_2_12",
"@maven//:com_lihaoyi_pprint_2_12",
"@maven//:com_lihaoyi_sourcecode_2_12",
"@maven//:org_tpolecat_doobie_core_2_12",
"@maven//:org_tpolecat_doobie_free_2_12",
"@maven//:org_tpolecat_doobie_postgres_2_12",
"@maven//:com_typesafe_akka_akka_actor_2_12",
"@maven//:com_typesafe_akka_akka_stream_2_12",
"@maven//:org_scalaz_scalaz_core_2_12",
"@maven//:org_slf4j_slf4j_api",
"@maven//:com_typesafe_scala_logging_scala_logging_2_12",
"@maven//:io_circe_circe_core_2_12",
"@maven//:io_circe_circe_generic_2_12",
"@maven//:io_circe_circe_parser_2_12",
"@maven//:io_spray_spray_json_2_12",
"@maven//:io_grpc_grpc_netty",

# NAVIGATOR DEPS
"@maven//:com_github_pureconfig_pureconfig_2_12",
"@maven//:com_typesafe_akka_akka_http_2_12",
"@maven//:com_typesafe_akka_akka_http_core_2_12",
"@maven//:com_typesafe_akka_akka_http_spray_json_2_12",
"@maven//:com_typesafe_akka_akka_slf4j_2_12",
"@maven//:com_typesafe_akka_akka_stream_testkit_2_12",
"@maven//:com_typesafe_config",
"@maven//:ch_qos_logback_logback_classic",
"@maven//:org_sangria_graphql_sangria_2_12",
"@maven//:org_sangria_graphql_sangria_marshalling_api_2_12",
"@maven//:org_sangria_graphql_sangria_spray_json_2_12",
"@maven//:org_jline_jline",
"@maven//:org_jline_jline_reader",
"@maven//:org_scala_lang_modules_scala_parser_combinators_2_12",
"@maven//:org_gnieh_diffson_core_2_12",
"@maven//:org_gnieh_diffson_spray_json_2_12",
"@maven//:org_xerial_sqlite_jdbc",
]

runtimeDeps = [
"@maven//:ch_qos_logback_logback_classic",
"@maven//:ch_qos_logback_logback_core",
]

da_scala_library(
name = "sdk-lib",
srcs = glob(["src/main/scala/**/*.scala"]),
resources = glob(["src/main/resources/**/*"]),
visibility = ["//visibility:public"],
runtime_deps = runtimeDeps,
deps = compileDeps + [
"//triggers/runner:trigger-runner-lib",
"//daml-script/runner:script-runner-lib",
"//language-support/codegen-main:codegen-main-lib",
"//extractor:extractor",
"//ledger-service/http-json:http-json",
"//navigator/backend:navigator-library",
"//ledger/sandbox:sandbox",
],
)

da_scala_binary(
name = "sdk",
main_class = "com.digitalasset.daml.sdk.SdkMain",
tags = ["maven_coordinates=com.digitalasset.daml.sdk:sdk:__VERSION__"],
visibility = ["//visibility:public"],
deps = [
":sdk-lib",
"//navigator/backend:backend-resources",
"//navigator/backend:frontend-resources",
"//navigator/backend:version-resource",
],
)
15 changes: 15 additions & 0 deletions daml-assistant/daml-sdk/src/main/resources/logback.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>

<logger name="io.netty" level="WARN" />
<logger name="io.grpc.netty" level="WARN" />

<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
</configuration>
29 changes: 29 additions & 0 deletions daml-assistant/daml-sdk/src/main/scala/com/digitalasset/daml/sdk/SdkMain.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright (c) 2019 The DAML Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.digitalasset.daml.sdk
import com.digitalasset.daml.lf.engine.trigger.{RunnerMain => Trigger}
import com.digitalasset.daml.lf.engine.script.{RunnerMain => Script}
import com.digitalasset.codegen.{CodegenMain => Codegen}
import com.digitalasset.extractor.{Main => Extractor}
import com.digitalasset.http.{Main => JsonApi}
import com.digitalasset.navigator.{NavigatorBackend => Navigator}
import com.digitalasset.platform.sandbox.{SandboxMain => Sandbox}

object SdkMain {
def main(args: Array[String]): Unit = {
val command = args(0)
val rest = args.drop(1)
command match {
case "trigger" => Trigger.main(rest)
case "script" => Script.main(rest)
case "codegen" => Codegen.main(rest)
case "extractor" => Extractor.main(rest)
case "json-api" => JsonApi.main(rest)
case "navigator" => Navigator.main(rest)
case "sandbox" => Sandbox.main(rest)
case _ => sys.exit(1)
}
}
}

1 change: 1 addition & 0 deletions daml-lf/interface/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ da_scala_library(
scalacopts = lf_scalacopts,
tags = ["maven_coordinates=com.digitalasset:daml-lf-interface:__VERSION__"],
visibility = [
"//daml-assistant/daml-sdk:__subpackages__",
"//daml-lf:__subpackages__",
"//daml-script:__subpackages__",
"//extractor:__subpackages__",
Expand Down
1 change: 1 addition & 0 deletions extractor/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ da_scala_library(
name = "extractor",
srcs = glob(["src/main/scala/**/*.scala"]),
resources = glob(["src/main/resources/**/*"]),
visibility = ["//visibility:public"],
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
"@maven//:ch_qos_logback_logback_core",
Expand Down
16 changes: 16 additions & 0 deletions language-support/codegen-main/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
load(
"//bazel_tools:scala.bzl",
"da_scala_binary",
"da_scala_library",
"scala_source_jar",
"scaladoc_jar",
)
Expand All @@ -14,6 +15,21 @@ load(
load("//bazel_tools:pom_file.bzl", "pom_file")
load("@os_info//:os_info.bzl", "is_windows")

da_scala_library(
name = "codegen-main-lib",
srcs = glob(["src/main/**/*.scala"]),
visibility = ["//visibility:public"],
deps = [
"//daml-assistant/scala-daml-project-config",
"//language-support/codegen-common",
"//language-support/java/codegen:lib",
"//language-support/scala/codegen:codegen-main",
"@maven//:ch_qos_logback_logback_classic",
"@maven//:com_github_scopt_scopt_2_12",
"@maven//:com_typesafe_scala_logging_scala_logging_2_12",
],
)

da_scala_binary(
name = "codegen-main",
srcs = glob(["src/main/**/*.scala"]),
Expand Down
1 change: 1 addition & 0 deletions ledger-service/http-json/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ da_scala_library(
srcs = glob(["src/main/scala/**/*.scala"]),
scalacopts = hj_scalacopts,
tags = ["maven_coordinates=com.digitalasset.ledger-service:http-json:__VERSION__"],
visibility = ["//visibility:public"],
deps = http_json_deps,
)

Expand Down
4 changes: 4 additions & 0 deletions navigator/backend/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ java_import(
jars = [
"//navigator/frontend:frontend.jar",
],
visibility = ["//visibility:public"],
)

# Version file, as a top level resources.
Expand All @@ -35,12 +36,14 @@ java_library(
resources = [
"//:component-version",
],
visibility = ["//visibility:public"],
)

# Static backend resources.
java_library(
name = "backend-resources",
resources = glob(["src/main/resources/**/*"]),
visibility = ["//visibility:public"],
)

compileDependencies = [
Expand Down Expand Up @@ -110,6 +113,7 @@ da_scala_library(
"src/main/scala/com/digitalasset/navigator/**/*.scala",
]),
scalacopts = navigator_scalacopts,
visibility = ["//visibility:public"],
deps = compileDependencies,
)

Expand Down
32 changes: 4 additions & 28 deletions release/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,16 +67,10 @@ genrule(
"//compiler/damlc:damlc-dist",
"//compiler/daml-extension:vsix",
"//daml-assistant/daml-helper:daml-helper-dist",
"//ledger/sandbox:sandbox-binary_deploy.jar",
"//navigator/backend:navigator-binary_deploy.jar",
"//extractor:extractor-binary_deploy.jar",
"//ledger-service/http-json:http-json-binary_deploy.jar",
"//language-support/codegen-main:codegen-main_deploy.jar",
"//templates:templates-tarball.tar.gz",
"//triggers/daml:daml-trigger.dar",
"//daml-script/daml:daml-script.dar",
"//triggers/runner:trigger-runner_deploy.jar",
"//daml-script/runner:script-runner_deploy.jar",
"//daml-assistant/daml-sdk:sdk_deploy.jar",
],
outs = ["sdk-release-tarball.tar.gz"],
cmd = """
Expand Down Expand Up @@ -109,30 +103,12 @@ genrule(
mkdir -p $$OUT/studio
cp $(location //compiler/daml-extension:vsix) $$OUT/studio/daml-bundled.vsix
mkdir -p $$OUT/sandbox
cp $(location //ledger/sandbox:sandbox-binary_deploy.jar) $$OUT/sandbox/sandbox.jar
mkdir -p $$OUT/navigator
cp $(location //navigator/backend:navigator-binary_deploy.jar) $$OUT/navigator/navigator.jar
mkdir -p $$OUT/extractor
cp $(location //extractor:extractor-binary_deploy.jar) $$OUT/extractor/extractor.jar
mkdir -p $$OUT/json-api
cp $(location //ledger-service/http-json:http-json-binary_deploy.jar) $$OUT/json-api/json-api.jar
cp -L $(location //ledger-service/http-json:release/json-api-logback.xml) $$OUT/json-api/
mkdir -p $$OUT/codegen
cp $(location //language-support/codegen-main:codegen-main_deploy.jar) $$OUT/codegen/codegen.jar
mkdir -p $$OUT/templates
tar xf $(location //templates:templates-tarball.tar.gz) --strip-components=1 -C $$OUT/templates
mkdir -p $$OUT/daml-trigger
cp $(location //triggers/runner:trigger-runner_deploy.jar) $$OUT/daml-trigger/daml-trigger.jar
mkdir -p $$OUT/daml-script
cp $(location //daml-script/runner:script-runner_deploy.jar) $$OUT/daml-script/daml-script.jar
mkdir -p $$OUT/daml-sdk
cp $(location //daml-assistant/daml-sdk:sdk_deploy.jar) $$OUT/daml-sdk/daml-sdk.jar
cp -L $(location //ledger-service/http-json:release/json-api-logback.xml) $$OUT/daml-sdk/
tar zcf $(location sdk-release-tarball.tar.gz) --format=ustar $$OUT
""",
Expand Down
14 changes: 7 additions & 7 deletions release/sdk-config.yaml.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,15 @@ commands:
- name: sandbox
path: daml-helper/daml-helper
desc: "Launch the Sandbox"
args: ["run-jar", "sandbox/sandbox.jar"]
args: ["run-jar", "daml-sdk/daml-sdk.jar", "sandbox"]
- name: navigator
path: daml-helper/daml-helper
desc: "Launch the Navigator"
args: ["run-jar", "navigator/navigator.jar"]
args: ["run-jar", "daml-sdk/daml-sdk.jar", "navigator"]
- name: extractor
path: daml-helper/daml-helper
desc: "Launch the Extractor"
args: ["run-jar", "extractor/extractor.jar"]
args: ["run-jar", "daml-sdk/daml-sdk.jar", "extractor"]
- name: ledger
path: daml-helper/daml-helper
desc: "Interact with a DAML ledger (experimental)"
Expand All @@ -60,16 +60,16 @@ commands:
- name: json-api
path: daml-helper/daml-helper
desc: "Launch the HTTP JSON API (experimental)"
args: ["run-jar", "--logback-config=json-api/json-api-logback.xml", "json-api/json-api.jar"]
args: ["run-jar", "--logback-config=daml-sdk/json-api-logback.xml", "daml-sdk/daml-sdk.jar", "json-api"]
- name: codegen
path: daml-helper/daml-helper
desc: "Run the DAML codegen"
args: ["run-jar", "codegen/codegen.jar"]
args: ["run-jar", "daml-sdk/daml-sdk.jar", "codegen"]
- name: trigger
path: daml-helper/daml-helper
args: ["run-jar", "daml-trigger/daml-trigger.jar"]
args: ["run-jar", "daml-sdk/daml-sdk.jar", "trigger"]
desc: "Run a DAML trigger (experimental)"
- name: script
path: daml-helper/daml-helper
args: ["run-jar", "daml-script/daml-script.jar"]
args: ["run-jar", "daml-sdk/daml-sdk.jar", "script"]
desc: "Run a DAML trigger (experimental)"

0 comments on commit 2713d69

Please sign in to comment.