kutils: Make the validation and the preloading of PackageCommitter pa…

…rametric (#7460) This PR creates 3 validation modes: * `Strict`: Specifies that the committer should validate the package before committing them to the ledger. When using this mode, the packages committed to the ledger can be fully trusted and do not have to be validated when loaded into the engine. * `Lenient`: Specifies that the committer should perform a fast validation of the packages before committing them to the ledger. This mode is useful for ledger integrations that cannot handle long-running submissions (> 10s). When using this mode, the packages committed to the ledger cannot be trusted and must be validated every time they are loaded into the engine. * `No`: Specifies that the committer should not perform any validation the packages before committing them to the ledger. This should be used only by non distributed ledgers, like DAML-on-SQL, where the validation done in the API server can be trusted. This PR creates 3 preloading modes: * `Synchronous` : Specifies that the packages should be preloading into the engine before committed. * `Asynchronous`: Specifies that the packages should be preloaded into the engine asynchronously with the rest of the commit process. This mode is useful for ledger integrations that cannot handle long-running submissions (> 10s). Failure of the preloading process will not affect the commit. * `No`: Specifies that the packages should not be preloaded into the engine. CHANGELOG_BEGIN - [Integration Kit] In kvutils, add metric daml.kvutils.committer.package_upload.validate_timer to track package validation time. CHANGELOG_END
digital-asset · Oct 8, 2020 · be35f3a · be35f3a
1 parent cf89f6a
commit be35f3a
Show file tree

Hide file tree

Showing 16 changed files with 963 additions and 248 deletions.
diff --git a/daml-lf/archive/src/main/scala/com/digitalasset/daml/lf/archive/Reader.scala b/daml-lf/archive/src/main/scala/com/digitalasset/daml/lf/archive/Reader.scala
@@ -100,6 +100,7 @@ abstract class Reader[+Pkg] {
 }
 
 object Reader extends Reader[(PackageId, DamlLf.ArchivePayload)] {
+
   final case class ParseError(error: String) extends RuntimeException(error)
 
   def damlLfCodedInputStreamFromBytes(
@@ -133,4 +134,14 @@ object Reader extends Reader[(PackageId, DamlLf.ArchivePayload)] {
       lf: DamlLf.ArchivePayload,
       version: LanguageVersion,
   ): (PackageId, DamlLf.ArchivePayload) = (hash, lf)
+
+  // Archive Reader that just checks package hash.
+  val HashChecker = new Reader[Unit] {
+    override protected[this] def readArchivePayloadOfVersion(
+        hash: PackageId,
+        lf: DamlLf.ArchivePayload,
+        version: LanguageVersion,
+    ): Unit = ()
+  }
+
 }
diff --git a/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/Engine.scala b/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/Engine.scala
@@ -17,6 +17,8 @@ import com.daml.lf.transaction.Node._
 import com.daml.lf.value.Value
 import java.nio.file.Files
 
+import com.daml.lf.validation.Validation
+
 /**
   * Allows for evaluating [[Commands]] and validating [[Transaction]]s.
   * <p>
@@ -402,10 +404,65 @@ class Engine(val config: EngineConfig = EngineConfig.Stable) {
   def preloadPackage(pkgId: PackageId, pkg: Package): Result[Unit] =
     compiledPackages.addPackage(pkgId, pkg)
 
+  /** This method checks a set of packages is self-consistent (it
+    * contains all its dependencies), contains only well-formed
+    * packages (See daml LF spec for more details) and uses only the
+    * allowed language versions (as described by the engine
+    * config).
+    * This is not affected by [[config.packageValidation]] flag.
+    * Package in [[pkgIds]] but not in [[pkgs]] are assumed to be
+    * preloaded.
+    * */
+  def validatePackages(
+      pkgIds: Set[PackageId],
+      pkgs: Map[PackageId, Package],
+  ): Either[Error, Unit] = {
+    val allPackages = pkgs orElse compiledPackages().packages
+    for {
+      _ <- pkgs
+        .collectFirst {
+          case (pkgId, pkg) if !config.allowedLanguageVersions.contains(pkg.languageVersion) =>
+            Error(
+              s"Disallowed language version in package $pkgId: " +
+                s"Expected version between ${config.allowedLanguageVersions.min.pretty} and ${config.allowedLanguageVersions.max.pretty} but got ${pkg.languageVersion.pretty}"
+            )
+        }
+        .toLeft(())
+      _ <- {
+        val unknownPackages = pkgIds.filterNot(allPackages.isDefinedAt)
+        Either.cond(
+          unknownPackages.isEmpty,
+          (),
+          Error(s"Unknown packages ${unknownPackages.mkString(", ")}")
+        )
+      }
+      _ <- {
+        val missingDeps = pkgIds.flatMap(pkgId => allPackages(pkgId).directDeps).filterNot(pkgIds)
+        Either.cond(
+          missingDeps.isEmpty,
+          (),
+          Error(
+            s"The set of packages ${pkgIds.mkString("{'", "', '", "'}")} is not self consistent, the missing dependencies are ${missingDeps
+              .mkString("{'", "', '", "'}")}.")
+        )
+      }
+      _ <- {
+        pkgIds.iterator
+        // we trust already loaded packages
+          .filterNot(compiledPackages.packageIds)
+          .map(Validation.checkPackage(allPackages, _))
+          .collectFirst { case Left(err) => Error(err.pretty) }
+      }.toLeft(())
+
+    } yield ()
+  }
+
 }
 
 object Engine {
 
+  type Packages = Map[PackageId, Package]
+
   def initialSeeding(
       submissionSeed: crypto.Hash,
       participant: Ref.ParticipantId,

diff --git a/daml-lf/validation/BUILD.bazel b/daml-lf/validation/BUILD.bazel
@@ -13,7 +13,10 @@ da_scala_library(
     srcs = glob(["src/main/**/*.scala"]),
     scalacopts = lf_scalacopts,
     tags = ["maven_coordinates=com.daml:daml-lf-validation:__VERSION__"],
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//compiler/scenario-service:__subpackages__",
+        "//daml-lf:__subpackages__",
+    ],
     deps = [
         "//daml-lf/data",
         "//daml-lf/language",

diff --git a/ledger/metrics/src/main/scala/com/daml/metrics/Metrics.scala b/ledger/metrics/src/main/scala/com/daml/metrics/Metrics.scala
@@ -117,6 +117,7 @@ final class Metrics(val registry: MetricRegistry) {
         object packageUpload {
           private val Prefix: MetricName = committer.Prefix :+ "package_upload"
 
+          val validateTimer: Timer = registry.timer(Prefix :+ "validate_timer")
           val preloadTimer: Timer = registry.timer(Prefix :+ "preload_timer")
           val decodeTimer: Timer = registry.timer(Prefix :+ "decode_timer")
           val accepts: Counter = registry.counter(Prefix :+ "accepts")

diff --git a/ledger/participant-integration-api/BUILD.bazel b/ledger/participant-integration-api/BUILD.bazel
@@ -15,7 +15,6 @@ compile_deps = [
     "//daml-lf/transaction",
     "//daml-lf/transaction:transaction_java_proto",
     "//daml-lf/transaction:value_java_proto",
-    "//daml-lf/validation",
     "//language-support/scala/bindings",
     "//ledger-api/rs-grpc-akka",
     "//ledger-api/rs-grpc-bridge",

diff --git a/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServices.scala b/ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServices.scala
@@ -262,6 +262,7 @@ private[daml] object ApiServices {
               transactionsService,
               writeService,
               managementServiceTimeout,
+              engine,
             )
 
         val apiConfigManagementService =

diff --git a/...on-api/src/main/scala/platform/apiserver/services/admin/ApiPackageManagementService.scala b/...on-api/src/main/scala/platform/apiserver/services/admin/ApiPackageManagementService.scala
@@ -22,7 +22,7 @@ import com.daml.ledger.participant.state.index.v2.{
 }
 import com.daml.ledger.participant.state.v1.{SubmissionId, SubmissionResult, WritePackagesService}
 import com.daml.lf.archive.{Dar, DarReader, Decode}
-import com.daml.lf.validation.Validation
+import com.daml.lf.engine.Engine
 import com.daml.logging.{ContextualizedLogger, LoggingContext}
 import com.daml.platform.api.grpc.GrpcApiService
 import com.daml.platform.apiserver.services.admin.ApiPackageManagementService._
@@ -40,6 +40,7 @@ private[apiserver] final class ApiPackageManagementService private (
     packagesWrite: WritePackagesService,
     managementServiceTimeout: Duration,
     materializer: Materializer,
+    engine: Engine,
 )(implicit loggingContext: LoggingContext)
     extends PackageManagementService
     with GrpcApiService {
@@ -87,7 +88,11 @@ private[apiserver] final class ApiPackageManagementService private (
     for {
       dar <- darReader.readArchive("package-upload", stream)
       packages <- Try(dar.all.iterator.map(Decode.decodeArchive).toMap)
-      _ <- Validation.checkPackages(packages).toTry
+      _ <- engine
+        .validatePackages(packages.keySet, packages)
+        .left
+        .map(e => new IllegalArgumentException(e.msg))
+        .toTry
     } yield dar
 
   override def uploadDarFile(request: UploadDarFileRequest): Future[UploadDarFileResponse] = {
@@ -124,6 +129,7 @@ private[apiserver] object ApiPackageManagementService {
       transactionsService: IndexTransactionsService,
       writeBackend: WritePackagesService,
       managementServiceTimeout: Duration,
+      engine: Engine,
   )(implicit mat: Materializer, loggingContext: LoggingContext)
     : PackageManagementServiceGrpc.PackageManagementService with GrpcApiService =
     new ApiPackageManagementService(
@@ -132,6 +138,7 @@ private[apiserver] object ApiPackageManagementService {
       writeBackend,
       managementServiceTimeout,
       mat,
+      engine,
     )
 
   private final class SynchronousResponseStrategy(

diff --git a/ledger/participant-state/kvutils/BUILD.bazel b/ledger/participant-state/kvutils/BUILD.bazel
@@ -108,6 +108,7 @@ da_scala_library(
 da_scala_test_suite(
     name = "kvutils-tests",
     size = "small",
+    timeout = "moderate",
     srcs = glob(["src/test/suite/scala/**/*.scala"]),
     data = [
         "//ledger/test-common:model-tests.dar",
@@ -124,8 +125,11 @@ da_scala_test_suite(
         "//daml-lf/archive:daml_lf_archive_reader",
         "//daml-lf/archive:daml_lf_dev_archive_java_proto",
         "//daml-lf/data",
+        "//daml-lf/encoder",
         "//daml-lf/engine",
+        "//daml-lf/interpreter",
         "//daml-lf/language",
+        "//daml-lf/parser",
         "//daml-lf/transaction",
         "//daml-lf/transaction:transaction_java_proto",
         "//daml-lf/transaction:value_java_proto",