Make max-inbound-message-size in DAML script/repl configurable (digit…

…al-asset#5996) Fixes digital-asset#5592 The CLI syntax and the defaults follow the JSON API here. changelog_begin - [DAML Script] The maximum inbound message size can now be configured using `--max-inbound-message-size``. This matches the flag in the JSON API. - [DAML REPL] The maximum inbound message size can now be configured using `--max-inbound-message-size``. This matches the flag in the JSON API. changelog_end
kgeonw · May 15, 2020 · 2853a20 · 2853a20
1 parent 8cd3cc4
commit 2853a20
Show file tree

Hide file tree

Showing 13 changed files with 111 additions and 16 deletions.
diff --git a/compiler/damlc/lib/DA/Cli/Damlc.hs b/compiler/damlc/lib/DA/Cli/Damlc.hs
@@ -263,6 +263,11 @@ cmdRepl numProcessors =
             <*> strOption (long "ledger-port" <> help "Port of the ledger API")
             <*> accessTokenFileFlag
             <*> sslConfig
+            <*> optional
+                    (option auto $
+                        long "max-inbound-message-size" <>
+                        help "Optional max inbound message size in bytes."
+                    )
     accessTokenFileFlag = optional . option str $
         long "access-token-file"
         <> metavar "TOKEN_PATH"
@@ -561,8 +566,9 @@ execRepl
     -> String -> String
     -> Maybe FilePath
     -> Maybe ReplClient.ClientSSLConfig
+    -> Maybe ReplClient.MaxInboundMessageSize
     -> Command
-execRepl projectOpts opts scriptDar mainDar ledgerHost ledgerPort mbAuthToken mbSslConf = Command Repl (Just projectOpts) effect
+execRepl projectOpts opts scriptDar mainDar ledgerHost ledgerPort mbAuthToken mbSslConf mbMaxInboundMessageSize = Command Repl (Just projectOpts) effect
   where effect = do
             -- We change directory so make this absolute
             mainDar <- makeAbsolute mainDar
@@ -573,7 +579,7 @@ execRepl projectOpts opts scriptDar mainDar ledgerHost ledgerPort mbAuthToken mb
             logger <- getLogger opts "repl"
             runfilesDir <- locateRunfiles (mainWorkspace </> "compiler/repl-service/server")
             let jar = runfilesDir </> "repl-service.jar"
-            ReplClient.withReplClient (ReplClient.Options jar ledgerHost ledgerPort mbAuthToken mbSslConf Inherit) $ \replHandle _stdout _ph ->
+            ReplClient.withReplClient (ReplClient.Options jar ledgerHost ledgerPort mbAuthToken mbSslConf mbMaxInboundMessageSize Inherit) $ \replHandle _stdout _ph ->
                 withTempDir $ \dir ->
                 withCurrentDirectory dir $ do
                 sdkVer <- fromMaybe SdkVersion.sdkVersion <$> lookupEnv sdkVersionEnvVar

diff --git a/compiler/damlc/tests/src/DA/Test/Repl/FuncTests.hs b/compiler/damlc/tests/src/DA/Test/Repl/FuncTests.hs
@@ -62,7 +62,7 @@ main = do
     withTempFile $ \portFile ->
         withBinaryFile nullDevice WriteMode $ \devNull ->
         bracket (createSandbox portFile devNull defaultSandboxConf { dars = [testDar] }) destroySandbox $ \SandboxResource{sandboxPort} ->
-        ReplClient.withReplClient (ReplClient.Options replJar "localhost" (show sandboxPort) Nothing Nothing CreatePipe) $ \replHandle mbServiceOut processHandle ->
+        ReplClient.withReplClient (ReplClient.Options replJar "localhost" (show sandboxPort) Nothing Nothing Nothing CreatePipe) $ \replHandle mbServiceOut processHandle ->
         -- TODO We could share some of this setup with the actual repl code in damlc.
         withTempDir $ \dir ->
         withCurrentDirectory dir $ do

diff --git a/compiler/repl-service/client/src/DA/Daml/LF/ReplClient.hs b/compiler/repl-service/client/src/DA/Daml/LF/ReplClient.hs
@@ -1,11 +1,12 @@
 -- Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 -- SPDX-License-Identifier: Apache-2.0
-
+{-# LANGUAGE DerivingStrategies #-}
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE MultiWayIf #-}
 module DA.Daml.LF.ReplClient
   ( Options(..)
+  , MaxInboundMessageSize(..)
   , Handle
   , withReplClient
   , loadPackage
@@ -34,12 +35,16 @@ import qualified System.IO as IO
 import System.IO.Extra (withTempFile)
 import System.Process
 
+newtype MaxInboundMessageSize = MaxInboundMessageSize Int
+  deriving newtype Read
+
 data Options = Options
   { optServerJar :: FilePath
   , optLedgerHost :: String
   , optLedgerPort :: String
   , optMbAuthTokenFile :: Maybe FilePath
   , optMbSslConfig :: Maybe ClientSSLConfig
+  , optMaxInboundMessageSize :: Maybe MaxInboundMessageSize
   , optStdout :: StdStream
   -- ^ This is intended for testing so we can redirect stdout there.
   }

diff --git a/compiler/repl-service/server/src/main/scala/com/digitalasset/daml/lf/ReplServiceMain.scala b/compiler/repl-service/server/src/main/scala/com/digitalasset/daml/lf/ReplServiceMain.scala
@@ -41,6 +41,7 @@ object ReplServiceMain extends App {
       ledgerHost: String,
       ledgerPort: Int,
       accessTokenFile: Option[Path],
+      maxInboundMessageSize: Int,
       tlsConfig: Option[TlsConfiguration],
   )
   object Config {
@@ -100,6 +101,12 @@ object ReplServiceMain extends App {
         .action((path, arguments) =>
           arguments.copy(tlsConfig =
             arguments.tlsConfig.fold(Some(TlsConfiguration(true, None, None, None)))(Some(_))))
+
+      opt[Int]("max-inbound-message-size")
+        .action((x, c) => c.copy(maxInboundMessageSize = x))
+        .optional()
+        .text(
+          s"Optional max inbound message size in bytes. Defaults to ${RunnerConfig.DefaultMaxInboundMessageSize}")
     }
     def parse(args: Array[String]): Option[Config] =
       parser.parse(
@@ -109,7 +116,9 @@ object ReplServiceMain extends App {
           ledgerHost = null,
           ledgerPort = 0,
           accessTokenFile = None,
-          tlsConfig = None)
+          tlsConfig = None,
+          maxInboundMessageSize = RunnerConfig.DefaultMaxInboundMessageSize,
+        )
       )
   }
 
@@ -137,7 +146,9 @@ object ReplServiceMain extends App {
     sslContext = config.tlsConfig.flatMap(_.client),
     token = config.accessTokenFile.map(new TokenHolder(_)).flatMap(_.token),
   )
-  val clients = Await.result(Runner.connect(participantParams, clientConfig), 30.seconds)
+  val clients = Await.result(
+    Runner.connect(participantParams, clientConfig, config.maxInboundMessageSize),
+    30.seconds)
 
   val server =
     NettyServerBuilder

diff --git a/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/Runner.scala b/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/Runner.scala
@@ -11,6 +11,7 @@ import akka.stream.Materializer
 import com.typesafe.scalalogging.StrictLogging
 import java.util.UUID
 
+import io.grpc.netty.NettyChannelBuilder
 import scala.concurrent.{ExecutionContext, Future}
 import scalaz.\/-
 import scalaz.std.either._
@@ -139,27 +140,39 @@ object Script {
 }
 
 object Runner {
-  private def connectApiParameters(params: ApiParameters, clientConfig: LedgerClientConfiguration)(
+  private def connectApiParameters(
+      params: ApiParameters,
+      clientConfig: LedgerClientConfiguration,
+      maxInboundMessageSize: Int)(
       implicit ec: ExecutionContext,
       seq: ExecutionSequencerFactory): Future[GrpcLedgerClient] = {
-    LedgerClient.singleHost(params.host, params.port, clientConfig).map(new GrpcLedgerClient(_))
+    LedgerClient
+      .fromBuilder(
+        NettyChannelBuilder
+          .forAddress(params.host, params.port)
+          .maxInboundMessageSize(maxInboundMessageSize),
+        clientConfig,
+      )
+      .map(new GrpcLedgerClient(_))
+//    LedgerClient.singleHost(params.host, params.port, clientConfig).map(new GrpcLedgerClient(_))
   }
   // We might want to have one config per participant at some point but for now this should be sufficient.
   def connect(
       participantParams: Participants[ApiParameters],
-      clientConfig: LedgerClientConfiguration)(
+      clientConfig: LedgerClientConfiguration,
+      maxInboundMessageSize: Int)(
       implicit ec: ExecutionContext,
       seq: ExecutionSequencerFactory): Future[Participants[GrpcLedgerClient]] = {
     for {
       // The standard library is incredibly weird. Option is not Traversable so we have to convert to a list and back.
       // Map is but it doesn’t return a Map so we have to call toMap afterwards.
       defaultClient <- Future
         .traverse(participantParams.default_participant.toList)(x =>
-          connectApiParameters(x, clientConfig))
+          connectApiParameters(x, clientConfig, maxInboundMessageSize))
         .map(_.headOption)
       participantClients <- Future
         .traverse(participantParams.participants: Map[Participant, ApiParameters])({
-          case (k, v) => connectApiParameters(v, clientConfig).map((k, _))
+          case (k, v) => connectApiParameters(v, clientConfig, maxInboundMessageSize).map((k, _))
         })
         .map(_.toMap)
     } yield Participants(defaultClient, participantClients, participantParams.party_participants)

diff --git a/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/RunnerConfig.scala b/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/RunnerConfig.scala
@@ -25,11 +25,13 @@ case class RunnerConfig(
     accessTokenFile: Option[Path],
     tlsConfig: Option[TlsConfiguration],
     jsonApi: Boolean,
+    maxInboundMessageSize: Int,
 )
 
 object RunnerConfig {
 
   val DefaultTimeProviderType: TimeProviderType = TimeProviderType.WallClock
+  val DefaultMaxInboundMessageSize: Int = 4194304
 
   private def validatePath(path: String, message: String): Either[String, Unit] = {
     val readable = Try(Paths.get(path).toFile.canRead).getOrElse(false)
@@ -140,6 +142,12 @@ object RunnerConfig {
       }
       .text("Run DAML Script via the HTTP JSON API instead of via gRPC (experimental).")
 
+    opt[Int]("max-inbound-message-size")
+      .action((x, c) => c.copy(maxInboundMessageSize = x))
+      .optional()
+      .text(
+        s"Optional max inbound message size in bytes. Defaults to $DefaultMaxInboundMessageSize")
+
     help("help").text("Print this usage text")
 
     checkConfig(c => {
@@ -185,6 +193,7 @@ object RunnerConfig {
         accessTokenFile = None,
         tlsConfig = None,
         jsonApi = false,
+        maxInboundMessageSize = DefaultMaxInboundMessageSize,
       )
     )
 }
diff --git a/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/RunnerMain.scala b/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/RunnerMain.scala
@@ -115,7 +115,7 @@ object RunnerMain {
               sslContext = config.tlsConfig.flatMap(_.client),
               token = tokenHolder.flatMap(_.token),
             )
-            Runner.connect(participantParams, clientConfig)
+            Runner.connect(participantParams, clientConfig, config.maxInboundMessageSize)
           }
           result <- Runner.run(dar, scriptId, inputValue, clients, applicationId, timeProvider)
           _ <- Future {

diff --git a/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/TestConfig.scala b/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/TestConfig.scala
@@ -15,6 +15,7 @@ case class TestConfig(
     participantConfig: Option[File],
     timeProviderType: TimeProviderType,
     commandTtl: Duration,
+    maxInboundMessageSize: Int,
 )
 
 object TestConfig {
@@ -53,6 +54,12 @@ object TestConfig {
       }
       .text("TTL in seconds used for commands emitted by the trigger. Defaults to 30s.")
 
+    opt[Int]("max-inbound-message-size")
+      .action((x, c) => c.copy(maxInboundMessageSize = x))
+      .optional()
+      .text(
+        s"Optional max inbound message size in bytes. Defaults to ${RunnerConfig.DefaultMaxInboundMessageSize}")
+
     help("help").text("Print this usage text")
 
     checkConfig(c => {
@@ -75,6 +82,7 @@ object TestConfig {
         participantConfig = None,
         timeProviderType = TimeProviderType.Static,
         commandTtl = Duration.ofSeconds(30L),
+        maxInboundMessageSize = RunnerConfig.DefaultMaxInboundMessageSize,
       )
     )
 }
diff --git a/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/TestMain.scala b/daml-script/runner/src/main/scala/com/digitalasset/daml/lf/engine/script/TestMain.scala
@@ -123,7 +123,7 @@ object TestMain extends StrictLogging {
         }
 
         val flow: Future[Boolean] = for {
-          clients <- Runner.connect(participantParams, clientConfig)
+          clients <- Runner.connect(participantParams, clientConfig, config.maxInboundMessageSize)
           _ <- clients.getParticipant(None) match {
             case Left(err) => throw new RuntimeException(err)
             case Right(client) =>

diff --git a/daml-script/test/daml-script-test-runner.sh b/daml-script/test/daml-script-test-runner.sh
@@ -21,7 +21,7 @@ GREP=$4
 SORT=$5
 
 set +e
-TEST_OUTPUT="$($TEST_RUNNER --dar=$DAR_FILE 2>&1)"
+TEST_OUTPUT="$($TEST_RUNNER --dar=$DAR_FILE --max-inbound-message-size 41943040 2>&1)"
 TEST_RESULT=$?
 set -e
 
@@ -52,6 +52,7 @@ ScriptTest:traceOrder SUCCESS
 ScriptTest:partyIdHintTest SUCCESS
 ScriptTest:sleepTest SUCCESS
 ScriptExample:initializeFixed SUCCESS
+ScriptTest:testMaxInboundMessageSize SUCCESS
 EOF
 )"
 

diff --git a/daml-script/test/daml/ScriptTest.daml b/daml-script/test/daml/ScriptTest.daml
@@ -209,3 +209,26 @@ jsonExpectedFailureCreateAndExercise p = submitMustFail p $ createAndExerciseCmd
 
 jsonAllocateParty : Text -> Script Party
 jsonAllocateParty p = allocatePartyWithHint p (PartyIdHint p)
+
+-- maxInboundMessageSize
+
+template MessageSize
+  with
+    p : Party
+  where
+    signatory p
+    nonconsuming choice CreateN : ()
+      with
+        n : Int
+      controller p
+      do
+        res <- forA [1..n] (\_ -> do
+              create this
+          )
+        return()
+
+testMaxInboundMessageSize : Script () = do
+  p <- allocateParty "p"
+  b <- submit p do createCmd (MessageSize p)
+  submit p do exerciseCmd b CreateN with n = 50000
+  return ()
diff --git a/daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/SingleParticipant.scala b/daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/SingleParticipant.scala
@@ -258,6 +258,22 @@ case class PartyIdHintTest(dar: Dar[(PackageId, Package)], runner: TestRunner) {
   }
 }
 
+case class TestMaxInboundMessageSize(dar: Dar[(PackageId, Package)], runner: TestRunner) {
+  val scriptId =
+    Identifier(dar.main._1, QualifiedName.assertFromString("ScriptTest:testMaxInboundMessageSize"))
+  def runTests(): Unit = {
+    runner.genericTest(
+      "MaxInboundMessageSize",
+      scriptId,
+      None, {
+        case SUnit => Right(())
+        case v => Left(s"Expected SUnit but got $v")
+      },
+      maxInboundMessageSize = RunnerConfig.DefaultMaxInboundMessageSize * 10,
+    )
+  }
+}
+
 // Runs the example from the docs to make sure it doesn’t produce a runtime error.
 case class ScriptExample(dar: Dar[(PackageId, Package)], runner: TestRunner) {
   val scriptId = Identifier(dar.main._1, QualifiedName.assertFromString("ScriptExample:test"))
@@ -369,6 +385,7 @@ object SingleParticipant {
             Time(dar, runner).runTests()
             Sleep(dar, runner).runTests()
             PartyIdHintTest(dar, runner).runTests()
+            TestMaxInboundMessageSize(dar, runner).runTests()
             ScriptExample(dar, runner).runTests()
           case Some(_) =>
             // We can’t test much with auth since most of our tests rely on party allocation and being

diff --git a/daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/TestRunner.scala b/daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/TestRunner.scala
@@ -93,7 +93,8 @@ class TestRunner(
       scriptId: Identifier,
       inputValue: Option[JsValue],
       assertResult: SValue => Either[String, Unit],
-      expectedLog: Option[Seq[String]] = None
+      expectedLog: Option[Seq[String]] = None,
+      maxInboundMessageSize: Int = RunnerConfig.DefaultMaxInboundMessageSize,
   ) = {
 
     LogCollector.clear()
@@ -106,7 +107,8 @@ class TestRunner(
     implicit val materializer: Materializer = Materializer(system)
     implicit val ec: ExecutionContext = system.dispatcher
 
-    val clientsF = Runner.connect(participantParams, clientConfig)
+    val clientsF =
+      Runner.connect(participantParams, clientConfig, maxInboundMessageSize)
 
     val testFlow: Future[Unit] = for {
       clients <- clientsF