Add a notice to the release testing rotation file (#11142)

Reminds people adding new users to the rotation that they should also be added to DACH-NY/daml-language-ad-hoc to have a box in the cloud for Windows testing. Adds the possibility of adding comments at the beginning of the rotation file. changelog_begin changelog_end
digital-asset · Oct 7, 2021 · d941a85 · d941a85
1 parent 4295ecd
commit d941a85
Show file tree

Hide file tree

Showing 20 changed files with 308 additions and 458 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -134,7 +134,7 @@ jobs:
         msg=$(git log -n1 --format=%b HEAD | head -1)
         # Keep this line in sync with RELEASE_PR_NOTIF in ci/cron/wednesday.yml
         if [ "$msg" = "This PR has been created by a script, which is not very smart" ]; then
-            pr_handler="<@$(head -1 release/rotation | awk '{print $1}')>"
+            pr_handler="<@$(awk '/^[^#]/ {print $1}' release/rotation | head -n 1)>"
         else
             pr_handler=""
         fi

diff --git a/ci/cron/tuesday.yml b/ci/cron/tuesday.yml
@@ -28,6 +28,6 @@ jobs:
 
       source "$(bash_lib)"
 
-      RELEASE_MANAGER=$(head -1 release/rotation | awk '{print $1}')
+      RELEASE_MANAGER=$(awk '/^[^#]/ {print $1}' release/rotation | head -n 1)
 
       tell_slack "$(echo -e "Hi <@$RELEASE_MANAGER>! According to the <https://github.com/digital-asset/daml/blob/main/release/rotation|rotation>, you are in charge of the release tomorrow. Please make sure you plan accordingly, or find a replacement.\n\nIf anyone knows of any reason to delay or block the release (e.g. a PR that needs to get merged first), please make it known in thread before EOD.")"
diff --git a/ci/cron/wednesday.yml b/ci/cron/wednesday.yml
@@ -47,7 +47,7 @@ jobs:
           git branch -D $branch || true
           git checkout -b $branch
           git add .
-          git -c user.name="Azure Pipelines DAML Build" \
+          git -c user.name="Azure Pipelines Daml Build" \
               -c user.email="support@digitalasset.com" \
               commit \
               -m "$(printf "$title\n\n$body\n\nCHANGELOG_BEGIN\nCHANGELOG_END\n")"
@@ -88,10 +88,14 @@ jobs:
           # labels, PRs are issues as far as the GH API is concerned.
       }
       rotate() {
-          local tmp
-          tmp=$(mktemp)
-          cp release/rotation $tmp
-          (tail -n +2 $tmp; head -1 $tmp) > release/rotation
+          # limitation: comments in the middle of the rotation will be hoisted to the top
+          local comments rotation
+          comments=$(mktemp)
+          rotation=$(mktemp)
+          awk '/^#/' release/rotation > $comments
+          awk '/^[^#]/' release/rotation > $rotation
+          cp $comments release/rotation
+          (tail -n +2 $rotation; head -1 $rotation) >> release/rotation
       }
       release_message() {
           local handler=$1
@@ -119,8 +123,8 @@ jobs:
 
       reset
 
-      NEXT_SLACK=$(head -1 release/rotation | awk '{print $1}')
-      NEXT_GH=$(head -1 release/rotation | awk '{print $2}')
+      NEXT_SLACK=$(awk '/^[^#]/ {print $1}' release/rotation | head -n 1)
+      NEXT_GH=$(awk '/^[^#]/ {print $2}' release/rotation | head -n 1)
 
       PREV="v$(head -1 LATEST | awk '{print $2}')"
       ./release.sh new snapshot

diff --git a/ci/prs.yml b/ci/prs.yml
@@ -115,7 +115,7 @@ jobs:
         # API, there is still value in getting the notification on Slack, as
         # we do have the build number and from there we can click through to
         # the PR. Hence the `|| echo ""`.
-        PR_HANDLER=$(head -1 release/rotation | awk '{print $1}')
+        PR_HANDLER=$(awk '/^[^#]/ {print $1}' release/rotation | head -n 1)
 
         case "$(status)" in
           Succeeded*)

diff --git a/ledger/error/src/main/scala/com/daml/error/BaseError.scala b/ledger/error/src/main/scala/com/daml/error/BaseError.scala
@@ -3,7 +3,6 @@
 
 package com.daml.error
 
-import com.daml.logging.{ContextualizedLogger, LoggingContext}
 import io.grpc.StatusRuntimeException
 
 /** The main error interface for everything that should be logged and notified.
@@ -46,17 +45,15 @@ trait BaseError extends LocationMixin {
     */
   def resources: Seq[(ErrorResource, String)] = Seq()
 
-  def logWithContext(
-      logger: ContextualizedLogger,
-      correlationId: Option[String],
-      extra: Map[String, String] = Map(),
-  )(implicit loggingContext: LoggingContext): Unit =
-    code.log(logger, this, correlationId, extra)
+  def logWithContext(extra: Map[String, String] = Map())(implicit
+      errorCodeLoggingContext: ErrorCodeLoggingContext
+  ): Unit =
+    code.log(this, extra)
 
-  def asGrpcErrorFromContext(correlationId: Option[String], logger: ContextualizedLogger)(
-      loggingContext: LoggingContext
+  def asGrpcErrorFromContext(implicit
+      errorCodeLoggingContext: ErrorCodeLoggingContext
   ): StatusRuntimeException =
-    code.asGrpcError(this, logger, correlationId)(loggingContext)
+    code.asGrpcError(this)
 
   /** Returns retryability information of this particular error
     *
@@ -88,7 +85,7 @@ trait LocationMixin {
 
 object BaseError {
   private val ignoreFields = Set("cause", "throwable", "loggingContext")
-  val SECURITY_SENSITIVE_MESSAGE_ON_API =
+  val SecuritySensitiveMessageOnApi =
     "An error occurred. Please contact the operator and inquire about the request"
 
   def extractContext[D](obj: D): Map[String, String] = {
@@ -102,16 +99,13 @@ object BaseError {
   }
 
   abstract class Impl(
-      correlationId: Option[String],
       override val cause: String,
       override val throwableO: Option[Throwable] = None,
   )(implicit override val code: ErrorCode)
       extends BaseError {
 
     /** The logging context obtained when we created the error, usually passed in as implicit */
-    def loggingContext: LoggingContext
-
-    def logger: ContextualizedLogger
+    def loggingContext: ErrorCodeLoggingContext
 
     /** Flag to control if an error should be logged at creation
       *
@@ -120,10 +114,10 @@ object BaseError {
       */
     def logOnCreation: Boolean = true
 
-    def log(): Unit = logWithContext(logger, correlationId)(loggingContext)
+    def log(): Unit = logWithContext()(loggingContext)
 
     def asGrpcError: StatusRuntimeException = {
-      code.asGrpcError(this, logger, correlationId)(loggingContext)
+      code.asGrpcError(this)(loggingContext)
     }
 
     // Automatically log the error on generation

diff --git a/ledger/error/src/main/scala/com/daml/error/ErrorCategory.scala b/ledger/error/src/main/scala/com/daml/error/ErrorCategory.scala
@@ -32,8 +32,7 @@ sealed trait ErrorCategory {
   def asInt: Int
 
   /** Rank used to order severity (internal only) */
-  private[error] def rank: Int
-
+  def rank: Int
 }
 
 object ErrorCategory {

diff --git a/ledger/error/src/main/scala/com/daml/error/ErrorCode.scala b/ledger/error/src/main/scala/com/daml/error/ErrorCode.scala
@@ -3,9 +3,8 @@
 
 package com.daml.error
 
-import com.daml.error.ErrorCode.{StatusInfo, loggingValueToString, truncateResourceForTransport}
+import com.daml.error.ErrorCode.{ValidMetadataKeyRegex, truncateResourceForTransport}
 import com.daml.logging.entries.LoggingValue
-import com.daml.logging.{ContextualizedLogger, LoggingContext}
 import io.grpc.Status.Code
 import io.grpc.StatusRuntimeException
 import io.grpc.protobuf.StatusProto
@@ -60,27 +59,28 @@ abstract class ErrorCode(val id: String, val category: ErrorCategory)(implicit
   def toMsg(cause: => String, correlationId: Option[String]): String =
     s"${codeStr(correlationId)}: ${ErrorCode.truncateCause(cause)}"
 
-  def asGrpcError(err: BaseError, logger: ContextualizedLogger, correlationId: Option[String])(
-      loggingContext: LoggingContext
+  def asGrpcError(err: BaseError)(implicit
+      loggingContext: ErrorCodeLoggingContext
   ): StatusRuntimeException = {
-    val StatusInfo(codeInt, message, contextMap, _) =
-      getStatusInfo(err, correlationId, logger)(loggingContext)
+    val ErrorCode.StatusInfo(codeGrpc, message, contextMap, correlationId) =
+      getStatusInfo(err)
 
-    // Provide error id and context via ErrorInfo
-    val errInfoBuilder = com.google.rpc.ErrorInfo
-      .newBuilder()
-      .setReason(id)
-    val errInfoPrep =
-      if (!code.category.securitySensitive) {
-        contextMap
-          .foldLeft(errInfoBuilder) { case (bld, (k, v)) =>
-            bld.putMetadata(k, v)
-          }
-      } else errInfoBuilder
-    val errInfo = com.google.protobuf.Any.pack(errInfoPrep.build())
-
-    // Build retry info
-    val retryInfo = err.retryable.map { ri =>
+    // provide error id and context via ErrorInfo
+    val errInfoBld = com.google.rpc.ErrorInfo.newBuilder().setReason(id)
+    if (!code.category.securitySensitive) {
+      contextMap.foreach { case (k, v) => errInfoBld.putMetadata(k, v) }
+    }
+
+    // TODO error codes: Resolve dependency and use constant
+    //    val definiteAnswerKey = com.daml.ledger.grpc.GrpcStatuses.DefiniteAnswerKey
+    val definiteAnswerKey = "definite_answer"
+    err.definiteAnswerO.foreach { definiteAnswer =>
+      errInfoBld.putMetadata(definiteAnswerKey, definiteAnswer.toString)
+    }
+    val errInfo = com.google.protobuf.Any.pack(errInfoBld.build())
+
+    // add retry info
+    val retryInfoO = err.retryable.map { ri =>
       val millis = ri.duration.toMillis % 1000
       val seconds = (ri.duration.toMillis - millis) / 1000
       val dt = com.google.protobuf.Duration
@@ -120,10 +120,10 @@ abstract class ErrorCode(val id: String, val category: ErrorCategory)(implicit
     // Build status
     val statusBuilder = com.google.rpc.Status
       .newBuilder()
-      .setCode(codeInt.value())
+      .setCode(codeGrpc.value())
       .setMessage(message)
 
-    (Seq(errInfo) ++ retryInfo.toList ++ requestInfo.toList ++ resourceInfo)
+    (Seq(errInfo) ++ retryInfoO.toList ++ requestInfo.toList ++ resourceInfo)
       .foldLeft(statusBuilder) { case (acc, item) =>
         acc.addDetails(item)
       }
@@ -134,12 +134,6 @@ abstract class ErrorCode(val id: String, val category: ErrorCategory)(implicit
     new ErrorCode.ApiException(ex.getStatus, ex.getTrailers)
   }
 
-  def formatContextAsString(contextMap: Map[String, String]): String =
-    contextMap.view
-      .filter(_._2.nonEmpty)
-      .map { case (k, v) => s"$k=$v" }
-      .mkString(", ")
-
   /** log level of the error code
     *
     * Generally, the log level is defined by the error category. In rare cases, it might be overridden
@@ -150,71 +144,46 @@ abstract class ErrorCode(val id: String, val category: ErrorCategory)(implicit
   /** True if this error may appear on the API */
   protected def exposedViaApi: Boolean = category.grpcCode.nonEmpty
 
-  // TODO error codes: support polymorphic logging/tracing
-  private[error] def log(
-      logger: ContextualizedLogger,
-      err: BaseError,
-      correlationId: Option[String],
-      extra: Map[String, String],
-  )(implicit
-      loggingContext: LoggingContext
-  ): Unit = {
-    val mergedContext = err.context ++ err.location.map(("location", _)).toList.toMap ++ extra
-
-    LoggingContext.withEnrichedLoggingContext(
-      "err-context" -> ("{" + formatContextAsString(mergedContext) + "}")
-    ) { implicit loggingContext =>
-      val message = toMsg(err.cause, correlationId)
-      (logLevel, err.throwableO) match {
-        case (Level.INFO, None) => logger.info(message)
-        case (Level.INFO, Some(tr)) => logger.info(message, tr)
-        case (Level.WARN, None) => logger.warn(message)
-        case (Level.WARN, Some(tr)) => logger.warn(message, tr)
-        // an error that is logged with < INFO is not an error ...
-        case (_, None) => logger.error(message)
-        case (_, Some(tr)) => logger.error(message, tr)
-      }
-    }
-  }
+  /** Log the cause while adding the context into the MDC
+    *
+    * We add the context twice to the MDC: first, every map item is added directly
+    * and then we add a second string version as "err-context". When we log to file,
+    * we add the err-context to the log output.
+    * When we log to JSON, we ignore the err-context field.
+    */
+  def log(err: BaseError, extra: Map[String, String] = Map())(implicit
+      errorCodeLoggingContext: ErrorCodeLoggingContext
+  ): Unit = errorCodeLoggingContext.logError(this, err, logLevel, extra)
 
   def getStatusInfo(
-      err: BaseError,
-      correlationId: Option[String],
-      logger: ContextualizedLogger,
-  )(loggingContext: LoggingContext): StatusInfo = {
+      err: BaseError
+  )(implicit loggingContext: ErrorCodeLoggingContext): ErrorCode.StatusInfo = {
+    val correlationId = loggingContext.correlationId
     val message =
       if (code.category.securitySensitive)
-        s"${BaseError.SECURITY_SENSITIVE_MESSAGE_ON_API}: ${correlationId.getOrElse("<no-correlation-id>")}"
+        s"${BaseError.SecuritySensitiveMessageOnApi} ${correlationId.getOrElse("<no-correlation-id>")}"
       else
-        code.toMsg(err.cause, correlationId)
-
-    val codeInt = category.grpcCode
+        code.toMsg(err.cause, loggingContext.correlationId)
+    val codeGrpc = category.grpcCode
       .getOrElse {
-        logger.warn(s"Passing non-grpc error via grpc $id ")(loggingContext)
+        loggingContext.warn(s"Passing non-grpc error via grpc ${id} ")
         Code.INTERNAL
       }
+    val contextMap = getTruncatedContext(err) + ("category" -> category.asInt.toString)
 
-    val contextMap =
-      getTruncatedContext(err, loggingContext) + ("category" -> category.asInt.toString)
-
-    StatusInfo(codeInt, message, contextMap, correlationId)
+    ErrorCode.StatusInfo(codeGrpc, message, contextMap, correlationId)
   }
 
   private[error] def getTruncatedContext(
-      err: BaseError,
-      loggingContext: LoggingContext,
-  ): Map[String, String] = {
-    val loggingContextEntries =
-      loggingContext.entries.contents.view.map { case (key, value) =>
-        key -> loggingValueToString(value)
-      }
+      err: BaseError
+  )(implicit loggingContext: ErrorCodeLoggingContext): Map[String, String] = {
     val raw =
-      (err.context ++ loggingContextEntries).toSeq.filter(_._2.nonEmpty).sortBy(_._2.length)
+      (err.context ++ loggingContext.properties).toSeq.filter(_._2.nonEmpty).sortBy(_._2.length)
     val maxPerEntry = ErrorCode.MaxContentBytes / Math.max(1, raw.size)
     // truncate smart, starting with the smallest value strings such that likely only truncate the largest args
     raw
       .foldLeft((Map.empty[String, String], 0)) { case ((map, free), (k, v)) =>
-        val adjustedKey = ErrorCode.ValidMetadataKeyRegex.replaceAllIn(k, "").take(63)
+        val adjustedKey = ValidMetadataKeyRegex.replaceAllIn(k, "").take(63)
         val maxSize = free + maxPerEntry - adjustedKey.length
         val truncatedValue = if (maxSize >= v.length || v.isEmpty) v else v.take(maxSize) + "..."
         // note that we silently discard empty context values and we automatically make the
@@ -297,6 +266,16 @@ object ErrorCode {
       ._1
   }
 
+  /** Formats the context as a string for e.g. transport or file logging */
+  def formatContextAsString(contextMap: Map[String, String]): String = {
+    contextMap
+      .filter(_._2.nonEmpty)
+      .map { case (k, v) =>
+        s"$k=$v"
+      }
+      .mkString(", ")
+  }
+
   private[error] def truncateCause(cause: String): String =
     if (cause.length > MaxCauseLogLength) {
       cause.take(MaxCauseLogLength) + "..."