From 3370e091e55df7fc208438ec35d20f5b377e134b Mon Sep 17 00:00:00 2001 From: Ratko Veprek Date: Wed, 13 Mar 2024 17:41:57 +0100 Subject: [PATCH] Removed dropwizard dependencies --- bazel-java-deps.bzl | 5 - canton/BUILD.bazel | 2 - .../common/PureConfigReaderWriter.scala | 16 --- .../common/PureConfigReaderWriterSpec.scala | 32 ----- .../canton/metrics/ServicesMetrics.scala | 13 --- .../pureconfigutils/SharedConfigReaders.scala | 9 -- daml-script/test/BUILD.bazel | 1 - ledger-service/cli-opts/BUILD.bazel | 29 ----- .../main/scala/cliopts/GlobalLogLevel.scala | 20 ---- .../src/main/scala/cliopts/Http.scala | 60 ---------- .../src/main/scala/cliopts/Logging.scala | 102 ---------------- .../src/main/scala/cliopts/Metrics.scala | 50 -------- .../src/main/scala/cliopts/package.scala | 13 --- ledger-service/metrics/BUILD.bazel | 31 ----- .../http/metrics/HttpJsonApiMetrics.scala | 103 ---------------- .../api/reporters/MetricsReporting.scala | 90 -------------- observability/metrics/BUILD.bazel | 13 --- .../scala/com/daml/metrics/JvmMetricSet.scala | 51 -------- .../com/daml/metrics/MetricsConfig.scala | 37 ------ .../scala/com/daml/metrics/api/Gauges.scala | 28 ----- .../dropwizard/DropwizardMetricsFactory.scala | 73 ------------ .../ExtendedDropwizardExports.scala | 62 ---------- .../daml/metrics/api/dropwizard/Metrics.scala | 93 --------------- .../OpenTelemetryMetricsFactory.scala | 69 ++++++----- .../api/reporters/MetricsReporter.scala | 110 ------------------ .../api/reporters/PrometheusReporter.scala | 64 ---------- .../dropwizard/DropwizardExportsAccess.scala | 30 ----- .../metrics/api/testing/MetricValues.scala | 23 +--- observability/telemetry/BUILD.bazel | 9 -- .../daml/telemetry/OpenTelemetryOwner.scala | 52 --------- release/artifacts.yaml | 2 - 31 files changed, 34 insertions(+), 1258 deletions(-) delete mode 100644 ledger-service/cli-opts/BUILD.bazel delete mode 100644 ledger-service/cli-opts/src/main/scala/cliopts/GlobalLogLevel.scala delete mode 100644 ledger-service/cli-opts/src/main/scala/cliopts/Http.scala delete mode 100644 ledger-service/cli-opts/src/main/scala/cliopts/Logging.scala delete mode 100644 ledger-service/cli-opts/src/main/scala/cliopts/Metrics.scala delete mode 100644 ledger-service/cli-opts/src/main/scala/cliopts/package.scala delete mode 100644 ledger-service/metrics/BUILD.bazel delete mode 100644 ledger-service/metrics/src/main/scala/com/digitalasset/http/metrics/HttpJsonApiMetrics.scala delete mode 100644 ledger-service/metrics/src/main/scala/com/digitalasset/metrics/api/reporters/MetricsReporting.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/JvmMetricSet.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/api/Gauges.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/DropwizardMetricsFactory.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/ExtendedDropwizardExports.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/Metrics.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/api/reporters/MetricsReporter.scala delete mode 100644 observability/metrics/src/main/scala/com/daml/metrics/api/reporters/PrometheusReporter.scala delete mode 100644 observability/metrics/src/main/scala/io/prometheus/client/dropwizard/DropwizardExportsAccess.scala diff --git a/bazel-java-deps.bzl b/bazel-java-deps.bzl index f44811db4ac4..6ca7ac51bee9 100644 --- a/bazel-java-deps.bzl +++ b/bazel-java-deps.bzl @@ -140,9 +140,6 @@ def install_java_deps(): "io.circe:circe-generic_{}:{}".format(scala_major_version, circe_version), "io.circe:circe-parser_{}:{}".format(scala_major_version, circe_version), "io.circe:circe-yaml_{}:{}".format(scala_major_version, "0.15.0-RC1"), - "io.dropwizard.metrics:metrics-core:{}".format(dropwizard_version), - "io.dropwizard.metrics:metrics-jmx:{}".format(dropwizard_version), - "io.dropwizard.metrics:metrics-jvm:{}".format(dropwizard_version), # "io.gatling.highcharts:gatling-charts-highcharts:{}".format(gatling_version), # "io.gatling:gatling-app:{}".format(gatling_version), # "io.gatling:gatling-charts:{}".format(gatling_version), @@ -199,7 +196,6 @@ def install_java_deps(): "io.opentelemetry:opentelemetry-sdk:{}".format(opentelemetry_version), "io.opentelemetry:opentelemetry-semconv:{}-alpha".format(opentelemetry_version), "io.prometheus:simpleclient:{}".format(prometheus_version), - "io.prometheus:simpleclient_dropwizard:{}".format(prometheus_version), "io.prometheus:simpleclient_httpserver:{}".format(prometheus_version), "io.prometheus:simpleclient_servlet:{}".format(prometheus_version), "io.reactivex.rxjava2:rxjava:2.2.21", @@ -266,7 +262,6 @@ def install_java_deps(): "org.wartremover:wartremover_{}:2.4.21".format(scala_version), "org.xerial:sqlite-jdbc:3.36.0.1", maven.artifact("com.github.pureconfig", "pureconfig-macros_2.12", "0.14.0", neverlink = True), - maven.artifact("io.dropwizard.metrics", "metrics-graphite", dropwizard_version, exclusions = ["com.rabbitmq:amqp-client"]), ], fetch_sources = True, maven_install_json = "@com_github_digital_asset_daml//:maven_install_{}.json".format(scala_major_version), diff --git a/canton/BUILD.bazel b/canton/BUILD.bazel index a0c6347980dc..889adc2a8f46 100644 --- a/canton/BUILD.bazel +++ b/canton/BUILD.bazel @@ -279,7 +279,6 @@ da_scala_library( "@maven//:com_typesafe_scala_logging_scala_logging_2_13", "@maven//:commons_codec_commons_codec", "@maven//:commons_io_commons_io", - "@maven//:io_dropwizard_metrics_metrics_core", "@maven//:io_grpc_grpc_api", "@maven//:io_grpc_grpc_netty", "@maven//:io_grpc_grpc_protobuf", @@ -728,7 +727,6 @@ scala_library( "@maven//:org_apache_pekko_pekko_actor_2_13", "@maven//:org_apache_pekko_pekko_http_2_13", "@maven//:org_apache_pekko_pekko_http_core_2_13", - "@maven//:org_parboiled_parboiled_2_13", "@maven//:org_scala_lang_scala_reflect", "@maven//:org_scalaz_scalaz_core_2_13", "@maven//:org_slf4j_slf4j_api", diff --git a/canton/community/ledger/ledger-api-core/src/main/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriter.scala b/canton/community/ledger/ledger-api-core/src/main/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriter.scala index 51ee411df6e6..babee9fc301d 100644 --- a/canton/community/ledger/ledger-api-core/src/main/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriter.scala +++ b/canton/community/ledger/ledger-api-core/src/main/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriter.scala @@ -5,7 +5,6 @@ package com.digitalasset.canton.ledger.runner.common import com.daml.jwt.JwtTimestampLeeway import com.daml.lf.data.Ref -import com.daml.metrics.api.reporters.MetricsReporter import com.daml.ports.Port import com.digitalasset.canton.ledger.api.tls.TlsVersion.TlsVersion import com.digitalasset.canton.ledger.api.tls.{SecretsUrl, TlsConfiguration, TlsVersion} @@ -63,21 +62,6 @@ class PureConfigReaderWriter(secure: Boolean = true) { .toRight(CannotConvert(str, Duration.getClass.getName, s"Could not convert $str")) } - implicit val metricReporterReader: ConfigReader[MetricsReporter] = { - ConfigReader.fromString[MetricsReporter](ConvertHelpers.catchReadError { s => - MetricsReporter.parseMetricsReporter(s) - }) - } - implicit val metricReporterWriter: ConfigWriter[MetricsReporter] = - ConfigWriter.toString { - case MetricsReporter.Console => "console" - case MetricsReporter.Csv(directory) => s"csv://${directory.toAbsolutePath.toString}" - case MetricsReporter.Graphite(address, prefix) => - s"graphite://${address.getHostName}:${address.getPort}/${prefix.getOrElse("")}" - case MetricsReporter.Prometheus(address) => - s"prometheus://${address.getHostName}:${address.getPort}" - } - implicit val secretsUrlReader: ConfigReader[SecretsUrl] = ConfigReader.fromString[SecretsUrl] { url => Right(SecretsUrl.fromString(url)) diff --git a/canton/community/ledger/ledger-api-core/src/test/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriterSpec.scala b/canton/community/ledger/ledger-api-core/src/test/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriterSpec.scala index afc2a3084b0c..cbac29744cf5 100644 --- a/canton/community/ledger/ledger-api-core/src/test/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriterSpec.scala +++ b/canton/community/ledger/ledger-api-core/src/test/scala/com/digitalasset/canton/ledger/runner/common/PureConfigReaderWriterSpec.scala @@ -4,7 +4,6 @@ package com.digitalasset.canton.ledger.runner.common import com.daml.jwt.JwtTimestampLeeway -import com.daml.metrics.api.reporters.MetricsReporter import com.digitalasset.canton.ledger.api.tls.{SecretsUrl, TlsConfiguration, TlsVersion} import com.digitalasset.canton.ledger.runner.common import com.digitalasset.canton.ledger.runner.common.OptConfigValue.{ @@ -75,7 +74,6 @@ class PureConfigReaderWriterSpec val readerWriter = new PureConfigReaderWriter(secure) import readerWriter.* testReaderWriterIsomorphism(secure, ArbitraryConfig.duration) - testReaderWriterIsomorphism(secure, ArbitraryConfig.metricsReporter) testReaderWriterIsomorphism(secure, Gen.oneOf(TlsVersion.allVersions)) testReaderWriterIsomorphism(secure, ArbitraryConfig.tlsConfiguration) testReaderWriterIsomorphism(secure, ArbitraryConfig.port) @@ -207,36 +205,6 @@ class PureConfigReaderWriterSpec .prettyPrint(0) should include("Unknown key") } - behavior of "MetricsReporter" - - it should "read/write against predefined values" in { - def compare( - reporter: MetricsReporter, - expectedString: String, - ): Assertion = { - metricReporterWriter.to(reporter) shouldBe fromAnyRef(expectedString) - metricReporterReader.from(fromAnyRef(expectedString)).value shouldBe reporter - } - compare( - MetricsReporter.Prometheus(new InetSocketAddress("localhost", 1234)), - "prometheus://localhost:1234", - ) - compare( - MetricsReporter.Graphite(new InetSocketAddress("localhost", 1234)), - "graphite://localhost:1234/", - ) - compare( - MetricsReporter.Graphite(new InetSocketAddress("localhost", 1234), Some("test")), - "graphite://localhost:1234/test", - ) - val path = Path.of("test").toAbsolutePath - compare( - MetricsReporter.Csv(path), - "csv://" + path.toString, - ) - compare(MetricsReporter.Console, "console") - } - behavior of "SecretsUrl" it should "read/write against predefined values" in { diff --git a/canton/community/ledger/ledger-common/src/main/scala/com/digitalasset/canton/metrics/ServicesMetrics.scala b/canton/community/ledger/ledger-common/src/main/scala/com/digitalasset/canton/metrics/ServicesMetrics.scala index f6d8b60973ac..7b11ebbf6b57 100644 --- a/canton/community/ledger/ledger-common/src/main/scala/com/digitalasset/canton/metrics/ServicesMetrics.scala +++ b/canton/community/ledger/ledger-common/src/main/scala/com/digitalasset/canton/metrics/ServicesMetrics.scala @@ -5,7 +5,6 @@ package com.digitalasset.canton.metrics import com.daml.metrics.api.MetricDoc.MetricQualification.{Debug, Saturation, Traffic} import com.daml.metrics.api.MetricHandle.* -import com.daml.metrics.api.dropwizard.DropwizardTimer import com.daml.metrics.api.{MetricDoc, MetricName} class ServicesMetrics( @@ -224,18 +223,6 @@ class ServicesMetrics( object write { val prefix: MetricName = ServicesMetrics.this.prefix :+ "write" - @MetricDoc.Tag( - summary = "The number of submitted transactions by the write service.", - description = """The write service is an internal interface for changing the state through - |the synchronization services. The methods in this interface are all methods - |that are supported uniformly across all ledger implementations. This metric - |exposes the total number of the sumbitted transactions.""", - qualification = Traffic, - ) - @SuppressWarnings(Array("org.wartremover.warts.Null")) - val submitOperationForDocs: Timer = - DropwizardTimer(prefix :+ "submit_transaction" :+ "count", null) - @MetricDoc.FanInstanceTag val submitTransaction: Timer = openTelemetryMetricsFactory.timer(prefix :+ "submit_transaction") @MetricDoc.FanInstanceTag diff --git a/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/pureconfigutils/SharedConfigReaders.scala b/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/pureconfigutils/SharedConfigReaders.scala index 78d6ac57d073..6dde56b5ad11 100644 --- a/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/pureconfigutils/SharedConfigReaders.scala +++ b/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/pureconfigutils/SharedConfigReaders.scala @@ -13,7 +13,6 @@ import scalaz.\/ import scalaz.syntax.std.option.* import java.nio.file.Path -import com.daml.metrics.api.reporters.MetricsReporter final case class HttpServerConfig( address: String = com.digitalasset.canton.cliopts.Http.defaultAddress, @@ -81,12 +80,4 @@ object SharedConfigReaders { } yield ident } - implicit val uriCfgReader: ConfigReader[Uri] = - ConfigReader.fromString[Uri](ConvertHelpers.catchReadError(s => Uri(s))) - - implicit val metricReporterReader: ConfigReader[MetricsReporter] = { - ConfigReader.fromString[MetricsReporter](ConvertHelpers.catchReadError { s => - MetricsReporter.parseMetricsReporter(s.toLowerCase()) - }) - } } diff --git a/daml-script/test/BUILD.bazel b/daml-script/test/BUILD.bazel index c14403713da5..e5a1daaeb3ac 100644 --- a/daml-script/test/BUILD.bazel +++ b/daml-script/test/BUILD.bazel @@ -507,6 +507,5 @@ da_scala_test_suite( "//observability/metrics", "//test-common/canton/it-lib", "@maven//:com_auth0_java_jwt", - "@maven//:io_dropwizard_metrics_metrics_core", ], ) diff --git a/ledger-service/cli-opts/BUILD.bazel b/ledger-service/cli-opts/BUILD.bazel deleted file mode 100644 index cdde4beb8223..000000000000 --- a/ledger-service/cli-opts/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -load( - "//bazel_tools:scala.bzl", - "da_scala_library", - "lf_scalacopts", -) - -da_scala_library( - name = "cli-opts", - srcs = glob(["src/main/scala/**/*.scala"]), - scala_deps = [ - "@maven//:com_github_scopt_scopt", - "@maven//:org_scalaz_scalaz_core", - ], - scalacopts = lf_scalacopts, - tags = ["maven_coordinates=com.daml:http-json-cli-opts:__VERSION__"], - visibility = ["//visibility:public"], - runtime_deps = [ - "@maven//:org_codehaus_janino_janino", - ], - deps = [ - "//observability/metrics", - "@maven//:ch_qos_logback_logback_classic", - "@maven//:ch_qos_logback_logback_core", - "@maven//:org_slf4j_slf4j_api", - ], -) diff --git a/ledger-service/cli-opts/src/main/scala/cliopts/GlobalLogLevel.scala b/ledger-service/cli-opts/src/main/scala/cliopts/GlobalLogLevel.scala deleted file mode 100644 index a6fb877d1dee..000000000000 --- a/ledger-service/cli-opts/src/main/scala/cliopts/GlobalLogLevel.scala +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.cliopts - -import org.slf4j.{Logger, LoggerFactory} -import ch.qos.logback.classic.{Level => LogLevel} - -object GlobalLogLevel { - def set(serviceName: String)(level: LogLevel): Unit = { - val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) - LoggerFactory.getILoggerFactory match { - case loggerContext: ch.qos.logback.classic.LoggerContext => - rootLogger.info(s"${serviceName} verbosity changed to $level") - loggerContext.getLoggerList.forEach(_.setLevel(level)) - case _ => - rootLogger.warn(s"${serviceName} verbosity cannot be set to requested $level") - } - } -} diff --git a/ledger-service/cli-opts/src/main/scala/cliopts/Http.scala b/ledger-service/cli-opts/src/main/scala/cliopts/Http.scala deleted file mode 100644 index 8618e74209e5..000000000000 --- a/ledger-service/cli-opts/src/main/scala/cliopts/Http.scala +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.cliopts - -import scalaz.syntax.id._ -import scalaz.syntax.std.option._ - -import java.io.File -import java.nio.file.Path - -object Http { - val defaultAddress: String = java.net.InetAddress.getLoopbackAddress.getHostAddress - - /** Add options for setting up an HTTP server to `parser`. - * - * @param defaultHttpPort - * If set, http-port is optional, with default given; otherwise the option is required. - * @param portFile - * If set, there will be an optional port-file option; otherwise it will be absent. - */ - def serverParse[C](parser: scopt.OptionParser[C], serviceName: String)( - address: Setter[C, String], - httpPort: Setter[C, Int], - defaultHttpPort: Option[Int], - portFile: Option[Setter[C, Option[Path]]], - ): Unit = { - import parser.opt - - opt[String]("address") - .action((x, c) => address(_ => x, c)) - .optional() - .text( - s"IP address that $serviceName service listens on. Defaults to ${defaultAddress: String}." - ) - - opt[Int]("http-port") - .action((x, c) => httpPort(_ => x, c)) - .into(o => if (defaultHttpPort.isDefined) o.optional() else o.required()) - .text( - s"$serviceName service port number. " + - defaultHttpPort.cata(p => s"Defaults to ${p: Int}. ", "") + - "A port number of 0 will let the system pick an ephemeral port." + - (if (portFile.isDefined) " Consider specifying `--port-file` option with port number 0." - else "") - ) - - portFile foreach { setPortFile => - opt[File]("port-file") - .action((x, c) => setPortFile(_ => Some(x.toPath), c)) - .optional() - .text( - "Optional unique file name where to write the allocated HTTP port number. " + - "If process terminates gracefully, this file will be deleted automatically. " + - s"Used to inform clients in CI about which port $serviceName listens on. " + - "Defaults to none, that is, no file gets created." - ) - } - } -} diff --git a/ledger-service/cli-opts/src/main/scala/cliopts/Logging.scala b/ledger-service/cli-opts/src/main/scala/cliopts/Logging.scala deleted file mode 100644 index 6e8569c961cb..000000000000 --- a/ledger-service/cli-opts/src/main/scala/cliopts/Logging.scala +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.cliopts - -import ch.qos.logback.classic.{Level => LogLevel} - -object Logging { - - def reconfigure(clazz: Class[_]): Unit = { - // Try reconfiguring the library - import ch.qos.logback.core.joran.spi.JoranException - import ch.qos.logback.classic.LoggerContext - import ch.qos.logback.classic.joran.JoranConfigurator - import java.io.File - import java.net.URL - import org.slf4j.LoggerFactory - def reloadConfig(path: URL): Unit = - try { - val context = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] - val configurator = new JoranConfigurator() - configurator.setContext(context) - context.reset() - configurator.doConfigure(path) - } catch { - case je: JoranException => - // Fallback to System.err.println because the logger won't work in any way anymore. - System.err.println( - s"reconfigured failed using url $path: $je" - ) - } - System.getProperty("logback.configurationFile") match { - case null => reloadConfig(clazz.getClassLoader.getResource("logback.xml")) - case path => reloadConfig(new File(path).toURI().toURL()) - } - } - - private val KnownLogLevels = Set("ERROR", "WARN", "INFO", "DEBUG", "TRACE") - - sealed trait LogEncoder - - object LogEncoder { - case object Plain extends LogEncoder - case object Json extends LogEncoder - } - - private implicit val scoptLogLevel: scopt.Read[LogLevel] = scopt.Read.reads { level => - Either - .cond( - KnownLogLevels.contains(level.toUpperCase), - LogLevel.toLevel(level.toUpperCase), - s"Unrecognized logging level $level", - ) - .getOrElse(throw new java.lang.IllegalArgumentException(s"Unknown logging level $level")) - } - - private implicit val scoptLogEncoder: scopt.Read[LogEncoder] = - scopt.Read.reads { encoder => - encoder.toLowerCase match { - case "plain" => LogEncoder.Plain - case "json" => LogEncoder.Json - case _ => - throw new java.lang.IllegalArgumentException(s"Unrecognized logging encoder $encoder") - } - } - - /** Parse in the cli option for the logging level. - */ - def logLevelParse[C]( - parser: scopt.OptionParser[C] - )(logLevel: Setter[C, Option[LogLevel]]): Unit = { - import parser.opt - - opt[LogLevel]("log-level") - .optional() - .action((level, c) => logLevel(_ => Some(level), c)) - .text( - s"Default logging level to use. Available values are ${KnownLogLevels.mkString(", ")}. Defaults to INFO." - ) - () - } - - def logEncoderParse[C]( - parser: scopt.OptionParser[C] - )(logEncoder: Setter[C, LogEncoder]): Unit = { - import parser.opt - - opt[LogEncoder]("log-encoder") - .optional() - .action { - case (LogEncoder.Plain, c) => c - case (encoder, c) => logEncoder(_ => encoder, c) - } - .text("Which encoder to use: plain|json") - () - } - - def setUseJsonLogEncoderSystemProp(): Unit = { - System.setProperty("LOG_FORMAT_JSON", "true") - () - } -} diff --git a/ledger-service/cli-opts/src/main/scala/cliopts/Metrics.scala b/ledger-service/cli-opts/src/main/scala/cliopts/Metrics.scala deleted file mode 100644 index f58c3600eafd..000000000000 --- a/ledger-service/cli-opts/src/main/scala/cliopts/Metrics.scala +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.cliopts - -import com.daml.metrics.api.reporters.MetricsReporter -import scala.concurrent.duration.{Duration, FiniteDuration, NANOSECONDS} -import scala.util.Try - -object Metrics { - private case class DurationFormat(unwrap: FiniteDuration) - - // We're trying to parse the java duration first for backwards compatibility as - // removing it and only supporting the scala duration variant would be a breaking change. - private implicit val scoptDurationFormat: scopt.Read[DurationFormat] = scopt.Read.reads { - duration => - Try { - Duration.fromNanos( - java.time.Duration.parse(duration).toNanos - ) - }.orElse(Try { - Duration(duration) - }).flatMap(duration => - Try { - if (!duration.isFinite) - throw new IllegalArgumentException(s"Input duration $duration is not finite") - else DurationFormat(FiniteDuration(duration.toNanos, NANOSECONDS)) - } - ).get - } - - def metricsReporterParse[C](parser: scopt.OptionParser[C])( - metricsReporter: Setter[C, Option[MetricsReporter]], - metricsReportingInterval: Setter[C, FiniteDuration], - ): Unit = { - import parser.opt - - opt[MetricsReporter]("metrics-reporter") - .action((reporter, config) => metricsReporter(_ => Some(reporter), config)) - .optional() - .text(s"Start a metrics reporter. ${MetricsReporter.cliHint}") - - opt[DurationFormat]("metrics-reporting-interval") - .action((interval, config) => metricsReportingInterval(_ => interval.unwrap, config)) - .optional() - .text("Set metric reporting interval.") - - () - } -} diff --git a/ledger-service/cli-opts/src/main/scala/cliopts/package.scala b/ledger-service/cli-opts/src/main/scala/cliopts/package.scala deleted file mode 100644 index d6d0b0b8111f..000000000000 --- a/ledger-service/cli-opts/src/main/scala/cliopts/package.scala +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml - -package object cliopts { - - /** A lens-style setter. When you want nested structures to be possible, this - * is vastly superior to the more obvious `(B, T) => T`, because unlike that - * one, this form permits nesting via trivial composition. - */ - type Setter[T, B] = (B => B, T) => T -} diff --git a/ledger-service/metrics/BUILD.bazel b/ledger-service/metrics/BUILD.bazel deleted file mode 100644 index 56e764941387..000000000000 --- a/ledger-service/metrics/BUILD.bazel +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -load( - "//bazel_tools:scala.bzl", - "da_scala_library", -) - -da_scala_library( - name = "metrics", - srcs = glob(["src/main/scala/**/*.scala"]), - scala_deps = [], - tags = ["maven_coordinates=com.daml:http-json-metrics:__VERSION__"], - visibility = [ - "//visibility:public", - ], - runtime_deps = [], - deps = [ - "//libs-scala/ledger-resources", - "//libs-scala/resources", - "//libs-scala/resources-grpc", - "//libs-scala/resources-pekko", - "//observability/metrics", - "//observability/pekko-http-metrics", - "//observability/telemetry", - "@maven//:io_dropwizard_metrics_metrics_core", - "@maven//:io_dropwizard_metrics_metrics_jmx", - "@maven//:io_netty_netty_transport", - "@maven//:io_opentelemetry_opentelemetry_api", - ], -) diff --git a/ledger-service/metrics/src/main/scala/com/digitalasset/http/metrics/HttpJsonApiMetrics.scala b/ledger-service/metrics/src/main/scala/com/digitalasset/http/metrics/HttpJsonApiMetrics.scala deleted file mode 100644 index 2186c1e0f639..000000000000 --- a/ledger-service/metrics/src/main/scala/com/digitalasset/http/metrics/HttpJsonApiMetrics.scala +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.http.metrics - -import com.codahale.metrics.MetricRegistry -import com.daml.metrics.api.MetricHandle.{Counter, LabeledMetricsFactory, MetricsFactory, Timer} -import com.daml.metrics.api.MetricName -import com.daml.metrics.api.dropwizard.DropwizardMetricsFactory -import com.daml.metrics.api.noop.NoOpMetricsFactory -import com.daml.metrics.http.{DamlHttpMetrics, DamlWebSocketMetrics} -import com.daml.metrics.{CacheMetrics, HealthMetrics} - -import scala.annotation.nowarn - -object HttpJsonApiMetrics { - lazy val ForTesting = - new HttpJsonApiMetrics( - NoOpMetricsFactory, - NoOpMetricsFactory, - ) - - final val ComponentName = "json_api" -} - -class HttpJsonApiMetrics( - @nowarn @deprecated defaultMetricsFactory: MetricsFactory, - labeledMetricsFactory: LabeledMetricsFactory, -) { - import HttpJsonApiMetrics._ - - @nowarn - def getMetricRegistry: Option[MetricRegistry] = - defaultMetricsFactory match { - case mf: DropwizardMetricsFactory => Some(mf.registry) - case _ => None - } - - val prefix: MetricName = MetricName.Daml :+ "http_json_api" - - @nowarn - object Db { - private val prefix: MetricName = HttpJsonApiMetrics.this.prefix :+ "db" - - val fetchByIdFetch: Timer = defaultMetricsFactory.timer(prefix :+ "fetch_by_id_fetch") - val fetchByIdQuery: Timer = defaultMetricsFactory.timer(prefix :+ "fetch_by_id_query") - val fetchByKeyFetch: Timer = defaultMetricsFactory.timer(prefix :+ "fetch_by_key_fetch") - val fetchByKeyQuery: Timer = defaultMetricsFactory.timer(prefix :+ "fetch_by_key_query") - val searchFetch: Timer = defaultMetricsFactory.timer(prefix :+ "search_fetch") - - /** * Search query metrics ** - */ - val searchQuery: Timer = defaultMetricsFactory.timer(prefix :+ "search_query") - // The search completed count can be approximated by the count of the `searchQuery` timer. - val searchStarted: Counter = defaultMetricsFactory.counter(prefix :+ "search_started_count") - val searchFailed: Counter = - defaultMetricsFactory.counter(prefix :+ "search_failed_count") - - /** * cache metrics ** - */ - val warmCache: Timer = defaultMetricsFactory.timer(prefix :+ "warm_cache") - val cacheUpdate: Timer = defaultMetricsFactory.timer(prefix :+ "cache_update") - // The cache update completed count can be derived from the count of the `cacheUpdate` timer - val cacheUpdateStarted: Counter = - defaultMetricsFactory.counter(prefix :+ "cache_update_started_count") - val cacheUpdateFailed: Counter = - defaultMetricsFactory.counter(prefix :+ "cache_update_failed_count") - } - - val surrogateTemplateIdCache = - new CacheMetrics(prefix :+ "surrogate_tpid_cache", labeledMetricsFactory) - - // Meters how long parsing and decoding of an incoming json payload takes - @nowarn - val incomingJsonParsingAndValidationTimer: Timer = - defaultMetricsFactory.timer(prefix :+ "incoming_json_parsing_and_validation_timing") - // Meters how long the construction of the response json payload takes - @nowarn - val responseCreationTimer: Timer = - defaultMetricsFactory.timer(prefix :+ "response_creation_timing") - // Meters how long a find by contract key database operation takes - @nowarn - val dbFindByContractKey: Timer = - defaultMetricsFactory.timer(prefix :+ "db_find_by_contract_key_timing") - // Meters how long a find by contract id database operation takes - @nowarn - val dbFindByContractId: Timer = - defaultMetricsFactory.timer(prefix :+ "db_find_by_contract_id_timing") - // Meters how long processing of the command submission request takes on the ledger - @nowarn - val commandSubmissionLedgerTimer: Timer = - defaultMetricsFactory.timer(prefix :+ "command_submission_ledger_timing") - // Meters http requests throughput - // Meters how many websocket connections are currently active - @nowarn - val websocketRequestCounter: Counter = - defaultMetricsFactory.counter(prefix :+ "websocket_request_count") - - val http = new DamlHttpMetrics(labeledMetricsFactory, ComponentName) - val websocket = new DamlWebSocketMetrics(labeledMetricsFactory, ComponentName) - - val health = new HealthMetrics(labeledMetricsFactory) -} diff --git a/ledger-service/metrics/src/main/scala/com/digitalasset/metrics/api/reporters/MetricsReporting.scala b/ledger-service/metrics/src/main/scala/com/digitalasset/metrics/api/reporters/MetricsReporting.scala deleted file mode 100644 index 4c1098705186..000000000000 --- a/ledger-service/metrics/src/main/scala/com/digitalasset/metrics/api/reporters/MetricsReporting.scala +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api.reporters - -import java.util.concurrent.TimeUnit - -import com.codahale.metrics.Slf4jReporter.LoggingLevel -import com.codahale.metrics.jmx.JmxReporter -import com.codahale.metrics.{MetricRegistry, Reporter, Slf4jReporter} -import com.daml.ledger.resources.{Resource, ResourceContext, ResourceOwner} -import com.daml.metrics.{HistogramDefinition, JvmMetricSet} -import io.opentelemetry.api.metrics.{Meter => OtelMeter} -import com.daml.telemetry.OpenTelemetryOwner - -import scala.concurrent.Future -import scala.concurrent.duration.Duration - -/** Manages metrics and reporters. - * - * Creates the [[MetricRegistry]]. - * - * All out-of-the-box JVM metrics are added to the registry. - * - * Creates at least two reporters: - * - * - a [[JmxReporter]], which exposes metrics over JMX, and - * - an [[Slf4jReporter]], which logs metrics on shutdown at DEBUG level. - * - * Also optionally creates the reporter specified in the constructor. - * - * Note that metrics are in general light-weight and add negligible overhead. - * They are not visible to everyday users so they can be safely enabled all the time. - */ -final class MetricsReporting[M]( - jmxDomain: String, - extraMetricsReporter: Option[MetricsReporter], - extraMetricsReportingInterval: Duration, - registerGlobalOpenTelemetry: Boolean, - histograms: Seq[HistogramDefinition], -)(metrics: (MetricRegistry, OtelMeter) => M) - extends ResourceOwner[M] { - def acquire()(implicit context: ResourceContext): Resource[M] = { - val registry = new MetricRegistry - registry.registerAll(new JvmMetricSet) - for { - openTelemetry <- OpenTelemetryOwner( - registerGlobalOpenTelemetry, - extraMetricsReporter, - histograms, - ) - .acquire() - _ = if ( - registerGlobalOpenTelemetry - ) // if no global lib is registered there is no point in registering them - JvmMetricSet - .registerObservers() // has to be registered after opentelemetry is created as it uses the global lib - slf4JReporter <- acquire(newSlf4jReporter(registry)) - _ <- acquire(newJmxReporter(registry)) - .map(_.start()) - _ <- extraMetricsReporter.fold(Resource.unit) { reporter => - acquire(reporter.register(registry)) - .map(_.start(extraMetricsReportingInterval.toSeconds, TimeUnit.SECONDS)) - } - // Trigger a report to the SLF4J logger on shutdown. - _ <- Resource(Future.successful(slf4JReporter))(reporter => - Future.successful(reporter.report()) - ) - } yield metrics(registry, openTelemetry.getMeter("daml")) - } - - private def newJmxReporter(registry: MetricRegistry): JmxReporter = - JmxReporter - .forRegistry(registry) - .inDomain(jmxDomain) - .build() - - private def newSlf4jReporter(registry: MetricRegistry): Slf4jReporter = - Slf4jReporter - .forRegistry(registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .withLoggingLevel(LoggingLevel.DEBUG) - .build() - - private def acquire[T <: Reporter](reporter: => T)(implicit - context: ResourceContext - ): Resource[T] = - ResourceOwner.forCloseable(() => reporter).acquire() -} diff --git a/observability/metrics/BUILD.bazel b/observability/metrics/BUILD.bazel index a740ad0269d8..04c30f723157 100644 --- a/observability/metrics/BUILD.bazel +++ b/observability/metrics/BUILD.bazel @@ -12,10 +12,6 @@ da_scala_library( srcs = glob(["src/main/scala/**/*.scala"]), resources = glob(["src/main/resources/**/*"]), scala_deps = [ - "@maven//:com_chuusai_shapeless", - "@maven//:com_github_pureconfig_pureconfig_core", - "@maven//:com_github_pureconfig_pureconfig_generic", - "@maven//:com_github_scopt_scopt", "@maven//:com_thesamet_scalapb_scalapb_runtime", "@maven//:org_apache_pekko_pekko_actor", "@maven//:org_apache_pekko_pekko_stream", @@ -27,23 +23,15 @@ da_scala_library( ], runtime_deps = [], deps = [ - "//libs-scala/build-info", "//libs-scala/concurrent", "//libs-scala/scala-utils", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_protobuf_protobuf_java", - "@maven//:io_dropwizard_metrics_metrics_core", - "@maven//:io_dropwizard_metrics_metrics_graphite", - "@maven//:io_dropwizard_metrics_metrics_jvm", "@maven//:io_grpc_grpc_api", - "@maven//:io_opentelemetry_instrumentation_opentelemetry_runtime_metrics", "@maven//:io_opentelemetry_opentelemetry_api", "@maven//:io_opentelemetry_opentelemetry_context", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", - "@maven//:io_prometheus_simpleclient", - "@maven//:io_prometheus_simpleclient_dropwizard", - "@maven//:io_prometheus_simpleclient_httpserver", "@maven//:org_slf4j_slf4j_api", ], ) @@ -61,7 +49,6 @@ da_scala_library( deps = [ ":metrics", "//libs-scala/scala-utils", - "@maven//:io_dropwizard_metrics_metrics_core", ], ) diff --git a/observability/metrics/src/main/scala/com/daml/metrics/JvmMetricSet.scala b/observability/metrics/src/main/scala/com/daml/metrics/JvmMetricSet.scala deleted file mode 100644 index 874044fe04c2..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/JvmMetricSet.scala +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics - -import java.util - -import io.opentelemetry.instrumentation.runtimemetrics.{GarbageCollector, MemoryPools} -import com.codahale.metrics.jvm.{ - ClassLoadingGaugeSet, - GarbageCollectorMetricSet, - JvmAttributeGaugeSet, - MemoryUsageGaugeSet, - ThreadStatesGaugeSet, -} -import com.codahale.metrics.{Metric, MetricSet} -import com.daml.metrics.JvmMetricSet._ -import com.daml.metrics.api.MetricName - -import scala.jdk.CollectionConverters._ - -class JvmMetricSet extends MetricSet { - private val metricSets = Map( - "class_loader" -> new ClassLoadingGaugeSet, - "garbage_collector" -> new GarbageCollectorMetricSet, - "attributes" -> new JvmAttributeGaugeSet, - "memory_usage" -> new MemoryUsageGaugeSet, - "thread_states" -> new ThreadStatesGaugeSet, - ) - - override def getMetrics: util.Map[String, Metric] = - metricSets.flatMap { case (metricSetName, metricSet) => - val metricSetPrefix = Prefix :+ metricSetName - metricSet.getMetrics.asScala.map { case (metricName, metric) => - (metricSetPrefix :+ metricName).toString -> metric - } - }.asJava -} - -object JvmMetricSet { - private val Prefix = MetricName("jvm") - - def registerObservers(): Unit = { -// BufferPools.registerObservers(openTelemetry) -// Classes.registerObservers(openTelemetry) -// Cpu.registerObservers(openTelemetry) -// Threads.registerObservers(openTelemetry) - MemoryPools.registerObservers() - GarbageCollector.registerObservers() - } -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/MetricsConfig.scala b/observability/metrics/src/main/scala/com/daml/metrics/MetricsConfig.scala index 08943301b2cf..8e694209a7da 100644 --- a/observability/metrics/src/main/scala/com/daml/metrics/MetricsConfig.scala +++ b/observability/metrics/src/main/scala/com/daml/metrics/MetricsConfig.scala @@ -3,41 +3,4 @@ package com.daml.metrics -import com.daml.metrics.api.reporters.MetricsReporter -import pureconfig.{ConfigReader, ConvertHelpers} -import pureconfig.generic.semiauto.deriveReader - -import scala.concurrent.duration._ - final case class HistogramDefinition(nameRegex: String, bucketBoundaries: Seq[Double]) - -final case class MetricsConfig( - reporter: MetricsReporter, - reportingInterval: FiniteDuration, - histograms: Seq[HistogramDefinition], -) - -object MetricsConfig { - - val DefaultMetricsReportingInterval: FiniteDuration = 10.seconds - - implicit val metricReporterReader: ConfigReader[MetricsReporter] = { - ConfigReader.fromString[MetricsReporter](ConvertHelpers.catchReadError { s => - MetricsReporter.parseMetricsReporter(s.toLowerCase()) - }) - } - - implicit val histogramDefinitionReader: ConfigReader[HistogramDefinition] = - deriveReader[HistogramDefinition] - - implicit val metricsConfigReader: ConfigReader[MetricsConfig] = - ConfigReader.forProduct3[MetricsConfig, MetricsReporter, FiniteDuration, Option[ - Seq[HistogramDefinition] - ]]( - "reporter", - "reporting-interval", - "histograms", - ) { (reporter, reportingInterval, optionalHistograms) => - MetricsConfig(reporter, reportingInterval, optionalHistograms.getOrElse(Seq.empty)) - } -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/Gauges.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/Gauges.scala deleted file mode 100644 index b3d0ef83d63f..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/Gauges.scala +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api - -import java.util.concurrent.atomic.AtomicReference - -import com.codahale.metrics.Gauge - -object Gauges { - - trait GaugeWithUpdate[T] extends Gauge[T] { - - def updateValue(x: T): Unit - } - - case class VarGauge[T](initial: T) extends GaugeWithUpdate[T] { - private val ref = new AtomicReference[T](initial) - def updateValue(x: T): Unit = ref.set(x) - def updateValue(up: T => T): Unit = { - val _ = ref.updateAndGet { value => - up(value) - } - } - override def getValue: T = ref.get() - - } -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/DropwizardMetricsFactory.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/DropwizardMetricsFactory.scala deleted file mode 100644 index e3816d38c357..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/DropwizardMetricsFactory.scala +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api.dropwizard - -import com.codahale.{metrics => codahale} -import com.daml.metrics.api.Gauges.VarGauge -import com.daml.metrics.api.MetricHandle.Gauge.{CloseableGauge, SimpleCloseableGauge} -import com.daml.metrics.api.MetricHandle.{Counter, Gauge, Histogram, Meter, MetricsFactory, Timer} -import com.daml.metrics.api.{Gauges, MetricName, MetricsContext} -import com.daml.scalautil.Statement.discard - -import scala.annotation.nowarn -import scala.concurrent.blocking - -@nowarn("cat=deprecation") -class DropwizardMetricsFactory(val registry: codahale.MetricRegistry) extends MetricsFactory { - - override def timer(name: MetricName, description: String = "")(implicit - context: MetricsContext = MetricsContext.Empty - ): Timer = DropwizardTimer(name, registry.timer(name)) - - override def gauge[T](name: MetricName, initial: T, description: String = "")(implicit - context: MetricsContext = MetricsContext.Empty - ): Gauge[T] = { - val registeredGauge = reRegisterGauge[T, VarGauge[T]](name, Gauges.VarGauge(initial)) - DropwizardGauge(name, registeredGauge, () => discard(registry.remove(name))) - } - - override def gaugeWithSupplier[T]( - name: MetricName, - gaugeSupplier: () => T, - description: String = "", - )(implicit - context: MetricsContext = MetricsContext.Empty - ): CloseableGauge = { - reRegisterGauge[T, AsyncGauge[T]]( - name, - AsyncGauge(gaugeSupplier), - ) - SimpleCloseableGauge(name, () => discard(registry.remove(name))) - } - - override def meter(name: MetricName, description: String = "")(implicit - context: MetricsContext = MetricsContext.Empty - ): Meter = { - // This is idempotent - DropwizardMeter(name, registry.meter(name)) - } - - override def counter(name: MetricName, description: String = "")(implicit - context: MetricsContext = MetricsContext.Empty - ): Counter = { - // This is idempotent - DropwizardCounter(name, registry.counter(name)) - } - - override def histogram(name: MetricName, description: String = "")(implicit - context: MetricsContext = MetricsContext.Empty - ): Histogram = { - DropwizardHistogram(name, registry.histogram(name)) - } - - protected def reRegisterGauge[T, G <: codahale.Gauge[T]]( - name: MetricName, - gauge: G, - ): G = blocking { - synchronized { - registry.remove(name) - registry.register(name, gauge) - } - } -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/ExtendedDropwizardExports.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/ExtendedDropwizardExports.scala deleted file mode 100644 index cb7b488aa665..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/ExtendedDropwizardExports.scala +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api.dropwizard - -import java.util.Collections - -import com.codahale.metrics.{MetricRegistry, Snapshot} -import io.prometheus.client.Collector.MetricFamilySamples -import io.prometheus.client.dropwizard.DropwizardExportsAccess - -import scala.jdk.CollectionConverters._ - -final class ExtendedDropwizardExports(metricRegistry: MetricRegistry) - extends DropwizardExportsAccess(metricRegistry) { - - override def fromSnapshotAndCount( - dropwizardName: String, - snapshot: Snapshot, - count: Long, - factor: Double, - helpMessage: String, - ): MetricFamilySamples = { - - val basicMetricFamilySamples = - super.fromSnapshotAndCount(dropwizardName, snapshot, count, factor, helpMessage) - - val extendedMetrics = basicMetricFamilySamples.samples.asScala ++ List( - sampleBuilder - .createSample( - dropwizardName, - "_min", - EmptyJavaList, - EmptyJavaList, - snapshot.getMin * factor, - ), - sampleBuilder.createSample( - dropwizardName, - "_mean", - EmptyJavaList, - EmptyJavaList, - snapshot.getMean * factor, - ), - sampleBuilder.createSample( - dropwizardName, - "_max", - EmptyJavaList, - EmptyJavaList, - snapshot.getMax * factor, - ), - ) - - new MetricFamilySamples( - basicMetricFamilySamples.name, - basicMetricFamilySamples.`type`, - basicMetricFamilySamples.help, - extendedMetrics.asJava, - ) - } - - private val EmptyJavaList = Collections.emptyList[String]() -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/Metrics.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/Metrics.scala deleted file mode 100644 index fb276b31fd43..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/dropwizard/Metrics.scala +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api.dropwizard - -import java.time.Duration -import java.util.concurrent.TimeUnit - -import com.codahale.metrics.Timer.Context -import com.codahale.{metrics => codahale} -import com.daml.metrics.api.MetricHandle.Timer.TimerHandle -import com.daml.metrics.api.MetricHandle.{Counter, Gauge, Histogram, Meter, Timer} -import com.daml.metrics.api.{Gauges, MetricHandle, MetricsContext} - -case class DropwizardTimer(name: String, metric: codahale.Timer) extends Timer { - - def update(duration: Long, unit: TimeUnit)(implicit - context: MetricsContext = MetricsContext.Empty - ): Unit = metric.update(duration, unit) - - def update(duration: Duration)(implicit - context: MetricsContext - ): Unit = metric.update(duration) - override def time[T](call: => T)(implicit - context: MetricsContext = MetricsContext.Empty - ): T = metric.time(() => call) - override def startAsync()(implicit - context: MetricsContext = MetricsContext.Empty - ): TimerHandle = { - val ctx = metric.time() - DropwizardTimerHandle(ctx) - } -} - -final case class DropwizardTimerHandle(ctx: Context) extends TimerHandle { - - override def stop()(implicit context: MetricsContext): Unit = ctx.close() - -} - -sealed case class DropwizardMeter(name: String, metric: codahale.Meter) extends Meter { - - def mark(value: Long)(implicit - context: MetricsContext - ): Unit = metric.mark(value) - -} - -sealed case class DropwizardCounter(name: String, metric: codahale.Counter) extends Counter { - - override def inc()(implicit - context: MetricsContext = MetricsContext.Empty - ): Unit = metric.inc - - override def inc(n: Long)(implicit - context: MetricsContext - ): Unit = metric.inc(n) - - override def dec()(implicit - context: MetricsContext = MetricsContext.Empty - ): Unit = metric.dec - - override def dec(n: Long)(implicit - context: MetricsContext - ): Unit = metric.dec(n) - -} - -sealed case class DropwizardGauge[T](name: String, metric: Gauges.VarGauge[T], cleanUp: () => Unit) - extends Gauge[T] { - def updateValue(newValue: T): Unit = metric.updateValue(newValue) - override def getValue: T = metric.getValue - - override def updateValue(f: T => T): Unit = metric.updateValue(f) - - override def close(): Unit = cleanUp() -} - -sealed case class DropwizardHistogram(name: String, metric: codahale.Histogram) - extends MetricHandle - with Histogram { - override def update(value: Long)(implicit - context: MetricsContext = MetricsContext.Empty - ): Unit = metric.update(value) - override def update(value: Int)(implicit - context: MetricsContext - ): Unit = metric.update(value) -} - -sealed case class AsyncGauge[T](valueSupplier: () => T) extends codahale.Gauge[T] { - - override def getValue: T = valueSupplier() -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/opentelemetry/OpenTelemetryMetricsFactory.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/opentelemetry/OpenTelemetryMetricsFactory.scala index 709f3fd75094..1f53cae4fce8 100644 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/opentelemetry/OpenTelemetryMetricsFactory.scala +++ b/observability/metrics/src/main/scala/com/daml/metrics/api/opentelemetry/OpenTelemetryMetricsFactory.scala @@ -5,9 +5,6 @@ package com.daml.metrics.api.opentelemetry import java.time.Duration import java.util.concurrent.TimeUnit - -import com.daml.buildinfo.BuildInfo -import com.daml.metrics.api.Gauges.VarGauge import com.daml.metrics.api.MetricHandle.Gauge.{CloseableGauge, SimpleCloseableGauge} import com.daml.metrics.api.MetricHandle.Timer.TimerHandle import com.daml.metrics.api.MetricHandle.{ @@ -34,11 +31,11 @@ import io.opentelemetry.api.metrics.{ Meter => OtelMeter, } +import java.util.concurrent.atomic.AtomicReference + class OpenTelemetryMetricsFactory( otelMeter: OtelMeter, - globalMetricsContext: MetricsContext = MetricsContext( - Map("daml_version" -> BuildInfo.Version) - ), + globalMetricsContext: MetricsContext = MetricsContext(), ) extends LabeledMetricsFactory { override def timer(name: MetricName, description: String)(implicit @@ -60,33 +57,28 @@ class OpenTelemetryMetricsFactory( context: MetricsContext = MetricsContext.Empty ): MetricHandle.Gauge[T] = { val attributes = globalMetricsContext.merge(context).asAttributes - initial match { - case longInitial: Int => - val varGauge = new VarGauge[Int](longInitial) - val registeredGauge = - otelMeter.gaugeBuilder(name).ofLongs().setDescription(description).buildWithCallback { - consumer => - consumer.record(varGauge.getValue.toLong, attributes) - } - OpenTelemetryGauge(name, varGauge.asInstanceOf[VarGauge[T]], registeredGauge) - case longInitial: Long => - val varGauge = new VarGauge[Long](longInitial) - val registeredGauge = - otelMeter.gaugeBuilder(name).ofLongs().setDescription(description).buildWithCallback { - consumer => - consumer.record(varGauge.getValue, attributes) - } - OpenTelemetryGauge(name, varGauge.asInstanceOf[VarGauge[T]], registeredGauge) - case doubleInitial: Double => - val varGauge = new VarGauge[Double](doubleInitial) - val registeredGauge = - otelMeter.gaugeBuilder(name).setDescription(description).buildWithCallback { consumer => - consumer.record(varGauge.getValue, attributes) - } - OpenTelemetryGauge(name, varGauge.asInstanceOf[VarGauge[T]], registeredGauge) + val gauge = OpenTelemetryGauge(name, initial) + + val registered = initial match { + case _: Int => + otelMeter.gaugeBuilder(name).ofLongs().setDescription(description).buildWithCallback { + consumer => + consumer.record(gauge.getValue.asInstanceOf[Int].toLong, attributes) + } + case _: Long => + otelMeter.gaugeBuilder(name).ofLongs().setDescription(description).buildWithCallback { + consumer => + consumer.record(gauge.getValue.asInstanceOf[Long], attributes) + } + case _: Double => + otelMeter.gaugeBuilder(name).setDescription(description).buildWithCallback { consumer => + consumer.record(gauge.getValue.asInstanceOf[Double], attributes) + } case _ => throw new IllegalArgumentException("Gauges support only numeric values.") } + gauge.reference.set(Some(registered)) + gauge } override def gaugeWithSupplier[T]( @@ -217,16 +209,21 @@ object OpenTelemetryTimer { } } -case class OpenTelemetryGauge[T](name: String, varGauge: VarGauge[T], reference: AutoCloseable) - extends Gauge[T] { +case class OpenTelemetryGauge[T](name: String, initial: T) extends Gauge[T] { - override def updateValue(newValue: T): Unit = varGauge.updateValue(newValue) + private val ref = new AtomicReference[T](initial) + private[opentelemetry] val reference = new AtomicReference[Option[AutoCloseable]](None) - override def getValue: T = varGauge.getValue + override def updateValue(newValue: T): Unit = ref.set(newValue) + + override def getValue: T = ref.get() + + override def updateValue(f: T => T): Unit = { + val _ = ref.updateAndGet(f(_)) + } - override def close(): Unit = reference.close() + override def close(): Unit = reference.getAndSet(None).foreach(_.close()) - override def updateValue(f: T => T): Unit = varGauge.updateValue(f) } case class OpenTelemetryMeter(name: String, counter: LongCounter, meterContext: MetricsContext) diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/reporters/MetricsReporter.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/reporters/MetricsReporter.scala deleted file mode 100644 index d404257562e3..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/reporters/MetricsReporter.scala +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api.reporters - -import java.net.{InetSocketAddress, URI} -import java.nio.file.{Files, Path, Paths} - -import com.codahale.metrics -import com.codahale.metrics.{MetricRegistry, ScheduledReporter} -import scopt.Read - -import scala.util.control.NonFatal - -sealed abstract class MetricsReporter { - def register(registry: MetricRegistry): ScheduledReporter -} - -object MetricsReporter { - - case object Console extends MetricsReporter { - override def register(registry: MetricRegistry): ScheduledReporter = - metrics.ConsoleReporter - .forRegistry(registry) - .build() - } - - final case class Csv(directory: Path) extends MetricsReporter { - override def register(registry: MetricRegistry): ScheduledReporter = { - Files.createDirectories(directory) - metrics.CsvReporter - .forRegistry(registry) - .build(directory.toFile) - } - } - - final case class Graphite(address: InetSocketAddress, prefix: Option[String] = None) - extends MetricsReporter { - override def register(registry: MetricRegistry): ScheduledReporter = - metrics.graphite.GraphiteReporter - .forRegistry(registry) - .prefixedWith(prefix.orNull) - .build(new metrics.graphite.Graphite(address)) - } - - object Graphite { - val defaultPort: Int = 2003 - } - - final case class Prometheus(address: InetSocketAddress) extends MetricsReporter { - override def register(registry: MetricRegistry): ScheduledReporter = - PrometheusReporter - .forRegistry(registry) - .build(address) - } - - object Prometheus { - val defaultPort: Int = 55001 - } - - def parseMetricsReporter(s: String): MetricsReporter = { - def getAddress(uri: URI, defaultPort: Int) = { - if (uri.getHost == null) { - throw invalidRead - } - val port = if (uri.getPort > 0) uri.getPort else defaultPort - new InetSocketAddress(uri.getHost, port) - } - s match { - case "console" => - Console - case value if value.startsWith("csv://") => - try { - Csv(Paths.get(value.substring("csv://".length))) - } catch { - case NonFatal(exception) => - throw new RuntimeException(cliHint, exception) - } - case value if value.startsWith("graphite://") => - val uri = parseUri(value) - val address = getAddress(uri, Graphite.defaultPort) - val metricPrefix = Some(uri.getPath.stripPrefix("/")).filter(_.nonEmpty) - Graphite(address, metricPrefix) - case value if value.startsWith("prometheus://") => - val uri = parseUri(value) - val address = getAddress(uri, Prometheus.defaultPort) - Prometheus(address) - case _ => - throw invalidRead - } - } - - implicit val metricsReporterRead: Read[MetricsReporter] = { - Read.reads(parseMetricsReporter) - } - - val cliHint: String = - """Must be one of "console", "csv:///PATH", "graphite://HOST[:PORT][/METRIC_PREFIX]", or "prometheus://HOST[:PORT]".""" - - def parseUri(value: String): URI = - try { - new URI(value) - } catch { - case NonFatal(exception) => - throw new RuntimeException(cliHint, exception) - } - - private def invalidRead: RuntimeException = - new RuntimeException(cliHint) -} diff --git a/observability/metrics/src/main/scala/com/daml/metrics/api/reporters/PrometheusReporter.scala b/observability/metrics/src/main/scala/com/daml/metrics/api/reporters/PrometheusReporter.scala deleted file mode 100644 index 20be60281ff8..000000000000 --- a/observability/metrics/src/main/scala/com/daml/metrics/api/reporters/PrometheusReporter.scala +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.metrics.api.reporters - -import java.net.InetSocketAddress -import java.util -import java.util.concurrent.TimeUnit - -import com.codahale.metrics._ -import com.daml.metrics.api.dropwizard.ExtendedDropwizardExports -import io.prometheus.client.CollectorRegistry -import io.prometheus.client.dropwizard.DropwizardExports -import io.prometheus.client.exporter.HTTPServer -import org.slf4j.LoggerFactory - -import scala.annotation.nowarn - -object PrometheusReporter { - - def forRegistry(registry: MetricRegistry) = PrometheusReporter.Builder(registry) - - case class Builder(registry: MetricRegistry, filter: MetricFilter = MetricFilter.ALL) { - def withFilter(filter: MetricFilter): Builder = copy(filter = filter) - def build(address: InetSocketAddress) = new PrometheusReporter(registry, filter, address) - } - -} - -final class PrometheusReporter private ( - registry: MetricRegistry, - filter: MetricFilter, - address: InetSocketAddress, -) extends ScheduledReporter( - registry, - "prometheus-reporter", - filter, - TimeUnit.SECONDS, - TimeUnit.MILLISECONDS, - ) { - private val logger = LoggerFactory.getLogger(getClass) - private val server = new HTTPServer(address.getHostName, address.getPort) - private val exports = new ExtendedDropwizardExports(registry).register[DropwizardExports]() - logger.info(s"Reporting prometheus metrics on ${address}") - - override def report( - gauges: util.SortedMap[String, Gauge[_]], - counters: util.SortedMap[String, Counter], - histograms: util.SortedMap[String, Histogram], - meters: util.SortedMap[String, Meter], - timers: util.SortedMap[String, Timer], - ): Unit = { - // PrometheusReporter does nothing in the standard report callback - // Prometheus infrastructure runs its own background thread that periodically - // pokes DropwizardExports for new metrics - } - - @nowarn("cat=deprecation") - override def stop(): Unit = { - server.stop() - CollectorRegistry.defaultRegistry.unregister(exports) - super.stop() - } -} diff --git a/observability/metrics/src/main/scala/io/prometheus/client/dropwizard/DropwizardExportsAccess.scala b/observability/metrics/src/main/scala/io/prometheus/client/dropwizard/DropwizardExportsAccess.scala deleted file mode 100644 index ebd1ae442b2f..000000000000 --- a/observability/metrics/src/main/scala/io/prometheus/client/dropwizard/DropwizardExportsAccess.scala +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package io.prometheus.client.dropwizard - -import com.codahale.metrics.{MetricRegistry, Snapshot} -import io.prometheus.client.Collector -import io.prometheus.client.dropwizard.samplebuilder.DefaultSampleBuilder - -/** This is a work-around to open up package-private `DropwizardExports` methods so that we can expose - * min/mean/max metrics. Those values are absent in the official prometheus implementation but are - * an essential part of the ledger performance reporting and feature heavily in other back-end - * integrations as well as metrics dashboards. - * - * @param metricRegistry metric registry instance to wrap - */ -class DropwizardExportsAccess(metricRegistry: MetricRegistry) - extends DropwizardExports(metricRegistry) { - - final val sampleBuilder = new DefaultSampleBuilder() - - override def fromSnapshotAndCount( - dropwizardName: String, - snapshot: Snapshot, - count: Long, - factor: Double, - helpMessage: String, - ): Collector.MetricFamilySamples = - super.fromSnapshotAndCount(dropwizardName, snapshot, count, factor, helpMessage) -} diff --git a/observability/metrics/src/test/lib/scala/com/daml/metrics/api/testing/MetricValues.scala b/observability/metrics/src/test/lib/scala/com/daml/metrics/api/testing/MetricValues.scala index 304320ce1a9c..9a090d5446a7 100644 --- a/observability/metrics/src/test/lib/scala/com/daml/metrics/api/testing/MetricValues.scala +++ b/observability/metrics/src/test/lib/scala/com/daml/metrics/api/testing/MetricValues.scala @@ -3,15 +3,8 @@ package com.daml.metrics.api.testing -import com.codahale.metrics.Snapshot import com.daml.metrics.api.MetricHandle.{Counter, Histogram, Meter, Timer} import com.daml.metrics.api.{MetricName, MetricsContext} -import com.daml.metrics.api.dropwizard.{ - DropwizardCounter, - DropwizardHistogram, - DropwizardMeter, - DropwizardTimer, -} import com.daml.metrics.api.testing.InMemoryMetricsFactory.{ InMemoryCounter, InMemoryHistogram, @@ -71,7 +64,6 @@ trait MetricValues { class CounterValues(counter: Counter) { def value: Long = counter match { - case DropwizardCounter(_, metric) => metric.getCount case timer: InMemoryCounter => singleValueFromContexts(timer.markers.toMap).get() case other => throw new IllegalArgumentException(s"Value not supported for $other") @@ -81,7 +73,7 @@ trait MetricValues { class MeterValues(meter: Meter) { def value: Long = meter match { - case DropwizardMeter(_, metric) => metric.getCount + case meter: InMemoryMeter => val contextWithValues = meter.markers.view.mapValues(_.get()).toMap singleValueFromContexts(contextWithValues) @@ -103,12 +95,6 @@ trait MetricValues { class HistogramValues(histogram: Histogram) { - def snapshot: Snapshot = histogram match { - case DropwizardHistogram(_, metric) => metric.getSnapshot - case other => - throw new IllegalArgumentException(s"Snapshot not supported for $other") - } - def values: Seq[Long] = histogram match { case histogram: InMemoryHistogram => singleValueFromContexts(histogram.recordedValues) @@ -129,14 +115,7 @@ trait MetricValues { class TimerValues(timer: Timer) { - def snapshot: Snapshot = timer match { - case DropwizardTimer(_, metric) => metric.getSnapshot - case other => - throw new IllegalArgumentException(s"Snapshot not supported for $other") - } - def count: Long = timer match { - case DropwizardTimer(_, metric) => metric.getCount case timer: InMemoryTimer => singleValueFromContexts(timer.data.recordedValues.view.mapValues(_.size.toLong).toMap) case other => diff --git a/observability/telemetry/BUILD.bazel b/observability/telemetry/BUILD.bazel index 60367e050fd2..db3bf7380dac 100644 --- a/observability/telemetry/BUILD.bazel +++ b/observability/telemetry/BUILD.bazel @@ -18,16 +18,7 @@ da_scala_library( ], runtime_deps = [], deps = [ - "//libs-scala/ledger-resources", - "//libs-scala/resources", "//observability/metrics", - "@maven//:io_opentelemetry_opentelemetry_api", - "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", - "@maven//:io_opentelemetry_opentelemetry_sdk", - "@maven//:io_opentelemetry_opentelemetry_sdk_extension_autoconfigure", - "@maven//:io_opentelemetry_opentelemetry_sdk_extension_autoconfigure_spi", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", - "@maven//:io_opentelemetry_opentelemetry_sdk_trace", - "@maven//:io_prometheus_simpleclient", ], ) diff --git a/observability/telemetry/src/main/scala/com/daml/telemetry/OpenTelemetryOwner.scala b/observability/telemetry/src/main/scala/com/daml/telemetry/OpenTelemetryOwner.scala index 57f80f66f16e..150f4e8c393c 100644 --- a/observability/telemetry/src/main/scala/com/daml/telemetry/OpenTelemetryOwner.scala +++ b/observability/telemetry/src/main/scala/com/daml/telemetry/OpenTelemetryOwner.scala @@ -3,68 +3,16 @@ package com.daml.telemetry -import com.daml.ledger.resources.{Resource, ResourceContext, ResourceOwner} import com.daml.metrics.{ExecutorServiceMetrics, HistogramDefinition} import com.daml.metrics.api.MetricHandle.Histogram import com.daml.metrics.api.opentelemetry.OpenTelemetryTimer -import com.daml.metrics.api.reporters.MetricsReporter -import com.daml.metrics.api.reporters.MetricsReporter.Prometheus import com.daml.metrics.grpc.DamlGrpcServerMetrics -import com.daml.telemetry.OpenTelemetryOwner.addViewsToProvider -import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.exporter.prometheus.PrometheusCollector -import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder import io.opentelemetry.sdk.metrics.common.InstrumentType import io.opentelemetry.sdk.metrics.view.{Aggregation, InstrumentSelector, View} -import scala.annotation.nowarn -import scala.concurrent.Future import scala.jdk.CollectionConverters.SeqHasAsJava -@nowarn("msg=deprecated") -case class OpenTelemetryOwner( - setAsGlobal: Boolean, - reporter: Option[MetricsReporter], - histograms: Seq[HistogramDefinition], -) extends ResourceOwner[OpenTelemetry] { - - override def acquire()(implicit - context: ResourceContext - ): Resource[OpenTelemetry] = { - Resource( - Future { - if (sys.props.get("otel.traces.exporter").isEmpty) { - // if no trace exporter is configured then default to none instead of the oltp default used by the library - sys.props.addOne("otel.traces.exporter" -> "none") - } - AutoConfiguredOpenTelemetrySdk - .builder() - .addMeterProviderCustomizer { case (builder, _) => - val meterProviderBuilder = addViewsToProvider(builder, histograms) - /* To integrate with prometheus we're using the deprecated [[PrometheusCollector]]. - * More details about the deprecation here: https://github.com/open-telemetry/opentelemetry-java/issues/4284 - * This forces us to keep the current OpenTelemetry version (see ticket for potential paths forward). - */ - if (reporter.exists(_.isInstanceOf[Prometheus])) { - meterProviderBuilder.registerMetricReader(PrometheusCollector.create()) - } else meterProviderBuilder - } - .registerShutdownHook(false) - .setResultAsGlobal(setAsGlobal) - .build() - .getOpenTelemetrySdk - } - ) { sdk => - Future { - sdk.getSdkMeterProvider.close() - sdk.getSdkTracerProvider.close() - } - } - } - -} - object OpenTelemetryOwner { def addViewsToProvider( diff --git a/release/artifacts.yaml b/release/artifacts.yaml index d839fa441f91..8257e1f29b0d 100644 --- a/release/artifacts.yaml +++ b/release/artifacts.yaml @@ -75,8 +75,6 @@ type: jar-scala - target: //ledger-service/lf-value-json:lf-value-json type: jar-scala -- target: //ledger-service/cli-opts:cli-opts - type: jar-scala - target: //libs-scala/jwt:jwt type: jar-scala - target: //ledger-service/utils:utils