Skip to content

Commit

Permalink
Make hikari connection pool timeout configurable (#9750)
Browse files Browse the repository at this point in the history 
* 2 seconds rather than 250ms for default hikari connection pool timeout

CHANGELOG_BEGIN
[jdbc ledger] increase default hikari connection pool timeout to 2s from 250ms
CHANGELOG_END

* extract hikari connection timeout configuration

* default of 250ms

* read hikari connection timeout default from config rather than hardcoding for sandbox next

* remove unused import
  • Loading branch information
@dasormeter
dasormeter authored May 20, 2021
1 parent 38cb8b4 commit 92568eb
Show file tree
Hide file tree
Showing 23 changed files with 88 additions and 10 deletions.
4 changes: 3 additions & 1 deletion ledger/participant-integration-api/src/main/scala/platform/apiserver/ApiServerConfig.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,22 @@ package com.daml.platform.apiserver
import java.io.File
import java.nio.file.Path
import java.time.Duration

import com.daml.ledger.api.tls.TlsConfiguration
import com.daml.ledger.participant.state.v1.ParticipantId
import com.daml.ledger.participant.state.v1.SeedService.Seeding
import com.daml.platform.configuration.IndexConfiguration
import com.daml.ports.Port

import scala.concurrent.duration.FiniteDuration

case class ApiServerConfig(
participantId: ParticipantId,
archiveFiles: List[File],
port: Port,
address: Option[String], // This defaults to "localhost" when set to `None`.
jdbcUrl: String,
databaseConnectionPoolSize: Int,
databaseConnectionTimeout: FiniteDuration,
tlsConfig: Option[TlsConfiguration],
maxInboundMessageSize: Int,
eventsPageSize: Int = IndexConfiguration.DefaultEventsPageSize,
Expand Down
1 change: 1 addition & 0 deletions ...r/participant-integration-api/src/main/scala/platform/apiserver/StandaloneApiServer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ final class StandaloneApiServer(
participantId = participantId,
jdbcUrl = config.jdbcUrl,
databaseConnectionPoolSize = config.databaseConnectionPoolSize,
databaseConnectionTimeout = config.databaseConnectionTimeout,
eventsPageSize = config.eventsPageSize,
servicesExecutionContext = servicesExecutionContext,
metrics = metrics,
Expand Down
3 changes: 3 additions & 0 deletions ledger/participant-integration-api/src/main/scala/platform/index/JdbcIndex.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import com.daml.platform.configuration.ServerRole
import com.daml.platform.store.LfValueTranslationCache

import scala.concurrent.ExecutionContext
import scala.concurrent.duration.FiniteDuration

private[platform] object JdbcIndex {
def owner(
Expand All @@ -23,6 +24,7 @@ private[platform] object JdbcIndex {
participantId: ParticipantId,
jdbcUrl: String,
databaseConnectionPoolSize: Int,
databaseConnectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -37,6 +39,7 @@ private[platform] object JdbcIndex {
serverRole = serverRole,
jdbcUrl = jdbcUrl,
databaseConnectionPoolSize = databaseConnectionPoolSize,
databaseConnectionTimeout = databaseConnectionTimeout,
initialLedgerId = ledgerId,
eventsPageSize = eventsPageSize,
servicesExecutionContext = servicesExecutionContext,
Expand Down
3 changes: 3 additions & 0 deletions ledger/participant-integration-api/src/main/scala/platform/index/ReadOnlySqlLedger.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ private[platform] object ReadOnlySqlLedger {
jdbcUrl: String,
initialLedgerId: LedgerId,
databaseConnectionPoolSize: Int,
databaseConnectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand Down Expand Up @@ -125,6 +126,7 @@ private[platform] object ReadOnlySqlLedger {
serverRole,
jdbcUrl,
databaseConnectionPoolSize,
databaseConnectionTimeout,
eventsPageSize,
servicesExecutionContext,
metrics,
Expand All @@ -137,6 +139,7 @@ private[platform] object ReadOnlySqlLedger {
serverRole,
jdbcUrl,
databaseConnectionPoolSize,
databaseConnectionTimeout,
eventsPageSize,
servicesExecutionContext,
metrics,
Expand Down
4 changes: 3 additions & 1 deletion ledger/participant-integration-api/src/main/scala/platform/indexer/IndexerConfig.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ case class IndexerConfig(
jdbcUrl: String,
startupMode: IndexerStartupMode,
databaseConnectionPoolSize: Int = DefaultDatabaseConnectionPoolSize,
databaseConnectionTimeout: FiniteDuration = DefaultDatabaseConnectionTimeout,
restartDelay: FiniteDuration = DefaultRestartDelay,
eventsPageSize: Int = IndexConfiguration.DefaultEventsPageSize,
updatePreparationParallelism: Int = DefaultUpdatePreparationParallelism,
Expand All @@ -36,8 +37,9 @@ object IndexerConfig {

val DefaultUpdatePreparationParallelism = 2
val DefaultRestartDelay: FiniteDuration = 10.seconds
// Should be greater than or equal to the number of pipeline stages.
// Should be greater than or equal to the number of pipeline stages
val DefaultDatabaseConnectionPoolSize: Int = 3
val DefaultDatabaseConnectionTimeout: FiniteDuration = 250.millis
val DefaultAsyncCommitMode: DbType.AsyncCommitMode = DbType.AsynchronousCommit

val DefaultMaxInputBufferSize: Int = 50
Expand Down
2 changes: 2 additions & 0 deletions ledger/participant-integration-api/src/main/scala/platform/indexer/JdbcIndexer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ object JdbcIndexer {
serverRole,
config.jdbcUrl,
config.databaseConnectionPoolSize,
config.databaseConnectionTimeout,
config.eventsPageSize,
servicesExecutionContext,
metrics,
Expand Down Expand Up @@ -130,6 +131,7 @@ object JdbcIndexer {
serverRole,
config.jdbcUrl,
config.databaseConnectionPoolSize,
config.databaseConnectionTimeout,
config.eventsPageSize,
servicesExecutionContext,
metrics,
Expand Down
4 changes: 4 additions & 0 deletions ledger/participant-integration-api/src/main/scala/platform/store/IndexMetadata.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import com.daml.logging.LoggingContext
import com.daml.metrics.Metrics
import com.daml.platform.ApiOffset
import com.daml.platform.configuration.ServerRole

import scala.concurrent.duration._
import scalaz.Tag

import scala.concurrent.{ExecutionContext, Future}
Expand Down Expand Up @@ -47,6 +49,7 @@ object IndexMetadata {
serverRole = ServerRole.ReadIndexMetadata,
jdbcUrl = jdbcUrl,
connectionPoolSize = 1,
connectionTimeout = 250.millis,
eventsPageSize = 1000,
servicesExecutionContext = executionContext,
metrics = new Metrics(new MetricRegistry),
Expand All @@ -61,6 +64,7 @@ object IndexMetadata {
serverRole = ServerRole.ReadIndexMetadata,
jdbcUrl = jdbcUrl,
connectionPoolSize = 1,
connectionTimeout = 250.millis,
eventsPageSize = 1000,
servicesExecutionContext = executionContext,
metrics = new Metrics(new MetricRegistry),
Expand Down
9 changes: 8 additions & 1 deletion ...rticipant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1000,6 +1000,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -1011,6 +1012,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
connectionPoolSize,
connectionTimeout,
eventsPageSize,
validate = false,
servicesExecutionContext,
Expand All @@ -1025,6 +1027,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -1038,6 +1041,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
dbType.maxSupportedWriteConnections(connectionPoolSize),
connectionTimeout,
eventsPageSize,
validate = false,
servicesExecutionContext,
Expand All @@ -1054,6 +1058,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -1067,6 +1072,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
dbType.maxSupportedWriteConnections(connectionPoolSize),
connectionTimeout,
eventsPageSize,
validate = true,
servicesExecutionContext,
Expand Down Expand Up @@ -1128,6 +1134,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
validate: Boolean,
servicesExecutionContext: ExecutionContext,
Expand All @@ -1143,7 +1150,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
connectionPoolSize,
250.millis,
connectionTimeout,
metrics,
jdbcAsyncCommitMode,
)
Expand Down
9 changes: 8 additions & 1 deletion ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1013,6 +1013,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -1023,6 +1024,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
connectionPoolSize,
connectionTimeout,
eventsPageSize,
validate = false,
servicesExecutionContext,
Expand All @@ -1037,6 +1039,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -1049,6 +1052,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
dbType.maxSupportedWriteConnections(connectionPoolSize),
connectionTimeout,
eventsPageSize,
validate = false,
servicesExecutionContext,
Expand All @@ -1065,6 +1069,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
servicesExecutionContext: ExecutionContext,
metrics: Metrics,
Expand All @@ -1077,6 +1082,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
dbType.maxSupportedWriteConnections(connectionPoolSize),
connectionTimeout,
eventsPageSize,
validate = true,
servicesExecutionContext,
Expand Down Expand Up @@ -1116,6 +1122,7 @@ private[platform] object JdbcLedgerDao {
serverRole: ServerRole,
jdbcUrl: String,
connectionPoolSize: Int,
connectionTimeout: FiniteDuration,
eventsPageSize: Int,
validate: Boolean,
servicesExecutionContext: ExecutionContext,
Expand All @@ -1131,7 +1138,7 @@ private[platform] object JdbcLedgerDao {
serverRole,
jdbcUrl,
connectionPoolSize,
250.millis,
connectionTimeout,
metrics,
jdbcAsyncCommitMode,
)
Expand Down
1 change: 1 addition & 0 deletions ...ticipant-integration-api/src/test/lib/scala/platform/store/dao/JdbcLedgerDaoBackend.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ private[dao] trait JdbcLedgerDaoBackend extends AkkaBeforeAndAfterAll {
// this was the previous default.
// keeping it hardcoded here to keep tests working as before extracting the parameter
connectionPoolSize = 16,
connectionTimeout = 250.millis,
eventsPageSize = eventsPageSize,
servicesExecutionContext = executionContext,
metrics = new Metrics(new MetricRegistry),
Expand Down
4 changes: 3 additions & 1 deletion ...ion-api/src/test/lib/scala/platform/store/dao/JdbcLedgerDaoPostCommitValidationSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package com.daml.platform.store.dao

import java.util.UUID

import com.codahale.metrics.MetricRegistry
import com.daml.ledger.resources.ResourceOwner
import com.daml.lf.value.Value.ContractId
Expand All @@ -16,6 +15,8 @@ import org.scalatest.LoneElement
import org.scalatest.flatspec.AsyncFlatSpec
import org.scalatest.matchers.should.Matchers

import scala.concurrent.duration.DurationInt

private[dao] trait JdbcLedgerDaoPostCommitValidationSpec extends LoneElement {
this: AsyncFlatSpec with Matchers with JdbcLedgerDaoSuite =>

Expand All @@ -27,6 +28,7 @@ private[dao] trait JdbcLedgerDaoPostCommitValidationSpec extends LoneElement {
serverRole = ServerRole.Testing(getClass),
jdbcUrl = jdbcUrl,
connectionPoolSize = 16,
connectionTimeout = 250.millis,
eventsPageSize = eventsPageSize,
servicesExecutionContext = executionContext,
metrics = new Metrics(new MetricRegistry),
Expand Down
15 changes: 14 additions & 1 deletion ...ate/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Config.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import java.nio.file.Path
import java.time.Duration
import java.util.UUID
import java.util.concurrent.TimeUnit

import com.daml.caching
import com.daml.ledger.api.tls.TlsConfiguration
import com.daml.ledger.participant.state.kvutils.app.Config.EngineMode
Expand Down Expand Up @@ -151,11 +150,13 @@ object Config {
"port-file, " +
"server-jdbc-url, " +
"api-server-connection-pool-size" +
"api-server-connection-timeout" +
"max-commands-in-flight, " +
"management-service-timeout, " +
"run-mode, " +
"shard-name, " +
"indexer-connection-pool-size, " +
"indexer-connection-timeout, " +
"indexer-max-input-buffer-size, " +
"indexer-input-mapping-parallelism, " +
"indexer-ingestion-parallelism, " +
Expand Down Expand Up @@ -195,10 +196,19 @@ object Config {
.get("api-server-connection-pool-size")
.map(_.toInt)
.getOrElse(ParticipantConfig.DefaultApiServerDatabaseConnectionPoolSize)

val apiServerConnectionTimeout = kv
.get("api-server-connection-timeout")
.map(Duration.parse)
.getOrElse(ParticipantConfig.DefaultApiServerDatabaseConnectionTimeout)
val indexerConnectionPoolSize = kv
.get("indexer-connection-pool-size")
.map(_.toInt)
.getOrElse(ParticipantIndexerConfig.DefaultDatabaseConnectionPoolSize)
val indexerConnectionTimeout = kv
.get("indexer-connection-timeout")
.map(Duration.parse)
.getOrElse(ParticipantConfig.DefaultApiServerDatabaseConnectionTimeout)
val indexerInputMappingParallelism = kv
.get("indexer-input-mapping-parallelism")
.map(_.toInt)
Expand Down Expand Up @@ -257,6 +267,8 @@ object Config {
jdbcUrl,
indexerConfig = ParticipantIndexerConfig(
databaseConnectionPoolSize = indexerConnectionPoolSize,
databaseConnectionTimeout =
FiniteDuration(indexerConnectionTimeout.toMillis, TimeUnit.MILLISECONDS),
allowExistingSchema = false,
maxInputBufferSize = indexerMaxInputBufferSize,
inputMappingParallelism = indexerInputMappingParallelism,
Expand All @@ -268,6 +280,7 @@ object Config {
enableCompression = indexerEnableCompression,
),
apiServerDatabaseConnectionPoolSize = apiServerConnectionPoolSize,
apiServerDatabaseConnectionTimeout = apiServerConnectionTimeout,
maxCommandsInFlight = maxCommandsInFlight,
managementServiceTimeout = managementServiceTimeout,
maxContractStateCacheSize = maxContractStateCacheSize,
Expand Down
6 changes: 6 additions & 0 deletions ...tils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/LedgerFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ import com.daml.platform.indexer.{IndexerConfig, IndexerStartupMode}
import io.grpc.ServerInterceptor
import scopt.OptionParser

import java.util.concurrent.TimeUnit
import scala.annotation.nowarn
import scala.concurrent.duration.FiniteDuration

@nowarn("msg=parameter value config .* is never used") // possibly used in overrides
trait ConfigProvider[ExtraConfig] {
Expand Down Expand Up @@ -65,6 +67,10 @@ trait ConfigProvider[ExtraConfig] {
address = participantConfig.address,
jdbcUrl = participantConfig.serverJdbcUrl,
databaseConnectionPoolSize = participantConfig.apiServerDatabaseConnectionPoolSize,
databaseConnectionTimeout = FiniteDuration(
participantConfig.apiServerDatabaseConnectionTimeout.toMillis,
TimeUnit.MILLISECONDS,
),
tlsConfig = config.tlsConfig,
maxInboundMessageSize = config.maxInboundMessageSize,
eventsPageSize = config.eventsPageSize,
Expand Down
Loading

0 comments on commit 92568eb

Please sign in to comment.