From 57a8d0b37ec9cd63858fb7f99a87579c2bc3ec7b Mon Sep 17 00:00:00 2001 From: Samir Talwar Date: Thu, 14 May 2020 09:06:34 +0200 Subject: [PATCH] CI: Run PostgreSQL once for all Scala tests. (#5919) --- WORKSPACE | 4 + bazel_tools/dev_env_tool/dev_env_tool.bzl | 44 +++++-- build.sh | 59 +++++++-- ci/postgresql.conf | 8 ++ daml-assistant/integration-tests/BUILD.bazel | 6 +- dev-env/bin/envsubst | 1 + .../extractor/services/ExtractorFixture.scala | 4 +- .../on/sql/MainWithEphemeralPostgresql.scala | 4 +- .../MainWithEphemeralPostgresql.scala | 4 +- .../MainWithEphemeralPostgresql.scala | 4 +- .../stores/ledger/sql/SqlLedgerSpec.scala | 28 +--- .../testing/postgresql/PostgresAround.scala | 121 +++++++++++------- .../postgresql/PostgresAroundAll.scala | 4 +- .../postgresql/PostgresAroundEach.scala | 4 +- .../testing/postgresql/PostgresDatabase.scala | 17 ++- .../testing/postgresql/PostgresResource.scala | 4 +- .../testing/postgresql/PostgresServer.scala | 13 ++ ...ixture.scala => PostgresServerPaths.scala} | 8 +- nix/default.nix | 1 + 19 files changed, 214 insertions(+), 124 deletions(-) create mode 100644 ci/postgresql.conf create mode 120000 dev-env/bin/envsubst create mode 100644 libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServer.scala rename libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/{PostgresFixture.scala => PostgresServerPaths.scala} (67%) diff --git a/WORKSPACE b/WORKSPACE index 09768d1d5cee..ac39e57a2d4e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -828,6 +828,10 @@ dev_env_tool( "bin/pg_ctl", "bin/postgres", ], + required_tools = { + "initdb": ["postgres"], + "pg_ctl": ["postgres"], + }, tools = [ "createdb", "dropdb", diff --git a/bazel_tools/dev_env_tool/dev_env_tool.bzl b/bazel_tools/dev_env_tool/dev_env_tool.bzl index 04a971c27334..f2ca6db2d8bd 100644 --- a/bazel_tools/dev_env_tool/dev_env_tool.bzl +++ b/bazel_tools/dev_env_tool/dev_env_tool.bzl @@ -4,31 +4,42 @@ load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") load("@rules_sh//sh:posix.bzl", "posix") -def _create_build_content(rule_name, tools, win_paths, nix_paths): +def _create_build_content(rule_name, is_windows, tools, required_tools, win_paths, nix_paths): content = """ # DO NOT EDIT: automatically generated BUILD file for dev_env_tool.bzl: {rule_name} + package(default_visibility = ["//visibility:public"]) filegroup( name = "all", srcs = glob(["**"]), ) - """.format(rule_name = rule_name) +""".format(rule_name = rule_name) for i in range(0, len(tools)): - content += """ + if is_windows: + content += """ +# Running tools with `bazel run` is not supported on Windows. filegroup( name = "{tool}", - srcs = select({{ - ":windows": ["{win_path}"], - "//conditions:default": ["{nix_path}"], - }}), + srcs = ["{path}"], ) - """.format( - tool = tools[i], - win_path = win_paths[i], - nix_path = nix_paths[i], - ) +""".format( + tool = tools[i], + path = win_paths[i], + ) + else: + content += """ +sh_binary( + name = "{tool}", + srcs = ["{path}"], + data = {dependencies}, +) +""".format( + tool = tools[i], + dependencies = [":{}".format(dep) for dep in required_tools.get(tools[i], [])], + path = nix_paths[i], + ) content += """ config_setting( @@ -95,7 +106,8 @@ dadew = repository_rule( ) def _dev_env_tool_impl(ctx): - if get_cpu_value(ctx) == "x64_windows": + is_windows = get_cpu_value(ctx) == "x64_windows" + if is_windows: ps = ctx.which("powershell") dadew = _dadew_where(ctx, ps) find = _dadew_tool_home(dadew, "msys2") + "\\usr\\bin\\find.exe" @@ -119,7 +131,9 @@ def _dev_env_tool_impl(ctx): build_path = ctx.path("BUILD") build_content = _create_build_content( rule_name = ctx.name, + is_windows = is_windows, tools = ctx.attr.tools, + required_tools = ctx.attr.required_tools, win_paths = [ "%s/%s" % (ctx.attr.prefix, path) for path in ctx.attr.win_paths @@ -137,6 +151,10 @@ dev_env_tool = repository_rule( "tools": attr.string_list( mandatory = True, ), + "required_tools": attr.string_list_dict( + mandatory = False, + default = {}, + ), "win_tool": attr.string( mandatory = True, ), diff --git a/build.sh b/build.sh index ea938642530b..a9f4aa5ae65b 100755 --- a/build.sh +++ b/build.sh @@ -2,10 +2,9 @@ # Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - set -euo pipefail -eval "$($(dirname "$0")/dev-env/bin/dade-assist)" +eval "$("$(dirname "$0")/dev-env/bin/dade-assist")" execution_log_postfix=${1:-} @@ -18,23 +17,59 @@ if [[ "$execution_log_postfix" == "_Darwin" ]]; then tag_filter="-dont-run-on-darwin,-scaladoc,-pdfdocs" fi -# Bazel test only builds targets that are dependencies of a test suite -# so do a full build first. +# Bazel test only builds targets that are dependencies of a test suite so do a full build first. bazel build //... --build_tag_filters "$tag_filter" -bazel test //... --build_tag_filters "$tag_filter" --test_tag_filters "$tag_filter" --experimental_execution_log_file "$ARTIFACT_DIRS/test_execution${execution_log_postfix}.log" + +# Set up a shared PostgreSQL instance. +export POSTGRESQL_ROOT_DIR="${TMPDIR:-/tmp}/daml/postgresql" +export POSTGRESQL_DATA_DIR="${POSTGRESQL_ROOT_DIR}/data" +export POSTGRESQL_LOG_FILE="${POSTGRESQL_ROOT_DIR}/postgresql.log" +export POSTGRESQL_HOST='localhost' +export POSTGRESQL_PORT=54321 +export POSTGRESQL_USERNAME='test' +export POSTGRESQL_PASSWORD='' +function start_postgresql() { + mkdir -p "$POSTGRESQL_DATA_DIR" + bazel run -- @postgresql_dev_env//:initdb --auth=trust --encoding=UNICODE --locale=en_US.UTF-8 --username="$POSTGRESQL_USERNAME" "$POSTGRESQL_DATA_DIR" + envsubst -no-unset -i ci/postgresql.conf -o "$POSTGRESQL_DATA_DIR/postgresql.conf" + bazel run -- @postgresql_dev_env//:pg_ctl -w --pgdata="$POSTGRESQL_DATA_DIR" --log="$POSTGRESQL_LOG_FILE" start || { + if [[ -f "$POSTGRESQL_LOG_FILE" ]]; then + echo >&2 'PostgreSQL logs:' + cat >&2 "$POSTGRESQL_LOG_FILE" + fi + return 1 + } +} +function stop_postgresql() { + if [[ -e "$POSTGRESQL_DATA_DIR" ]]; then + bazel run -- @postgresql_dev_env//:pg_ctl -w --pgdata="$POSTGRESQL_DATA_DIR" --mode=immediate stop || : + rm -rf "$POSTGRESQL_ROOT_DIR" + fi +} +trap stop_postgresql EXIT +stop_postgresql # in case it's running from a previous build +start_postgresql + +# Run the tests. +bazel test //... \ + --build_tag_filters "$tag_filter" \ + --test_tag_filters "$tag_filter" \ + --test_env "POSTGRESQL_HOST=${POSTGRESQL_HOST}" \ + --test_env "POSTGRESQL_PORT=${POSTGRESQL_PORT}" \ + --test_env "POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME}" \ + --test_env "POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD}" \ + --experimental_execution_log_file "$ARTIFACT_DIRS/test_execution${execution_log_postfix}.log" + # Make sure that Bazel query works. -bazel query 'deps(//...)' > /dev/null +bazel query 'deps(//...)' >/dev/null + # Check that we can load damlc in ghci -GHCI_SCRIPT=$(mktemp) -function cleanup { - rm -rf "$GHCI_SCRIPT" -} -trap cleanup EXIT # Disabled on darwin since it sometimes seem to hang and this only # tests our dev setup rather than our code so issues are not critical. if [[ "$(uname)" != "Darwin" ]]; then - da-ghci --data yes //compiler/damlc:damlc -e ':main --help' + da-ghci --data yes //compiler/damlc:damlc -e ':main --help' fi + # Check that our IDE works on our codebase ghcide compiler/damlc/exe/Main.hs 2>&1 | tee ide-log grep -q "1 file worked, 0 files failed" ide-log diff --git a/ci/postgresql.conf b/ci/postgresql.conf new file mode 100644 index 000000000000..ac0c5693a8a2 --- /dev/null +++ b/ci/postgresql.conf @@ -0,0 +1,8 @@ +listen_addresses = '${POSTGRESQL_HOST}' +port = ${POSTGRESQL_PORT} +unix_socket_directories = '${POSTGRESQL_ROOT_DIR}' +fsync = off +synchronous_commit = off +full_page_writes = off +log_min_duration_statement = 0 +log_connections = on diff --git a/daml-assistant/integration-tests/BUILD.bazel b/daml-assistant/integration-tests/BUILD.bazel index b85a00745f76..0d454308c653 100644 --- a/daml-assistant/integration-tests/BUILD.bazel +++ b/daml-assistant/integration-tests/BUILD.bazel @@ -27,8 +27,10 @@ genrule( set -euo pipefail TMP_DIR=$$(mktemp -d) MVN_DB="$$TMP_DIR/m2" + MVN=($(locations @mvn_dev_env//:mvn)) + MVN="$${{MVN[0]}}" install_mvn() {{ - $(location @mvn_dev_env//:mvn) -q install:install-file \ + "$$MVN" -q install:install-file \ -Dmaven.repo.local=$$MVN_DB \ "-DgroupId=$$1" \ "-DartifactId=$$2" \ @@ -62,7 +64,7 @@ genrule( "com.daml" "ledger-api-auth-client" \ $(location //ledger/ledger-api-auth-client:libledger-api-auth-client.jar) \ $(location //ledger/ledger-api-auth-client:ledger-api-auth-client_pom.xml) - $(location @mvn_dev_env//:mvn) -q -Dmaven.repo.local=$$MVN_DB -f "$$TMP_DIR/quickstart-java/pom.xml" dependency:resolve dependency:resolve-plugins + "$$MVN" -q -Dmaven.repo.local=$$MVN_DB -f "$$TMP_DIR/quickstart-java/pom.xml" dependency:resolve dependency:resolve-plugins tar cf $(location integration-tests-mvn.tar) -C $$(dirname $$MVN_DB) $$(basename $$MVN_DB) \ --owner=0 --group=0 --numeric-owner --mtime=2000-01-01\ 00:00Z --sort=name """.format(mvn = mvn_version), diff --git a/dev-env/bin/envsubst b/dev-env/bin/envsubst new file mode 120000 index 000000000000..bd1b1819b200 --- /dev/null +++ b/dev-env/bin/envsubst @@ -0,0 +1 @@ +../lib/dade-exec-nix-bin-tool \ No newline at end of file diff --git a/extractor/src/test/lib/scala/com/digitalasset/extractor/services/ExtractorFixture.scala b/extractor/src/test/lib/scala/com/digitalasset/extractor/services/ExtractorFixture.scala index f4da7961d5dc..3b0d96bd4c0e 100644 --- a/extractor/src/test/lib/scala/com/digitalasset/extractor/services/ExtractorFixture.scala +++ b/extractor/src/test/lib/scala/com/digitalasset/extractor/services/ExtractorFixture.scala @@ -50,8 +50,8 @@ trait ExtractorFixture extends SandboxFixture with PostgresAroundSuite with Type protected def target: PostgreSQLTarget = PostgreSQLTarget( connectUrl = postgresDatabase.url, - user = PostgresAround.userName, - password = PostgresAround.password, + user = postgresDatabase.userName, + password = postgresDatabase.password, outputFormat = outputFormat, schemaPerPackage = false, mergeIdentical = false, diff --git a/ledger/ledger-on-sql/src/test/lib/scala/com/daml/ledger/on/sql/MainWithEphemeralPostgresql.scala b/ledger/ledger-on-sql/src/test/lib/scala/com/daml/ledger/on/sql/MainWithEphemeralPostgresql.scala index 8092baa7e66c..21efa4e403d4 100644 --- a/ledger/ledger-on-sql/src/test/lib/scala/com/daml/ledger/on/sql/MainWithEphemeralPostgresql.scala +++ b/ledger/ledger-on-sql/src/test/lib/scala/com/daml/ledger/on/sql/MainWithEphemeralPostgresql.scala @@ -14,9 +14,9 @@ object MainWithEphemeralPostgresql extends PostgresAround { .parse[Unit]("SQL Ledger", _ => (), (), args) .getOrElse(sys.exit(1)) - startEphemeralPostgres() + connectToPostgresqlServer() val database = createNewRandomDatabase() - sys.addShutdownHook(stopAndCleanUpPostgres()) + sys.addShutdownHook(disconnectFromPostgresqlServer()) val config = originalConfig.copy( participants = originalConfig.participants.map(_.copy(serverJdbcUrl = database.url)), extra = ExtraConfig(jdbcUrl = Some(database.url)), diff --git a/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandbox/persistence/MainWithEphemeralPostgresql.scala b/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandbox/persistence/MainWithEphemeralPostgresql.scala index bef461892f62..24917821c2f1 100644 --- a/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandbox/persistence/MainWithEphemeralPostgresql.scala +++ b/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandbox/persistence/MainWithEphemeralPostgresql.scala @@ -8,9 +8,9 @@ import com.daml.testing.postgresql.PostgresAround object MainWithEphemeralPostgresql extends PostgresAround { def main(args: Array[String]): Unit = { - startEphemeralPostgres() + connectToPostgresqlServer() val database = createNewRandomDatabase() - sys.addShutdownHook(stopAndCleanUpPostgres()) + sys.addShutdownHook(disconnectFromPostgresqlServer()) SandboxMain.main(args ++ Array("--sql-backend-jdbcurl", database.url)) } } diff --git a/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandboxnext/persistence/MainWithEphemeralPostgresql.scala b/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandboxnext/persistence/MainWithEphemeralPostgresql.scala index 55f25b742863..32b84256807e 100644 --- a/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandboxnext/persistence/MainWithEphemeralPostgresql.scala +++ b/ledger/sandbox/src/test/lib/scala/com/digitalasset/platform/sandboxnext/persistence/MainWithEphemeralPostgresql.scala @@ -8,9 +8,9 @@ import com.daml.testing.postgresql.PostgresAround object MainWithEphemeralPostgresql extends PostgresAround { def main(args: Array[String]): Unit = { - startEphemeralPostgres() + connectToPostgresqlServer() val database = createNewRandomDatabase() - sys.addShutdownHook(stopAndCleanUpPostgres()) + sys.addShutdownHook(disconnectFromPostgresqlServer()) Main.main(args ++ Array("--sql-backend-jdbcurl", database.url)) } } diff --git a/ledger/sandbox/src/test/suite/scala/com/digitalasset/platform/sandbox/stores/ledger/sql/SqlLedgerSpec.scala b/ledger/sandbox/src/test/suite/scala/com/digitalasset/platform/sandbox/stores/ledger/sql/SqlLedgerSpec.scala index fdad6118bc27..67f4a9fc353a 100644 --- a/ledger/sandbox/src/test/suite/scala/com/digitalasset/platform/sandbox/stores/ledger/sql/SqlLedgerSpec.scala +++ b/ledger/sandbox/src/test/suite/scala/com/digitalasset/platform/sandbox/stores/ledger/sql/SqlLedgerSpec.scala @@ -10,7 +10,7 @@ import com.daml.api.util.TimeProvider import com.daml.bazeltools.BazelRunfiles.rlocation import com.daml.daml_lf_dev.DamlLf import com.daml.ledger.api.domain.LedgerId -import com.daml.ledger.api.health.{Healthy, Unhealthy} +import com.daml.ledger.api.health.Healthy import com.daml.ledger.api.testing.utils.AkkaBeforeAndAfterAll import com.daml.ledger.participant.state.v1.ParticipantId import com.daml.lf.archive.DarReader @@ -146,32 +146,6 @@ class SqlLedgerSpec ledger.currentHealth() should be(Healthy) } } - - "be unhealthy if the underlying database is inaccessible 3 or more times in a row" in { - for { - ledger <- createSqlLedger() - } yield { - withClue("before shutting down postgres,") { - ledger.currentHealth() should be(Healthy) - } - - stopPostgres() - - eventually { - withClue("after shutting down postgres,") { - ledger.currentHealth() should be(Unhealthy) - } - } - - startPostgres() - - eventually { - withClue("after starting up postgres,") { - ledger.currentHealth() should be(Healthy) - } - } - } - } } private def createSqlLedger(): Future[Ledger] = diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAround.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAround.scala index 401ee7a30586..340091147d10 100644 --- a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAround.scala +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAround.scala @@ -10,54 +10,86 @@ import java.nio.file.{Files, Path} import java.util.UUID import java.util.concurrent.atomic.AtomicBoolean +import com.daml.ports.Port import com.daml.testing.postgresql.PostgresAround._ import org.apache.commons.io.{FileUtils, IOUtils} import org.slf4j.LoggerFactory import scala.collection.JavaConverters.asScalaBufferConverter +import scala.util.Try import scala.util.control.NonFatal trait PostgresAround { - @volatile - private var fixture: PostgresFixture = _ + @volatile private var server: PostgresServer = _ + @volatile private var paths: Option[PostgresServerPaths] = None private val started: AtomicBoolean = new AtomicBoolean(false) - protected def startEphemeralPostgres(): Unit = { + protected def connectToPostgresqlServer(): Unit = { + ( + sys.env.get("POSTGRESQL_HOST"), + sys.env.get("POSTGRESQL_PORT").map(port => Port(port.toInt)), + sys.env.get("POSTGRESQL_USERNAME"), + sys.env.get("POSTGRESQL_PASSWORD"), + ) match { + case (Some(hostName), Some(port), Some(userName), Some(password)) => + connectToSharedServer(hostName, port, userName, password) + case _ => + startEphemeralServer() + } + } + + private def connectToSharedServer( + hostName: String, + port: Port, + userName: String, + password: String, + ): Unit = { + logger.info(s"Connected to PostgreSQL on $hostName:$port.") + server = PostgresServer(hostName, port, userName, password) + } + + private def startEphemeralServer(): Unit = { logger.info("Starting an ephemeral PostgreSQL instance...") - val tempDir = Files.createTempDirectory("postgres_test") - val dataDir = tempDir.resolve("data") - val confFile = dataDir.resolve("postgresql.conf") - val logFile = Files.createFile(tempDir.resolve("postgresql.log")) + val root = Files.createTempDirectory("postgres_test") + val dataDir = root.resolve("data") + val configPath = dataDir.resolve("postgresql.conf") + val logFile = Files.createFile(root.resolve("postgresql.log")) val lockedPort = FreePort.find() + val hostName = InetAddress.getLoopbackAddress.getHostAddress val port = lockedPort.port - fixture = PostgresFixture(port, tempDir, dataDir, confFile, logFile) + val userName = "test" + val password = "" + server = PostgresServer(hostName, port, userName, password) + paths = Some(PostgresServerPaths(root, dataDir, logFile)) try { - initializeDatabase() - createConfigFile() - startPostgres() + initializeDatabase(dataDir, userName) + createConfigFile(configPath) + startPostgresql(dataDir, logFile) lockedPort.unlock() logger.info(s"PostgreSQL has started on port $port.") } catch { case NonFatal(e) => lockedPort.unlock() - stopPostgres() - deleteRecursively(tempDir) - fixture = null + stopPostgresql(dataDir) + deleteRecursively(root) throw e } } - protected def stopAndCleanUpPostgres(): Unit = { - logger.info("Stopping and cleaning up PostgreSQL...") - stopPostgres() - deleteRecursively(fixture.tempDir) - logger.info("PostgreSQL has stopped, and the data directory has been deleted.") - fixture = null + protected def disconnectFromPostgresqlServer(): Unit = { + paths foreach { + case PostgresServerPaths(root, dataDir, _) => + logger.info("Stopping and cleaning up PostgreSQL...") + stopPostgresql(dataDir) + deleteRecursively(root) + logger.info("PostgreSQL has stopped, and the data directory has been deleted.") + } + server = null } - protected def startPostgres(): Unit = { + private def startPostgresql(dataDir: Path, logFile: Path): Unit = { if (!started.compareAndSet(false, true)) { throw new IllegalStateException( "Attempted to start PostgreSQL, but it has already been started.", @@ -69,9 +101,9 @@ trait PostgresAround { Tool.pg_ctl, "-w", "-D", - fixture.dataDir.toString, + dataDir.toString, "-l", - fixture.logFile.toString, + logFile.toString, "start", ) } catch { @@ -82,7 +114,7 @@ trait PostgresAround { } } - protected def stopPostgres(): Unit = { + private def stopPostgresql(dataDir: Path): Unit = { if (started.compareAndSet(true, false)) { logger.info("Stopping PostgreSQL...") run( @@ -90,7 +122,7 @@ trait PostgresAround { Tool.pg_ctl, "-w", "-D", - fixture.dataDir.toString, + dataDir.toString, "-m", "immediate", "stop", @@ -103,12 +135,12 @@ trait PostgresAround { createNewDatabase(UUID.randomUUID().toString) protected def createNewDatabase(name: String): PostgresDatabase = { - val database = PostgresDatabase(hostName, fixture.port, userName, name) + val database = PostgresDatabase(server, name) createDatabase(database) database } - private def initializeDatabase(): Unit = run( + private def initializeDatabase(dataDir: Path, userName: String): Unit = run( "initialize the PostgreSQL database", Tool.initdb, s"--username=$userName", @@ -117,10 +149,10 @@ trait PostgresAround { "UNICODE", "-A", "trust", - fixture.dataDir.toString.replaceAllLiterally("\\", "/"), + dataDir.toString.replaceAllLiterally("\\", "/"), ) - private def createConfigFile(): Unit = { + private def createConfigFile(configPath: Path): Unit = { // taken from here: https://bitbucket.org/eradman/ephemeralpg/src/1b5a3c6be81c69a860b7bd540a16b1249d3e50e2/pg_tmp.sh?at=default&fileviewer=file-view-default#pg_tmp.sh-54 // We set unix_socket_directories to /tmp rather than tempDir // since the latter will refer to a temporary directory set by @@ -129,16 +161,16 @@ trait PostgresAround { // this option is ignored. val configText = s"""|unix_socket_directories = '/tmp' - |shared_buffers = 12MB - |fsync = off - |synchronous_commit = off - |full_page_writes = off - |log_min_duration_statement = 0 - |log_connections = on - |listen_addresses = '$hostName' - |port = ${fixture.port} + |shared_buffers = 12MB + |fsync = off + |synchronous_commit = off + |full_page_writes = off + |log_min_duration_statement = 0 + |log_connections = on + |listen_addresses = '${server.hostName}' + |port = ${server.port} """.stripMargin - Files.write(fixture.confFile, configText.getBytes(StandardCharsets.UTF_8)) + Files.write(configPath, configText.getBytes(StandardCharsets.UTF_8)) () } @@ -176,7 +208,7 @@ trait PostgresAround { IOUtils.copy(process.getInputStream, stdout, StandardCharsets.UTF_8) val stderr = new StringWriter IOUtils.copy(process.getErrorStream, stderr, StandardCharsets.UTF_8) - val logs = Files.readAllLines(fixture.logFile).asScala + val logs = readLogs() throw new ProcessFailedException( description = description, command = command, @@ -189,7 +221,7 @@ trait PostgresAround { case e: ProcessFailedException => throw e case NonFatal(e) => - val logs = Files.readAllLines(fixture.logFile).asScala + val logs = readLogs() throw new ProcessFailedException( description = description, command = command, @@ -199,6 +231,10 @@ trait PostgresAround { } } + private def readLogs(): Seq[String] = + Try(paths.map(paths => Files.readAllLines(paths.logFile).asScala).getOrElse(Seq.empty)) + .getOrElse(Seq.empty) + private def deleteRecursively(tempDir: Path): Unit = FileUtils.deleteDirectory(tempDir.toFile) } @@ -206,11 +242,6 @@ trait PostgresAround { object PostgresAround { private val logger = LoggerFactory.getLogger(getClass) - private val hostName = InetAddress.getLoopbackAddress.getHostName - - val userName = "test" - val password = "" - private class ProcessFailedException( description: String, command: Seq[String], diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundAll.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundAll.scala index c46ac768dca2..c9a5f5ce3982 100644 --- a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundAll.scala +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundAll.scala @@ -11,13 +11,13 @@ trait PostgresAroundAll extends PostgresAroundSuite with BeforeAndAfterAll { override protected def beforeAll(): Unit = { // We start PostgreSQL before calling `super` because _generally_ the database needs to be up // before everything else. - startEphemeralPostgres() + connectToPostgresqlServer() createNewDatabase() super.beforeAll() } override protected def afterAll(): Unit = { super.afterAll() - stopAndCleanUpPostgres() + disconnectFromPostgresqlServer() } } diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundEach.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundEach.scala index 8293435a97a6..f3a599948b13 100644 --- a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundEach.scala +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresAroundEach.scala @@ -14,13 +14,13 @@ trait PostgresAroundEach override protected def beforeAll(): Unit = { // We start PostgreSQL before calling `super` because _generally_ the database needs to be up // before everything else. - startEphemeralPostgres() + connectToPostgresqlServer() super.beforeAll() } override protected def afterAll(): Unit = { super.afterAll() - stopAndCleanUpPostgres() + disconnectFromPostgresqlServer() } override protected def beforeEach(): Unit = { diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresDatabase.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresDatabase.scala index 43e1e4fd5bb1..8bef95e61d61 100644 --- a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresDatabase.scala +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresDatabase.scala @@ -5,13 +5,20 @@ package com.daml.testing.postgresql import com.daml.ports.Port -final case class PostgresDatabase( - hostName: String, - port: Port, - userName: String, +final case class PostgresDatabase private[postgresql] ( + private val server: PostgresServer, databaseName: String, ) { - def url: String = s"jdbc:postgresql://$hostName:$port/$databaseName?user=$userName" + def hostName: String = server.hostName + + def port: Port = server.port + + def userName: String = server.userName + + def password: String = server.password + + def url: String = + s"jdbc:postgresql://$hostName:$port/$databaseName?user=$userName&password=$password" override def toString: String = url } diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresResource.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresResource.scala index c4c96cf1d57c..05c290584094 100644 --- a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresResource.scala +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresResource.scala @@ -14,8 +14,8 @@ object PostgresResource { implicit executionContext: ExecutionContext ): Resource[PostgresDatabase] = Resource(Future { - startEphemeralPostgres() + connectToPostgresqlServer() createNewRandomDatabase() - })(_ => Future(stopAndCleanUpPostgres())) + })(_ => Future(disconnectFromPostgresqlServer())) } } diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServer.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServer.scala new file mode 100644 index 000000000000..ae6b5cbf313f --- /dev/null +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServer.scala @@ -0,0 +1,13 @@ +// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.testing.postgresql + +import com.daml.ports.Port + +final case class PostgresServer( + hostName: String, + port: Port, + userName: String, + password: String, +) diff --git a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresFixture.scala b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServerPaths.scala similarity index 67% rename from libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresFixture.scala rename to libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServerPaths.scala index 85e00df8e5c6..c190652302c4 100644 --- a/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresFixture.scala +++ b/libs-scala/postgresql-testing/src/main/scala/com/digitalasset/testing/postgresql/PostgresServerPaths.scala @@ -5,12 +5,8 @@ package com.daml.testing.postgresql import java.nio.file.Path -import com.daml.ports.Port - -final case class PostgresFixture( - port: Port, - tempDir: Path, +case class PostgresServerPaths( + root: Path, dataDir: Path, - confFile: Path, logFile: Path, ) diff --git a/nix/default.nix b/nix/default.nix index 278d66a3b343..5397b48c05dd 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -204,6 +204,7 @@ in rec { xmlstarlet = pkgs.xmlstarlet; grep = pkgs.gnugrep; bc = pkgs.bc; + envsubst = pkgs.envsubst; # Cryptography tooling gnupg = pkgs.gnupg;