Skip to content

Commit

Permalink
improve build times (#450)
Browse files Browse the repository at this point in the history
- Split root project into `kyoJVM` and `kyoJS`. The JVM project is
selected by default, this way the regular development cycle is faster
but there's the downside that we might not catch issues in JS before CI.
It seems a reasonable tradeoff since it's possible to run the tests
manually by selecting the `kyoJS` project.
- Avoid the dependency on `kyo-core` tests in other modules. All other
modules end up stuck waiting for the core tests to compile because of
the dependency. I had to duplicate some of the code in `KyoTest` into
the other modules to achieve that.
- Move `scalafmt` from the build to a commit hook.
- Remove `-Wunused:all` compiler option. It significantly increases the
compilation times for tests. We can attempt to add it back once we're on
Scala 3.5, which seems to [improve build
times](#342 (comment))
with the option enabled.

Results with a cold JVM:

**Main**
```
$ sbt ";clean;compile"
[success] Total time: 24 s, completed May 30, 2024, 1:58:13 PM
$ sbt ";clean;test:compile"
[success] Total time: 58 s, completed May 30, 2024, 1:59:58 PM
$ sbt ";clean;test"
[success] Total time: 88 s (01:28), completed May 30, 2024, 2:01:45 PM
```

**This PR (default `kyoJVM` only)**
```
$ sbt ";clean;compile"
[success] Total time: 14 s, completed May 30, 2024, 1:54:29 PM
$ sbt ";clean;test:compile"
[success] Total time: 27 s, completed May 30, 2024, 1:55:51 PM
$ sbt ";clean;test"
[success] Total time: 39 s, completed May 30, 2024, 1:56:59 PM
```

I won't close #342 since there's
always opportunity to optimize the build.
  • Loading branch information
fwbrasil authored May 31, 2024
1 parent 42078ad commit d41f50e
Show file tree
Hide file tree
Showing 20 changed files with 265 additions and 114 deletions.
8 changes: 8 additions & 0 deletions .githooks/pre-commit
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/sh
echo "Running pre-commit hook to format Scala source files"
sbt scalafmtAll
if ! git diff --exit-code --quiet
then
echo "Formatting changes detected. Please stage the changes and commit again."
exit 1
fi
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ jobs:

- name: Build JVM
run: |
sbt 'kyo-bench/test' 'kyo-cache/test' '+kyo-scheduler/test' '+kyo-scheduler-zio/test' 'kyo-core/test' 'kyo-direct/test' 'kyo-test/test' 'kyo-zio/test' 'kyo-examples/test' 'kyo-os-lib/test' 'kyo-stats-otel/test' 'kyo-sttp/test' 'kyo-tapir/test'
sbt '+kyoJVM/test'
- name: Build JS
run: |
sbt '+kyo-schedulerJS/test' 'kyo-coreJS/test' 'kyo-directJS/test' 'kyo-sttpJS/test' 'kyo-testJS/test' 'kyo-zioJS/test'
sbt '+kyoJS/test'
116 changes: 63 additions & 53 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ val compilerOptions = Seq(
"-unchecked",
"-deprecation",
"-Wvalue-discard",
"-Wunused:all",
"-language:strictEquality"
)

Expand All @@ -27,7 +26,7 @@ lazy val `kyo-settings` = Seq(
scalaVersion := scala3Version,
crossScalaVersions := List(scala3Version),
scalacOptions ++= compilerOptions,
scalafmtOnCompile := true,
scalafmtOnCompile := false,
organization := "io.getkyo",
homepage := Some(url("https://getkyo.io")),
licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0")),
Expand All @@ -45,39 +44,50 @@ lazy val `kyo-settings` = Seq(
Test / testOptions += Tests.Argument("-oDG"),
ThisBuild / versionScheme := Some("early-semver"),
scalacOptions ++= Seq("-release:11"),
libraryDependencies += "org.scalatest" %%% "scalatest" % scalaTestVersion % Test,
Test / javaOptions += "--add-opens=java.base/java.lang=ALL-UNNAMED"
)

lazy val kyo =
crossProject(JVMPlatform)
.in(file("."))
.settings(
name := "kyo",
organization := "io.getkyo",
publishArtifact := false,
publish / skip := true,
Compile / packageBin / publishArtifact := false,
Compile / packageDoc / publishArtifact := false,
Compile / packageSrc / publishArtifact := false,
scalaVersion := scala3Version,
`kyo-settings`
).aggregate(
`kyo-scheduler`,
`kyo-scheduler-zio`,
`kyo-tag`,
`kyo-core`,
`kyo-direct`,
`kyo-stats-registry`,
`kyo-stats-otel`,
`kyo-cache`,
`kyo-sttp`,
`kyo-tapir`,
`kyo-caliban`,
`kyo-bench`,
`kyo-test`,
`kyo-zio`,
`kyo-examples`
)
lazy val kyoJVM = project
.in(file("."))
.settings(
name := "kyoJVM",
`kyo-settings`
)
.aggregate(
`kyo-scheduler`.jvm,
`kyo-scheduler-zio`.jvm,
`kyo-tag`.jvm,
`kyo-core`.jvm,
`kyo-direct`.jvm,
`kyo-stats-registry`.jvm,
`kyo-stats-otel`.jvm,
`kyo-cache`.jvm,
`kyo-sttp`.jvm,
`kyo-tapir`.jvm,
`kyo-caliban`.jvm,
`kyo-bench`.jvm,
`kyo-test`.jvm,
`kyo-zio`.jvm,
`kyo-examples`.jvm
)

lazy val kyoJS = project
.in(file("js"))
.settings(
name := "kyoJS",
`kyo-settings`
)
.aggregate(
`kyo-scheduler`.js,
`kyo-tag`.js,
`kyo-core`.js,
`kyo-direct`.js,
`kyo-stats-registry`.js,
`kyo-sttp`.js,
`kyo-test`.js,
`kyo-zio`.js
)

lazy val `kyo-scheduler` =
crossProject(JSPlatform, JVMPlatform)
Expand Down Expand Up @@ -142,14 +152,13 @@ lazy val `kyo-core` =
.in(file("kyo-core"))
.settings(
`kyo-settings`,
libraryDependencies += "com.lihaoyi" %%% "pprint" % "0.9.0",
libraryDependencies += "org.jctools" % "jctools-core" % "4.0.3",
libraryDependencies += "org.slf4j" % "slf4j-api" % "2.0.13",
libraryDependencies += "dev.zio" %%% "zio-laws-laws" % "1.0.0-RC26" % Test,
libraryDependencies += "dev.zio" %%% "zio-test-sbt" % "2.1.1" % Test,
libraryDependencies += "org.scalatest" %%% "scalatest" % scalaTestVersion % Test,
libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.5.6" % Test,
libraryDependencies += "javassist" % "javassist" % "3.12.1.GA" % Test
libraryDependencies += "com.lihaoyi" %%% "pprint" % "0.9.0",
libraryDependencies += "org.jctools" % "jctools-core" % "4.0.3",
libraryDependencies += "org.slf4j" % "slf4j-api" % "2.0.13",
libraryDependencies += "dev.zio" %%% "zio-laws-laws" % "1.0.0-RC26" % Test,
libraryDependencies += "dev.zio" %%% "zio-test-sbt" % "2.1.1" % Test,
libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.5.6" % Test,
libraryDependencies += "javassist" % "javassist" % "3.12.1.GA" % Test
)
.jsSettings(`js-settings`)

Expand All @@ -158,7 +167,7 @@ lazy val `kyo-direct` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-direct"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
libraryDependencies += "com.github.rssh" %%% "dotty-cps-async" % "0.9.21"
Expand Down Expand Up @@ -189,7 +198,7 @@ lazy val `kyo-stats-otel` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-stats-otel"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
libraryDependencies += "io.opentelemetry" % "opentelemetry-api" % "1.38.0",
Expand All @@ -202,7 +211,7 @@ lazy val `kyo-cache` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-cache"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
libraryDependencies += "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8"
Expand All @@ -213,7 +222,7 @@ lazy val `kyo-os-lib` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-os-lib"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
libraryDependencies += "com.lihaoyi" %% "os-lib" % "0.10.1"
Expand All @@ -224,7 +233,7 @@ lazy val `kyo-sttp` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-sttp"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
libraryDependencies += "com.softwaremill.sttp.client3" %%% "core" % "3.9.7"
Expand All @@ -236,7 +245,7 @@ lazy val `kyo-tapir` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-tapir"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.dependsOn(`kyo-sttp`)
.settings(
`kyo-settings`,
Expand All @@ -249,10 +258,10 @@ lazy val `kyo-caliban` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Pure)
.in(file("kyo-caliban"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.dependsOn(`kyo-tapir`)
.dependsOn(`kyo-zio`)
.dependsOn(`kyo-sttp` % "test->test")
.dependsOn(`kyo-sttp`)
.settings(
`kyo-settings`,
libraryDependencies += "com.github.ghostdogpr" %% "caliban" % "2.7.0",
Expand Down Expand Up @@ -280,12 +289,12 @@ lazy val `kyo-zio` =
.withoutSuffixFor(JVMPlatform)
.crossType(CrossType.Full)
.in(file("kyo-zio"))
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
libraryDependencies += "dev.zio" %% "zio" % zioVersion,
libraryDependencies += "dev.zio" %% "zio-test" % zioVersion,
libraryDependencies += "dev.zio" %% "zio-test-sbt" % zioVersion % Test
libraryDependencies += "dev.zio" %%% "zio" % zioVersion,
libraryDependencies += "dev.zio" %%% "zio-test" % zioVersion,
libraryDependencies += "dev.zio" %%% "zio-test-sbt" % zioVersion % Test
).jsSettings(
`js-settings`
)
Expand All @@ -298,7 +307,7 @@ lazy val `kyo-examples` =
.dependsOn(`kyo-tapir`)
.dependsOn(`kyo-direct`)
.dependsOn(`kyo-os-lib`)
.dependsOn(`kyo-core` % "test->test;compile->compile")
.dependsOn(`kyo-core`)
.settings(
`kyo-settings`,
Compile / doc / sources := Seq.empty,
Expand All @@ -316,6 +325,7 @@ lazy val `kyo-bench` =
.dependsOn(`kyo-scheduler-zio`)
.settings(
`kyo-settings`,
Test / testForkedParallel := true,
// Forks each test suite individually
Test / testGrouping := {
val javaOptionsValue = javaOptions.value.toVector
Expand Down
16 changes: 16 additions & 0 deletions kyo-cache/shared/src/test/scala/kyo/KyoTest.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package kyoTest

import kyo.*
import kyo.internal.BaseKyoTest
import org.scalatest.NonImplicitAssertions
import org.scalatest.freespec.AsyncFreeSpec
import scala.concurrent.ExecutionContext
import scala.language.implicitConversions

abstract class KyoTest extends AsyncFreeSpec with BaseKyoTest with NonImplicitAssertions:

type Assertion = org.scalatest.compatible.Assertion
def success = succeed

override given executionContext: ExecutionContext = kyo.internal.Platform.executionContext
end KyoTest
16 changes: 16 additions & 0 deletions kyo-caliban/src/test/scala/kyo/KyoTest.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package kyoTest

import kyo.*
import kyo.internal.BaseKyoTest
import org.scalatest.NonImplicitAssertions
import org.scalatest.freespec.AsyncFreeSpec
import scala.concurrent.ExecutionContext
import scala.language.implicitConversions

abstract class KyoTest extends AsyncFreeSpec with BaseKyoTest with NonImplicitAssertions:

type Assertion = org.scalatest.compatible.Assertion
def success = succeed

override given executionContext: ExecutionContext = kyo.internal.Platform.executionContext
end KyoTest
10 changes: 5 additions & 5 deletions kyo-caliban/src/test/scala/kyo/resolversTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ class resolversTest extends KyoTest:
zio.Runtime.default.unsafe.run(v).getOrThrow()
)

def testServer() =
def testServer(port: Int) =
import scala.concurrent.duration.*
NettyKyoServer(NettyConfig.default.copy(gracefulShutdownTimeout = Some(5.millis)))
NettyKyoServer(NettyConfig.default.copy(port = port, gracefulShutdownTimeout = Some(5.millis)))

case class Query(
k1: Int < Aborts[Throwable],
Expand Down Expand Up @@ -81,7 +81,7 @@ class resolversTest extends KyoTest:

ZIOs.run {
for
server <- Resolvers.run(testServer()) {
server <- Resolvers.run(testServer(8080)) {
Resolvers.get(api)
}
res <- Requests.run {
Expand All @@ -99,7 +99,7 @@ class resolversTest extends KyoTest:

ZIOs.run {
for
server <- Resolvers.run(testServer()) {
server <- Resolvers.run(testServer(8081)) {
Resolvers.get(api).map(_.configure(Configurator.setEnableIntrospection(true)))
}
res <- Requests.run {
Expand Down Expand Up @@ -130,7 +130,7 @@ class resolversTest extends KyoTest:

ZIOs.run {
for
server <- Resolvers.run(testServer(), runner) { Resolvers.get(api) }
server <- Resolvers.run(testServer(8082), runner) { Resolvers.get(api) }
res <- Requests.run {
Requests[String](_
.post(Uri.unsafeApply(server.hostName, server.port))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package kyoTest
package kyo.internal

object Platform:
def executionContext = org.scalajs.macrotaskexecutor.MacrotaskExecutor
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package kyoTest
package kyo.internal

object Platform:
def executionContext = scala.concurrent.ExecutionContext.global
Expand Down
56 changes: 56 additions & 0 deletions kyo-core/shared/src/main/scala/kyo/internal/BaseKyoTest.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package kyo.internal

import kyo.*
import scala.concurrent.Future
import scala.language.implicitConversions
import scala.util.Try

trait BaseKyoTest:

type Assertion

def success: Assertion

given tryCanEqual[T]: CanEqual[Try[T], Try[T]] = CanEqual.derived
given eitherCanEqual[T, U]: CanEqual[Either[T, U], Either[T, U]] = CanEqual.derived
given throwableCanEqual: CanEqual[Throwable, Throwable] = CanEqual.derived

def retry[S](f: => Boolean < S): Boolean < S =
def loop(): Boolean < S =
f.map {
case true => true
case false => loop()
}
loop()
end retry

def timeout =
if kyo.internal.Platform.isDebugEnabled then
Duration.Infinity
else
5.seconds

// given Conversion[Assertion, Future[Assertion]] = Future.successful(_)

def runJVM(
v: => Assertion < KyoApp.Effects
)(using Flat[Assertion]): Future[Assertion] =
if kyo.internal.Platform.isJVM then
run(v)
else
Future.successful(success)

def runJS(
v: => Assertion < KyoApp.Effects
)(using Flat[Assertion]): Future[Assertion] =
if kyo.internal.Platform.isJS then
run(v)
else
Future.successful(success)

def run(
v: => Assertion < KyoApp.Effects
)(using Flat[Assertion]): Future[Assertion] =
IOs.run(KyoApp.runFiber(timeout)(v).toFuture).map(_.get)(using Platform.executionContext)

end BaseKyoTest
Loading

0 comments on commit d41f50e

Please sign in to comment.