Generate GraalVM native image config (#5005)

Motivation: Armeria and its dependencies (e.g. Netty, JCTools and Caffeine) use fair amount of reflections, JNI and other advanced features of JVM, which require explicit configuration when building a native image with GraalVM. We should provide native image configuration in our artifacts so that a user can easily build a native image out of the box. Modifications: - Added a new project flag `native`. For a project with `native` flag, we add a task `nativeImageTrace` that runs select test cases annotated with a `GenerateNativeImageTrace` to collect the trace files generated by `native-image-agent`. - See https://www.graalvm.org/22.0/reference-manual/native-image/Agent/ - Added a new project `:native-image-config`, which: - aggregates all traces into native image config; and - cleans up the noise from the generated native image config - Added the initial version of native image config to `META-INF/native-image/com.linecorp.armeria/armeria`. - Renamed and relocated some test classes so that they are not included in the native image config Result: - A user can now easily build a native image of an Armeria application. - Future works: - Configure our build pipieline to generate native image config for different platforms automatically (macOS, Linux epoll, Linux io_uring and Windows). - Use annotation processors rather than reflection for scanning annotations. - Add entries for Windows and Linux io_uring. --------- Co-authored-by: Hannam Rhee <jrhee17@linecorp.com>
line · Aug 18, 2023 · d34cee7 · d34cee7
1 parent 3b3b504
commit d34cee7
Show file tree

Hide file tree

Showing 93 changed files with 10,742 additions and 278 deletions.
diff --git a/brave/src/test/java/com/linecorp/armeria/it/brave/BraveIntegrationTest.java b/brave/src/test/java/com/linecorp/armeria/it/brave/BraveIntegrationTest.java
@@ -65,6 +65,7 @@
 import com.linecorp.armeria.common.thrift.ThriftFuture;
 import com.linecorp.armeria.common.util.ThreadFactories;
 import com.linecorp.armeria.internal.testing.BlockingUtils;
+import com.linecorp.armeria.internal.testing.GenerateNativeImageTrace;
 import com.linecorp.armeria.server.AbstractHttpService;
 import com.linecorp.armeria.server.HttpService;
 import com.linecorp.armeria.server.ServerBuilder;
@@ -86,6 +87,7 @@
 import testing.brave.TestService;
 import testing.brave.TestService.AsyncIface;
 
+@GenerateNativeImageTrace
 class BraveIntegrationTest {
 
     private static final SpanHandlerImpl spanHandler = new SpanHandlerImpl();

diff --git a/...va/com/linecorp/armeria/server/throttling/bucket4j/TokenBucketThrottlingStrategyTest.java b/...va/com/linecorp/armeria/server/throttling/bucket4j/TokenBucketThrottlingStrategyTest.java
@@ -30,13 +30,15 @@
 import com.linecorp.armeria.common.HttpResponse;
 import com.linecorp.armeria.common.HttpStatus;
 import com.linecorp.armeria.common.throttling.ThrottlingHeaders;
+import com.linecorp.armeria.internal.testing.GenerateNativeImageTrace;
 import com.linecorp.armeria.server.AbstractHttpService;
 import com.linecorp.armeria.server.HttpService;
 import com.linecorp.armeria.server.ServerBuilder;
 import com.linecorp.armeria.server.ServiceRequestContext;
 import com.linecorp.armeria.server.throttling.ThrottlingService;
 import com.linecorp.armeria.testing.junit4.server.ServerRule;
 
+@GenerateNativeImageTrace
 public class TokenBucketThrottlingStrategyTest {
 
     static final HttpService SERVICE = new AbstractHttpService() {

diff --git a/build.gradle b/build.gradle
@@ -2,7 +2,10 @@ import org.jsoup.Jsoup
 import org.jsoup.nodes.Element
 import org.jsoup.select.Elements
 
+import java.nio.file.Files
+import java.nio.file.Paths
 import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicInteger
 
 import static java.lang.Math.min
 
@@ -13,6 +16,7 @@ buildscript {
 }
 
 plugins {
+    id 'jvm-toolchains'
     alias libs.plugins.osdetector apply false
     alias libs.plugins.scalafmt apply false
     alias libs.plugins.nexus.publish
@@ -51,7 +55,7 @@ allprojects {
     tasks.withType(JavaForkOptions) {
         maxHeapSize = '768m'
 
-        if (project.ext.testJavaVersion >= 9) {
+        if (project.ext.testJavaVersion >= 9 || it.name == 'nativeImageTrace') {
             jvmArgs '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED'
         }
 
@@ -76,7 +80,7 @@ allprojects {
         // `disallow` in Java 18.
         // - https://openjdk.org/jeps/411
         // - https://bugs.openjdk.org/browse/JDK-8270380
-        if (project.ext.testJavaVersion >= 17) {
+        if (project.ext.testJavaVersion >= 17 || it.name == 'nativeImageTrace') {
             systemProperty "java.security.manager", "allow"
         }
 
@@ -89,6 +93,11 @@ allprojects {
         systemProperty 'com.linecorp.armeria.verboseExceptions', 'true'
         systemProperty 'com.linecorp.armeria.verboseResponses', 'true'
 
+        // Allow choosing the transport type by specifying `-PtransportType=<nio|epoll|io_uring|kqueue>`.
+        if (rootProject.hasProperty('transportType')) {
+            systemProperty('com.linecorp.armeria.transportType', "${rootProject.findProperty("transportType")}")
+        }
+
         // Pass special system property to tell our tests that we are measuring coverage.
         if (project.hasFlags('coverage')) {
             systemProperty 'com.linecorp.armeria.testing.coverage', 'true'
@@ -272,7 +281,7 @@ tasks.closeAndReleaseStagingRepository.doFirst {
     }
 }
 
-task reportFailedTests(type: TestsReportTask)
+tasks.register("reportFailedTests", TestsReportTask)
 
 /**
  * Summarizes the failed tests and reports as a file with the Markdown syntax.
@@ -350,3 +359,74 @@ class TestsReportTask extends DefaultTask {
         }
     }
 }
+
+def graalLauncher = javaToolchains.launcherFor {
+    languageVersion.set(JavaLanguageVersion.of(17))
+    vendor.set(JvmVendorSpec.GRAAL_VM)
+}.get()
+rootProject.ext.graalLauncher = graalLauncher
+
+def graalHome = graalLauncher.executablePath.asFile.toPath().parent.parent
+rootProject.ext.graalHome = graalHome
+
+tasks.register("installGraalNativeImage", Exec) {
+    onlyIf {
+        !Files.exists(graalHome.resolve("lib/svm/bin/native-image")) ||
+        !Files.exists(graalHome.resolve("lib/svm/bin/native-image-configure")) ||
+        !Files.exists(graalHome.resolve("lib/svm/bin/rebuild-images"))
+    }
+    commandLine(graalHome.resolve("lib/installer/bin/gu"), "install", "native-image")
+}
+
+def relocatedProjects = projectsWithFlags('java', 'relocate')
+def projectsWithNativeImageSupport = projectsWithFlags('java', 'relocate', 'native')
+def numConfiguredRelocatedProjects = new AtomicInteger()
+configure(relocatedProjects) {
+    if (projectsWithNativeImageSupport.contains(project)) {
+        def shadedTestTask = tasks.getByName('shadedTest') as Test
+        def copyShadedTestClassesTask = tasks.getByName('copyShadedTestClasses') as Copy
+        def nativeImageTraceFile =
+                "$rootDir/native-image-config/gen-src/traces/${project.path.replace(':', '-').substring(1)}.json"
+        project.ext.nativeImageTraceFile = Paths.get(nativeImageTraceFile)
+
+        tasks.register("nativeImageTrace", Test).configure {
+            group = "Build"
+            description = "Generates native-image trace for ${project.name} by running shaded tests."
+
+            dependsOn(rootProject.tasks.named('installGraalNativeImage'))
+            dependsOn(copyShadedTestClassesTask)
+            outputs.file(nativeImageTraceFile)
+
+            useJUnitPlatform {
+                includeTags('NATIVE_IMAGE_TRACE')
+            }
+
+            javaLauncher.set(graalLauncher)
+            maxParallelForks = 1
+
+            // Remove 'jetty-alpn-agent' and add 'native-image-agent'.
+            jvmArgs = jvmArgs.stream().filter {
+                !(it.startsWith('-javaagent:') && it.endsWith('jetty-alpn-agent.jar'))
+            }.collect() + [
+                    "-agentlib:native-image-agent=trace-output=$nativeImageTraceFile"
+            ]
+
+            testClassesDirs = files(copyShadedTestClassesTask.destinationDir)
+            classpath = testClassesDirs
+            setExcludes(shadedTestTask.excludes)
+            exclude("META-INF/versions/**") // Seems unsupported by GraalVM
+        }
+    }
+
+    afterEvaluate {
+        if (numConfiguredRelocatedProjects.incrementAndGet() == relocatedProjects.size()) {
+            relocatedProjects.each { p ->
+                if (projectsWithNativeImageSupport.contains(p)) {
+                    p.tasks.getByName('nativeImageTrace').configure {
+                        classpath += p.files(p.configurations.getByName('shadedJarTestRuntime').resolve())
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/consul/src/test/java/com/linecorp/armeria/client/consul/ConsulEndpointGroupTest.java b/consul/src/test/java/com/linecorp/armeria/client/consul/ConsulEndpointGroupTest.java
@@ -30,10 +30,12 @@
 
 import com.linecorp.armeria.client.Endpoint;
 import com.linecorp.armeria.internal.consul.ConsulTestBase;
+import com.linecorp.armeria.internal.testing.GenerateNativeImageTrace;
 import com.linecorp.armeria.server.Server;
 import com.linecorp.armeria.server.ServerListener;
 import com.linecorp.armeria.server.consul.ConsulUpdatingListener;
 
+@GenerateNativeImageTrace
 class ConsulEndpointGroupTest extends ConsulTestBase {
 
     private static final List<Server> servers = new ArrayList<>();

diff --git a/consul/src/test/java/com/linecorp/armeria/server/consul/ConsulUpdatingListenerTest.java b/consul/src/test/java/com/linecorp/armeria/server/consul/ConsulUpdatingListenerTest.java
@@ -35,9 +35,11 @@
 import com.linecorp.armeria.common.HttpMethod;
 import com.linecorp.armeria.common.SessionProtocol;
 import com.linecorp.armeria.internal.consul.ConsulTestBase;
+import com.linecorp.armeria.internal.testing.GenerateNativeImageTrace;
 import com.linecorp.armeria.server.Server;
 import com.linecorp.armeria.server.ServerListener;
 
+@GenerateNativeImageTrace
 class ConsulUpdatingListenerTest extends ConsulTestBase {
 
     private static final List<Server> servers = new ArrayList<>();

diff --git a/...TA-INF/native-image/com.linecorp.armeria/armeria/agent-extracted-predefined-classes/.keep b/...TA-INF/native-image/com.linecorp.armeria/armeria/agent-extracted-predefined-classes/.keep