From f15e4246ee412614da46c861f9c39f6d41cfe0aa Mon Sep 17 00:00:00 2001 From: Jorge Aguilera <98739503+jorgeaguileraseqera@users.noreply.github.com> Date: Tue, 30 Jul 2024 13:13:57 +0200 Subject: [PATCH] Remove capsule launcher dependencies (#3395) This commit replaces the Capsule launcher with "fat" jar approach that contains all core nextflow classes and dependencies. Main changes: * Remove the dependency on deprecated Capsule loader * Do not rely on Maven for Nextflow installation process * The `nextflow-*-one.jar` file includes all Nextflow core modules (not plugins) and dependencies * Use the Gradle ShadowJar plugin to create the `one` jar file * The `nextflow-*-all` is replaced by the `nextflow-*-dist` package that's the same as the `one` package with self-executing ability. * Enable the default Nextflow plugin install/update capability for `dist` package. Signed-off-by: Jorge Aguilera Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso Co-authored-by: Ben Sherman --- Makefile | 5 +- VERSION | 2 +- docs/config.md | 3 - modules/nextflow/build.gradle | 12 ++ .../main/nextflow/plugin/PluginsFacade.groovy | 20 ++-- nextflow | 40 +++++-- packing.gradle | 108 +++--------------- 7 files changed, 66 insertions(+), 124 deletions(-) diff --git a/Makefile b/Makefile index fec716036d..4a12211e00 100644 --- a/Makefile +++ b/Makefile @@ -95,10 +95,7 @@ upload: # Create self-contained distribution package # pack: - BUILD_PACK=1 ./gradlew packAll - -packCore: - BUILD_PACK=1 ./gradlew packCore + BUILD_PACK=1 ./gradlew pack # # Upload NF launcher to nextflow.io web site diff --git a/VERSION b/VERSION index 4e8327ae10..d4c0237907 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.06.0-edge +24.07.0-edge diff --git a/docs/config.md b/docs/config.md index ba97797cdb..e63aedd1dd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1894,9 +1894,6 @@ The following environment variables control the configuration of the Nextflow ru `NXF_CHARLIECLOUD_CACHEDIR` : Directory where remote Charliecloud images are stored. When using a computing cluster it must be a shared folder accessible from all compute nodes. -`NXF_CLASSPATH` -: Allows the extension of the Java runtime classpath with extra JAR files or class folders. - `NXF_CLOUDCACHE_PATH` : :::{versionadded} 23.07.0-edge ::: diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 929bd6a161..ff7f48dc83 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -1,3 +1,6 @@ +plugins { + id "com.github.johnrengelman.shadow" version "8.1.1" +} apply plugin: 'groovy' apply plugin: 'application' @@ -72,3 +75,12 @@ application { run{ args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') ) } + +shadowJar { + archiveClassifier='one' + manifest { + attributes 'Main-Class': "$mainClassName" + } + mergeServiceFiles() + mergeGroovyExtensionModules() +} diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy index 01df157273..4bf9640577 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy @@ -318,8 +318,8 @@ class PluginsFacade implements PluginStateListener { } void start( String pluginId ) { - if( isSelfContained() && defaultPlugins.hasPlugin(pluginId) ) { - log.debug "Plugin 'start' is not required in self-contained mode -- ignoring for plugin: $pluginId" + if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) ) { + log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugin: $pluginId" return } @@ -327,8 +327,8 @@ class PluginsFacade implements PluginStateListener { } void start(PluginSpec plugin) { - if( isSelfContained() && defaultPlugins.hasPlugin(plugin.id) ) { - log.debug "Plugin 'start' is not required in self-contained mode -- ignoring for plugin: $plugin.id" + if( isEmbedded() && defaultPlugins.hasPlugin(plugin.id) ) { + log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugin: $plugin.id" return } @@ -346,19 +346,19 @@ class PluginsFacade implements PluginStateListener { } /** - * @return {@code true} when running in self-contained mode ie. the nextflow distribution + * @return {@code true} when running in embedded mode ie. the nextflow distribution * include also plugin libraries. When running is this mode, plugins should not be started * and cannot be updated. */ - protected boolean isSelfContained() { - return env.get('NXF_PACK')=='all' || embedded + protected boolean isEmbedded() { + return embedded } protected List pluginsRequirement(Map config) { def specs = parseConf(config) - if( isSelfContained() && specs ) { + if( isEmbedded() && specs ) { // custom plugins are not allowed for nextflow self-contained package - log.warn "Nextflow self-contained distribution allows only core plugins -- User config plugins will be ignored: ${specs.join(',')}" + log.warn "Nextflow embedded mode only core plugins -- User config plugins will be ignored: ${specs.join(',')}" return Collections.emptyList() } if( specs ) { @@ -441,7 +441,7 @@ class PluginsFacade implements PluginStateListener { boolean startIfMissing(String pluginId) { if( env.NXF_PLUGINS_DEFAULT == 'false' ) return false - if( isSelfContained() && defaultPlugins.hasPlugin(pluginId) ) + if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) ) return false if( isStarted(pluginId) ) diff --git a/nextflow b/nextflow index 2e6a9e0718..9b7395b7a9 100755 --- a/nextflow +++ b/nextflow @@ -15,7 +15,7 @@ # limitations under the License. [[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'24.06.0-edge'} +NXF_VER=${NXF_VER:-'24.07.0-edge'} NXF_ORG=${NXF_ORG:-'nextflow-io'} NXF_HOME=${NXF_HOME:-$HOME/.nextflow} NXF_PROT=${NXF_PROT:-'https'} @@ -26,6 +26,13 @@ NXF_CLI="$0 $@" NXF_CLI_OPTS=${NXF_CLI_OPTS:-} NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005} +NXF_LEGACY_LAUNCHER=1 +NXF_VER_MAJOR=$(echo $NXF_VER| cut -d'.' -f1) +NXF_VER_MINOR=$(echo $NXF_VER| cut -d'.' -f2) +if [[ NXF_VER_MAJOR -ge 24 ]] && [[ NXF_VER_MINOR -ge 7 ]]; then + unset NXF_LEGACY_LAUNCHER +fi + export NXF_CLI export NXF_ORG export NXF_HOME @@ -249,18 +256,19 @@ NXF_JAR=${NXF_JAR:-nextflow-$NXF_VER-$NXF_PACK.jar} NXF_BIN=${NXF_BIN:-$NXF_DIST/$NXF_VER/$NXF_JAR} NXF_PATH=$(dirname "$NXF_BIN") NXF_URL=${NXF_URL:-$NXF_BASE/v$NXF_VER/$NXF_JAR} -NXF_GRAB=${NXF_GRAB:-''} -NXF_CLASSPATH=${NXF_CLASSPATH:-''} NXF_HOST=${HOSTNAME:-localhost} [[ $NXF_LAUNCHER ]] || NXF_LAUNCHER=${NXF_HOME}/tmp/launcher/nextflow-${NXF_PACK}_${NXF_VER}/${NXF_HOST} +# both NXF_GRAB and NXF_CLASSPATH are not supported any more as of version 24.04.7-edge +NXF_GRAB=${NXF_GRAB:-''} +NXF_CLASSPATH=${NXF_CLASSPATH:-''} # Determine the path to this file -if [[ $NXF_PACK = all ]]; then +if [[ $NXF_PACK = dist ]]; then NXF_BIN=$(which "$0" 2>/dev/null) [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" fi -# use nextflow custom java home path +# use nextflow custom java home path if [[ "$NXF_JAVA_HOME" ]]; then JAVA_HOME="$NXF_JAVA_HOME" unset JAVA_CMD @@ -346,10 +354,12 @@ if [[ $cmd == console ]]; then bg=1; else JAVA_OPTS+=(-Djava.awt.headless=true) fi +if [[ $NXF_LEGACY_LAUNCHER ]]; then + [[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") + [[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) + [[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) +fi [[ "$JAVA_VER" =~ ^(21|22) ]] && [[ ! "$NXF_ENABLE_VIRTUAL_THREADS" ]] && NXF_ENABLE_VIRTUAL_THREADS=true -[[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") -[[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) -[[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) [[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1) [[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS) [[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH @@ -401,9 +411,14 @@ LAUNCH_FILE="${NXF_LAUNCHER}/classpath-$(env_md5)" if [ -s "$LAUNCH_FILE" ] && [ "$LAUNCH_FILE" -nt "$NXF_BIN" ] && [[ "$remote_debug" -ne 1 ]]; then declare -a launcher="($(cat "$LAUNCH_FILE"))" else - # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file - cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) - [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1 + if [[ $NXF_LEGACY_LAUNCHER ]]; then + # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file + cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) + [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1 + else + # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file + cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN") + fi # first string between double quotes is the full path to java, also blank spaces are included # remainder string are arguments @@ -445,6 +460,8 @@ else launcher+=("${cmd_tail[@]}") fi + # create the launch file only when using the legacy launcher (capsule) + if [[ $NXF_LEGACY_LAUNCHER ]]; then # Don't show errors if the LAUNCH_FILE can't be created if mkdir -p "${NXF_LAUNCHER}" 2>/dev/null; then STR='' @@ -455,6 +472,7 @@ else else echo_yellow "Warning: Couldn't create cached classpath folder: $NXF_LAUNCHER -- Maybe NXF_HOME is not writable?" fi + fi fi diff --git a/packing.gradle b/packing.gradle index 42e2b49089..ccbfdaf40e 100644 --- a/packing.gradle +++ b/packing.gradle @@ -1,5 +1,4 @@ configurations { - capsule defaultCfg.extendsFrom api //provided console.extendsFrom defaultCfg @@ -17,9 +16,6 @@ dependencies { defaultCfg "org.apache.ivy:ivy:2.5.2" // default cfg = runtime + httpfs + amazon + tower client + wave client defaultCfg project(':nf-httpfs') - // Capsule manages the fat jar building process - capsule 'io.nextflow:capsule:1.1.1' - capsule 'io.nextflow:capsule-maven:1.0.3.2' console project(':plugins:nf-console') google project(':plugins:nf-google') amazon project(':plugins:nf-amazon') @@ -92,121 +88,43 @@ protected coordinates( it ) { } /* - * Default nextflow package. It contains the capsule loader + * Compile and pack all packages */ -task packOne(type: Jar) { - dependsOn configurations.capsule, configurations.defaultCfg - archiveFileName = "nextflow-${version}-one.jar" - - from (configurations.capsule.collect { zipTree(it) }) - - // main manifest attributes - def deps = resolveDeps('defaultCfg') - - manifest.attributes( - 'Main-Class' : 'NextflowLoader', - 'Application-Name' : 'nextflow', - 'Application-Class' : mainClassName, - 'Application-Version': version, - 'Min-Java-Version' : '1.8.0', - 'Caplets' : 'MavenCapsule', - 'Dependencies' : deps - ) - - // enable snapshot dependencies lookup - if( version.endsWith('-SNAPSHOT') ) { - manifest.attributes 'Allow-Snapshots': true - manifest.attributes 'Repositories': 'local https://oss.sonatype.org/content/repositories/snapshots central seqera' - } - else { - manifest.attributes 'Repositories': 'central seqera' - } - - doLast { - ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true) - ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true) - println "\n+ Nextflow package `ONE` copied to: $releaseDir" - } -} - -task packAll(type: Jar) { - dependsOn configurations.capsule, configurations.defaultCfg - archiveFileName = "nextflow-${version}-all.jar" - - from jar // embed our application jar - from (configurations.amazon + configurations.google + configurations.tower + configurations.wave) - from (configurations.capsule.collect { zipTree(it) }) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest.attributes( 'Main-Class' : 'NextflowLoader', - 'Application-Name' : 'nextflow-all', - 'Application-Class' : mainClassName, - 'Application-Version': version, - 'Min-Java-Version' : '1.8.0' - ) - - manifest.attributes('Main-Class': 'NextflowLoader', 'amazon') - manifest.attributes('Main-Class': 'NextflowLoader', 'google') - +task packOne( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { doLast { - file(releaseDir).mkdir() - // cleanup - def source = file("$buildDir/libs/nextflow-${version}-all.jar") - def target = file("$releaseDir/nextflow-${version}-all"); target.delete() - // append the big jar - target.withOutputStream { - it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all') - it << new FileInputStream(source) - } - // execute permission - "chmod +x $target".execute() - // done - println "+ Nextflow package `ALL` copied to: $releaseDir\n" + def source = "modules/nextflow/build/libs/nextflow-${version}-one.jar" + ant.copy(file: source, todir: releaseDir, overwrite: true) + ant.copy(file: source, todir: nextflowDir, overwrite: true) + println "\n+ Nextflow package `ONE` copied to: $releaseDir/nextflow-${version}-one.jar" } } -task packCore(type: Jar) { - dependsOn configurations.capsule, configurations.defaultCfg - archiveFileName = "nextflow-${version}-core.jar" - - from jar // embed our application jar - from (configurations.defaultCfg) - from (configurations.capsule.collect { zipTree(it) }) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest.attributes( 'Main-Class' : 'NextflowLoader', - 'Application-Name' : 'nextflow-core', - 'Application-Class' : mainClassName, - 'Application-Version': version, - 'Min-Java-Version' : '1.8.0' - ) +task packDist( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { doLast { - file(releaseDir).mkdir() + file(releaseDir).mkdirs() // cleanup - def source = file("$buildDir/libs/nextflow-${version}-core.jar") - def target = file("$releaseDir/nextflow-${version}-core"); target.delete() + def source = file("modules/nextflow/build/libs/nextflow-${version}-one.jar") + def target = file("$releaseDir/nextflow-${version}-dist"); target.delete() // append the big jar target.withOutputStream { - it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all') + it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=dist') it << new FileInputStream(source) } // execute permission "chmod +x $target".execute() // done - println "+ Nextflow package `CORE` copied to: $releaseDir\n" + println "+ Nextflow package `ALL` copied to: $target\n" } } - /* * Compile and pack all packages */ -task pack( dependsOn: [packOne, packAll]) { +task pack( dependsOn: [packOne, packDist]) { } - task deploy( type: Exec, dependsOn: [clean, compile, pack]) { def temp = File.createTempFile('upload',null)