Skip to content

Commit

Permalink
Sunsetting Spack support in Wave
Browse files Browse the repository at this point in the history 
Spack directive is not supported any more by Wave containers

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
  • Loading branch information
@pditommaso
pditommaso committed Oct 7, 2024
1 parent 64bb5a9 commit 3a54cb3
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 191 deletions.
52 changes: 3 additions & 49 deletions plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ class WaveClient {

public static final List<String> DEFAULT_CONDA_CHANNELS = ['conda-forge','bioconda']

private static final String DEFAULT_SPACK_ARCH = 'x86_64'

private static final String DEFAULT_DOCKER_PLATFORM = 'linux/amd64'

final private HttpClient httpClient
Expand Down Expand Up @@ -352,12 +350,6 @@ class WaveClient {
if( attrs.container && attrs.conda ) {
throw new IllegalArgumentException("Process '${name}' declares both 'container' and 'conda' directives that conflict each other")
}
if( attrs.container && attrs.spack ) {
throw new IllegalArgumentException("Process '${name}' declares both 'container' and 'spack' directives that conflict each other")
}
if( attrs.spack && attrs.conda ) {
throw new IllegalArgumentException("Process '${name}' declares both 'spack' and 'conda' directives that conflict each other")
}
checkConflicts0(attrs, name, 'dockerfile')
checkConflicts0(attrs, name, 'singularityfile')
}
Expand All @@ -369,9 +361,6 @@ class WaveClient {
if( attrs.container && attrs.get(fileType) ) {
throw new IllegalArgumentException("Process '${name}' declares both a 'container' directive and a module bundle $fileType that conflict each other")
}
if( attrs.get(fileType) && attrs.spack ) {
throw new IllegalArgumentException("Process '${name}' declares both a 'spack' directive and a module bundle $fileType that conflict each other")
}
}

Map<String,String> resolveConflicts(Map<String,String> attrs, List<String> strategy) {
Expand Down Expand Up @@ -413,15 +402,13 @@ class WaveClient {
WaveAssets resolveAssets(TaskRun task, String containerImage, boolean singularity) {
// get the bundle
final bundle = task.getModuleBundle()
// get the Spack architecture
// get the architecture
final arch = task.config.getArchitecture()
final spackArch = arch ? arch.spackArch : DEFAULT_SPACK_ARCH
final dockerArch = arch? arch.dockerArch : DEFAULT_DOCKER_PLATFORM
// compose the request attributes
def attrs = new HashMap<String,String>()
attrs.container = containerImage
attrs.conda = task.config.conda as String
attrs.spack = task.config.spack as String
if( bundle!=null && bundle.dockerfile ) {
attrs.dockerfile = bundle.dockerfile.text
}
Expand All @@ -437,10 +424,10 @@ class WaveClient {
checkConflicts(attrs, task.lazyName())

// resolve the wave assets
return resolveAssets0(attrs, bundle, singularity, dockerArch, spackArch)
return resolveAssets0(attrs, bundle, singularity, dockerArch)
}

protected WaveAssets resolveAssets0(Map<String,String> attrs, ResourcesBundle bundle, boolean singularity, String dockerArch, String spackArch) {
protected WaveAssets resolveAssets0(Map<String,String> attrs, ResourcesBundle bundle, boolean singularity, String dockerArch) {

final scriptType = singularity ? 'singularityfile' : 'dockerfile'
String containerScript = attrs.get(scriptType)
Expand Down Expand Up @@ -485,33 +472,6 @@ class WaveClient {
}
}

/*
* If 'spack' directive is specified use it to create a container file
* to assemble the target container
*/
if( attrs.spack ) {
if( containerScript )
throw new IllegalArgumentException("Unexpected spack and dockerfile conflict while resolving wave container")

if( isSpackFile(attrs.spack) ) {
// create a minimal spack file with package spec from user input
final spackFile = Path.of(attrs.spack)
final spackEnv = addPackagesToSpackYaml(spackFile.text, config.spackOpts())
packagesSpec = new PackagesSpec()
.withType(PackagesSpec.Type.SPACK)
.withSpackOpts(config.spackOpts())
.withEnvironment(spackEnv.bytes.encodeBase64().toString())
}
else {
// create a minimal spack file with package spec from user input
final spackEnv = spackPackagesToSpackYaml(attrs.spack, config.spackOpts())
packagesSpec = new PackagesSpec()
.withType(PackagesSpec.Type.SPACK)
.withSpackOpts(config.spackOpts())
.withEnvironment(spackEnv.bytes.encodeBase64().toString())
}
}

/*
* The process should declare at least a container image name via 'container' directive
* or a dockerfile file to build, otherwise there's no job to be done by wave
Expand Down Expand Up @@ -644,12 +604,6 @@ class WaveClient {
value.startsWith('http://') || value.startsWith('https://')
}

protected boolean isSpackFile(String value) {
if( value.contains('\n') )
return false
return value.endsWith('.yaml') || value.endsWith('.yml')
}

protected boolean refreshJwtToken0(String refresh) {
log.debug "Token refresh request >> $refresh"

Expand Down
142 changes: 0 additions & 142 deletions plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package io.seqera.wave.plugin

import static java.nio.file.StandardOpenOption.*
import static test.TestHelper.*

import java.net.http.HttpRequest
import java.nio.file.Files
Expand Down Expand Up @@ -571,33 +570,6 @@ class WaveClientTest extends Specification {
assets.packagesSpec.channels == WaveClient.DEFAULT_CONDA_CHANNELS
}

def 'should create asset with spack recipe' () {
given:
def session = Mock(Session) { getConfig() >> [:]}
and:
def task = Mock(TaskRun) {getConfig() >> [spack:"rseqc@3.0.1 'rbase@3.5'", arch:"amd64"] }
and:
def client = new WaveClient(session)

when:
def assets = client.resolveAssets(task, null, false)
then:
!assets.containerFile
!assets.moduleResources
!assets.containerImage
!assets.containerConfig
!assets.projectResources
and:
assets.packagesSpec.type == PackagesSpec.Type.SPACK
assets.packagesSpec.entries == null
and:
new String(assets.packagesSpec.environment.decodeBase64()) == '''\
spack:
specs: [rseqc@3.0.1, rbase@3.5]
concretizer: {unify: true, reuse: false}
'''.stripIndent(true)
}

def 'should create asset with conda file' () {
given:
def folder = Files.createTempDirectory('test')
Expand Down Expand Up @@ -625,32 +597,6 @@ class WaveClientTest extends Specification {
folder?.deleteDir()
}

def 'should create asset with spack file' () {
given:
def folder = Files.createTempDirectory('test')
def spackFile = folder.resolve('spack.yaml'); spackFile.text = 'the-spack-recipe-here'
and:
def session = Mock(Session) { getConfig() >> [:]}
def task = Mock(TaskRun) {getConfig() >> [spack:spackFile.toString(), arch: 'amd64'] }
and:
def client = new WaveClient(session)

when:
def assets = client.resolveAssets(task, null, false)
then:
!assets.containerFile
!assets.moduleResources
!assets.containerImage
!assets.containerConfig
!assets.projectResources
and:
assets.packagesSpec.type == PackagesSpec.Type.SPACK
new String(assets.packagesSpec.environment.decodeBase64()) == 'the-spack-recipe-here'
!assets.packagesSpec.entries

cleanup:
folder?.deleteDir()
}

// ==== singularity native build + conda ====

Expand Down Expand Up @@ -729,70 +675,6 @@ class WaveClientTest extends Specification {
folder?.deleteDir()
}

def 'should create assets with spack recipe for singularity' () {
given:
def session = Mock(Session) { getConfig() >> [wave:[build:[spack:[commands: ['cmd-foo','cmd-bar']]]]]}
and:
def task = Mock(TaskRun) {getConfig() >> [spack:"rseqc@3.0.1 'rbase@3.5'", arch:"amd64"] }
and:
def client = new WaveClient(session)

when:
def assets = client.resolveAssets(task, null, true)
then:
!assets.containerFile
!assets.moduleResources
!assets.containerImage
!assets.containerConfig
!assets.projectResources
and:
assets.packagesSpec.type == PackagesSpec.Type.SPACK
assets.packagesSpec.entries == null
assets.packagesSpec.spackOpts.commands == ['cmd-foo','cmd-bar']
decodeBase64(assets.packagesSpec.environment) == '''\
spack:
specs: [rseqc@3.0.1, rbase@3.5]
concretizer: {unify: true, reuse: false}
'''.stripIndent(true)
}

def 'should create asset with spack file for singularity' () {
given:
def folder = Files.createTempDirectory('test')
def spackFile = folder.resolve('spack.yml');
spackFile.text = '''\
spack:
specs: [rseqc@3.0.1, rbase@3.5]
concretizer: {unify: true, reuse: false}
'''.stripIndent(true)
and:
def session = Mock(Session) { getConfig() >> [wave:[build:[spack:[basePackages: 'nano@1.2.3']]]]}
def task = Mock(TaskRun) {getConfig() >> [spack:spackFile.toString()] }
and:
def client = new WaveClient(session)

when:
def assets = client.resolveAssets(task, null, true)
then:
assets.singularity
and:
!assets.containerFile
!assets.moduleResources
!assets.containerImage
!assets.containerConfig
!assets.projectResources
and:
assets.packagesSpec.type == PackagesSpec.Type.SPACK
assets.packagesSpec.spackOpts.basePackages == 'nano@1.2.3'
decodeBase64(assets.packagesSpec.environment) == '''\
spack:
specs: [rseqc@3.0.1, rbase@3.5, nano@1.2.3]
concretizer: {unify: true, reuse: false}
'''.stripIndent(true)

cleanup:
folder?.deleteDir()
}

def 'should create assets with project resources' () {
given:
Expand Down Expand Up @@ -901,24 +783,6 @@ class WaveClientTest extends Specification {
e = thrown(IllegalArgumentException)
e.message == "Process 'foo' declares both a 'container' directive and a module bundle dockerfile that conflict each other"

when:
client.checkConflicts([spack:'this', dockerfile:'that'], 'foo')
then:
e = thrown(IllegalArgumentException)
e.message == "Process 'foo' declares both a 'spack' directive and a module bundle dockerfile that conflict each other"

when:
client.checkConflicts([spack:'this', container:'that'], 'foo')
then:
e = thrown(IllegalArgumentException)
e.message == "Process 'foo' declares both 'container' and 'spack' directives that conflict each other"

when:
client.checkConflicts([conda:'this', spack:'that'], 'foo')
then:
e = thrown(IllegalArgumentException)
e.message == "Process 'foo' declares both 'spack' and 'conda' directives that conflict each other"

// singularity file checks
when:
client.checkConflicts([conda:'this', singularityfile:'that'], 'foo')
Expand All @@ -932,12 +796,6 @@ class WaveClientTest extends Specification {
e = thrown(IllegalArgumentException)
e.message == "Process 'foo' declares both a 'container' directive and a module bundle singularityfile that conflict each other"

when:
client.checkConflicts([spack:'this', singularityfile:'that'], 'foo')
then:
e = thrown(IllegalArgumentException)
e.message == "Process 'foo' declares both a 'spack' directive and a module bundle singularityfile that conflict each other"

}

def 'should get project resource bundle' () {
Expand Down

0 comments on commit 3a54cb3

Please sign in to comment.