Inline reference image configuration [BT-122] (broadinstitute#6176)

lifebit-ai · Feb 9, 2021 · 3ca261a · 3ca261a
1 parent d954310
commit 3ca261a
Show file tree

Hide file tree

Showing 10 changed files with 5,045 additions and 158 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # Cromwell Change Log
 
+
+## 57 Release Notes
+
+### Breaking configuration change to reference disk support on PAPI v2
+
+Beginning with Cromwell 57, reference disk manifests are now specified completely within Cromwell configuration
+rather than through a level of indirection to a manifest file stored in GCS. More details can be found
+[here](https://cromwell.readthedocs.io/en/develop/backends/Google#reference-disk-support).
+
 ## 56 Release Notes
 
 ### Retry with More Memory as workflow option

diff --git a/cromwell.example.backends/PAPIv2.conf b/cromwell.example.backends/PAPIv2.conf
@@ -148,7 +148,7 @@ backend {
           zones: ["us-central1-a", "us-central1-b"]
         }
 
-        reference-disk-localization-manifest-files = ["gs://gcp-public-data--broad-references/refdisk_manifest.json"]
+        include "papi_v2_reference_image_manifest.conf"
       }
     }
   }

diff --git a/docs/backends/Google.md b/docs/backends/Google.md
@@ -496,9 +496,11 @@ before eventually giving up and running the job. This behavior may be corrected
 ### Reference Disk Support
 
 Cromwell 55 and later support mounting reference disks from prebuilt GCP disk images as an alternative to localizing large
-input reference files on PAPI v2. Within the `config` stanza of a PAPI v2 backend the `reference-disk-localization-manifest-files`
-key specifies an array of manifest JSONs in GCS:  
+input reference files on PAPI v2. Please note the configuration of reference disk manifests has changed starting with
+Cromwell 57 and now uses the format documented below. 
 
+Within the `config` stanza of a PAPI v2 backend the `reference-disk-localization-manifests`
+key specifies an array of reference disk manifests:  
 
 ```hocon
 backend {
@@ -509,32 +511,28 @@ backend {
       actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory"
       config {
         ...
-        reference-disk-localization-manifest-files = ["gs://path/to/a/reference/disk/manifest.json"]
+        reference-disk-localization-manifests = [
+          {
+            "imageIdentifier" : "projects/broad-dsde-cromwell-dev/global/images/broad-references-disk-image",
+            "diskSizeGb" : 500,
+            "files" : [ {
+              "path" : "gcp-public-data--broad-references/Homo_sapiens_assembly19_1000genomes_decoy/Homo_sapiens_assembly19_1000genomes_decoy.fasta.nhr",
+              "crc32c" : 407769621
+            }, {
+              "path" : "gcp-public-data--broad-references/Homo_sapiens_assembly19_1000genomes_decoy/Homo_sapiens_assembly19_1000genomes_decoy.fasta.sa",
+              "crc32c" : 1902048083
+            },
+            ...
+          },
+          ...
+        ]
         ...
       }
     }
   }
 }
 ```
 
-Reference manifest JSONs have a format like:
-
-```json
-{
-  "imageIdentifier" : "projects/my_project/global/images/my-references-disk-image",
-  "diskSizeGb" : 30,
-  "files" : [ {
-    "path" : "my-references/enormous_reference.bam",
-    "crc32c" : 407769621
-  }, {
-    "path" : "my-references/enormous_reference.bam.bai",
-    "crc32c" : 1902048083
-  },
-...
-  ]
-}
-```
-
 Reference disk usage is an opt-in feature, so workflow submissions must specify this workflow option:
 
 ```json
@@ -552,7 +550,7 @@ reference image without the leading `gs://`, Cromwell would
 arrange for a reference disk based on this image to be mounted and for the call's input to refer to the 
 copy of the file on the reference disk, bypassing localization of the input.     
 
-The Cromwell git repository includes a Java-based tool to facilitate the creation of manifest files called
+The Cromwell git repository includes a Java-based tool to facilitate the creation of manifests called
 [CromwellRefdiskManifestCreatorApp](https://github.com/broadinstitute/cromwell/tree/develop/CromwellRefdiskManifestCreator).
 Please see the help command of that tool for more details.
 

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -15,6 +15,7 @@ object Dependencies {
   private val betterFilesV = "3.9.1"
   private val catsEffectV = "2.3.0"
   private val catsV = "2.3.0"
+  private val circeConfigV = "0.8.0"
   private val circeGenericExtrasV = "0.13.0"
   private val circeOpticsV = "0.13.0"
   private val circeV = "0.13.0"
@@ -151,7 +152,8 @@ object Dependencies {
     "refined",
     "literal"
   ).map(m => "io.circe" %% s"circe-$m" % circeV) :+ circeYamlDependency :+
-  "io.circe" %% "circe-generic-extras" % circeGenericExtrasV
+  "io.circe" %% "circe-generic-extras" % circeGenericExtrasV :+
+  "io.circe" %% "circe-config" % circeConfigV
 
   private val catsDependencies = List(
     "org.typelevel" %% "cats-core" % catsV,
-Original file line number
+Diff line change
@@ Expand Up / @@ -148,7 +148,7 @@ backend { @@
               zones: ["us-central1-a", "us-central1-b"]
             }
-            reference-disk-localization-manifest-files = ["gs://gcp-public-data--broad-references/refdisk_manifest.json"]
+            include "papi_v2_reference_image_manifest.conf"
           }
         }
       }
@@ Expand Down @@