Allow non-call-cached tasks (broadinstitute#5426)

lifebit-ai · Feb 25, 2020 · 6d2875d · 6d2875d
1 parent 5545a66
commit 6d2875d
Show file tree

Hide file tree

Showing 7 changed files with 91 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,11 @@ rows in this table:
 SELECT table_rows FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'cromwell' AND table_name = 'JOB_STORE_SIMPLETON_ENTRY';
 ```
 
+### Disable call-caching for tasks
+
+It is now possible to indicate in a workflow that a task should not be call-cached. See details 
+[here](https://cromwell.readthedocs.io/en/stable/optimizations/VolatileTasks).
+
 ### Delete Intermediate Outputs on PapiV2
 
 * **Experimental:** When a new workflow option `delete_intermediate_output_files` is submitted with the workflow,

diff --git a/centaur/src/main/resources/standardTestCases/volatile_disables_cache.test b/centaur/src/main/resources/standardTestCases/volatile_disables_cache.test
@@ -0,0 +1,12 @@
+name: volatile_disables_cache
+testFormat: runtwiceexpectingnocallcaching
+
+files {
+  workflow: volatile_disables_cache/volatile_disables_cache.wdl
+  options: volatile_disables_cache/volatile_disables_cache.options
+}
+
+metadata {
+  workflowName: volatile_disables_cache
+  status: Succeeded
+}
diff --git a/.../main/resources/standardTestCases/volatile_disables_cache/volatile_disables_cache.options b/.../main/resources/standardTestCases/volatile_disables_cache/volatile_disables_cache.options
@@ -0,0 +1,4 @@
+{
+  "read_from_cache": true,
+  "write_to_cache": true
+}
diff --git a/.../src/main/resources/standardTestCases/volatile_disables_cache/volatile_disables_cache.wdl b/.../src/main/resources/standardTestCases/volatile_disables_cache/volatile_disables_cache.wdl
@@ -0,0 +1,24 @@
+version 1.0
+
+workflow volatile_disables_cache {
+    call volatile_task
+    output {
+        Int random = volatile_task.out
+    }
+}
+
+task volatile_task {
+    meta {
+        description: "Do something stochastic. We don't want this to call cache!"
+        volatile: true
+    }
+    command <<<
+        echo $RANDOM
+    >>>
+    runtime {
+        docker: "ubuntu:latest"
+    }
+    output {
+        Int out = read_int(stdout())
+    }
+}
diff --git a/docs/optimizations/VolatileTasks.md b/docs/optimizations/VolatileTasks.md
@@ -0,0 +1,43 @@
+# The 'volatile' Optimization
+
+Available in Cromwell version 49 and higher.
+
+### Effect on Call Caching:
+
+The 'volatile' optimization is applied to tasks in their `meta` section.
+Call caching will be disabled for any call to that task during the execution of the workflow. 
+
+This is particularly useful if:
+
+* One task can produce stochastic results but you still want to use call caching in the rest of the workflow.
+* You want to guarantee that a task is never call cached for any other reason.
+
+## Language Support
+
+### WDL
+
+In a WDL `task`, this optimization is specified by adding a `volatile` field to 
+the task's `meta` section. Here's an example:
+
+```wdl
+version 1.0
+ 
+task make_random_int {
+  
+  meta {
+    volatile: true
+  }
+  
+  command <<<
+    echo $RANDOM
+  >>>
+
+  output {
+    Int random = read_string(stdout())
+  }
+}
+```
+
+## Backend Support
+
+The volatile keyword applies equally to all backends.
diff --git a/...main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala b/...main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala
@@ -39,6 +39,7 @@ import cromwell.services.EngineServicesStore
 import cromwell.services.metadata.CallMetadataKeys.CallCachingKeys
 import cromwell.services.metadata.{CallMetadataKeys, MetadataKey}
 import cromwell.webservice.EngineStatsActor
+import wom.callable.MetaValueElement.MetaValueElementBoolean
 
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
@@ -87,6 +88,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   // NB: this can also change (e.g. if we have a HashError we just force this to CallCachingOff)
   private[execution] var effectiveCallCachingMode = {
     if (backendLifecycleActorFactory.fileHashingActorProps.isEmpty) CallCachingOff
+    else if (jobDescriptorKey.node.callable.meta.get("volatile").contains(MetaValueElementBoolean(true))) CallCachingOff
     else if (backendLifecycleActorFactory.cacheHitCopyingActorProps.isEmpty || jobDescriptorKey.attempt > 1) {
       callCachingMode.withoutRead
     } else callCachingMode

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -41,6 +41,7 @@ nav:
 - Workflow Optimizations:
   - Optimizations: optimizations/optimizations.md
   - "Task inputs: localization_optional": optimizations/FileLocalization.md
+  - "Volatile Tasks: volatile": optimizations/VolatileTasks.md
 - Configuration: Configuring.md
 - Ecosystem: Ecosystem.md
 - WOMtool: WOMtool.md