Updates to better deal with missing aliquos and read groups

ding-lab · Aug 31, 2022 · 83cdc6f · 83cdc6f
1 parent 8aca724
commit 83cdc6f
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 322 deletions.
diff --git a/2_make_catalog2.sh b/2_make_catalog2.sh
@@ -4,7 +4,21 @@ source discovery_config.sh
 SUFFIX_LIST="/home/mwyczalk_test/Projects/CPTAC3/CPTAC3.catalog/SampleRename.dat"
 ARGS="-c -s $SUFFIX_LIST"
 
-CMD="bash src/process_catalog.sh $@ $ARGS $PROJECT $CASES"
+LOGE="logs/process_catalog2.err"
+LOGO="logs/process_catalog2.out"
+
+CMD="bash src/process_catalog.sh $@ $ARGS $PROJECT $CASES > $LOGO 2> $LOGE"
 >&2 echo Running: $CMD
+>&2 echo Writing logs to $LOGO and $LOGE
 eval $CMD
 
+echo ' '
+>&2 echo The following errors were observed
+grep -h -i error $LOGE $LOGO | sort -u 
+
+echo ' '
+>&2 echo The following warnings were observed
+# ignoring file exist warnings, whihch are common on reruns
+grep -h -i warning $LOGE $LOGO | grep -v "exists. Deleting" | sort -u 
+
+
diff --git a/3_make_catalog3.sh b/3_make_catalog3.sh
@@ -1,6 +1,21 @@
 source discovery_config.sh
 
-CMD="bash src/process_catalog.sh $@ $PROJECT $CASES"
+LOGE="logs/process_catalog3.err"
+LOGO="logs/process_catalog3.out"
+
+#CMD="bash src/process_catalog.sh $@ $PROJECT $CASES > $LOGO 2> $LOGE "
+CMD="bash src/process_catalog.sh $@ $PROJECT $CASES "
 >&2 echo Running: $CMD
+>&2 echo Writing logs to $LOGO and $LOGE
 eval $CMD
 
+echo ' '
+>&2 echo The following errors were observed
+grep -h -i error $LOGE $LOGO | sort -u 
+
+echo ' '
+>&2 echo The following warnings were observed
+# ignoring file exist warnings, whihch are common on reruns
+grep -h -i warning $LOGE $LOGO | grep -v "exists. Deleting" | sort -u 
+
+
diff --git a/results/CPTAC3.Demographics.tsv b/results/CPTAC3.Demographics.tsv
@@ -0,0 +1,5 @@
+case	disease	ethnicity	gender	race	days_to_birth
+C3L-01838	unknown	not hispanic or latino	male	white	-25663
+C3L-01839	unknown	not hispanic or latino	male	white	-25831
+C3L-01840	unknown	not hispanic or latino	female	white	-20522
+C3L-01861	unknown	not hispanic or latino	male	white	-20974
diff --git a/src/get_read_groups.sh b/src/get_read_groups.sh
@@ -143,6 +143,15 @@ while read L; do
 
 done < $DAT
 
+# Do a check to make sure some data has been written to OUTFN. If not, mark this run as empty
 if [ ! -z $OUTFN ]; then
-    >&2 echo Written to $OUTFN
+    if [ -e $OUTFN ]; then
+        >&2 echo Written to $OUTFN
+    else
+        >&2 echo WARNING: $OUTFN is empty / not written.  Will skip this case
+        OUTD=$(dirname $OUTFN)
+        CMD="touch $OUTD/is_empty.flag" 
+        >&2 echo Running: $CMD
+        eval $CMD
+    fi
 fi
diff --git a/src/get_submitted_reads.sh b/src/get_submitted_reads.sh
@@ -151,6 +151,11 @@ if [ $VERBOSE ]; then
     VERBOSE_ARG=${VERBOSE%?}
 fi
 
+if [ ! -e $DAT ]; then
+    >&2 echo ERROR: $DAT is empty.  Continuing
+    return
+fi
+
 # Iterate over all read groups, which have a many-one relationship with submitted (un)aligned reads
 while read L; do
 #    * case

diff --git a/src/process_catalog.sh b/src/process_catalog.sh
@@ -81,6 +81,7 @@ function confirm {
     fi
 }
 
+
 function test_exit_status {
     # Evaluate return value for chain of pipes; see https://stackoverflow.com/questions/90418/exit-shell-script-based-on-process-exit-code
     rcs=${PIPESTATUS[*]};
@@ -271,9 +272,10 @@ function collect_catalog2 {
 
         CASE=$(echo "$L" | cut -f 1 )
         CATALOG="$LOGBASE/outputs/$CASE/${PROJECT}.Catalog.dat"
+        EMPTY_FLAG="$LOGBASE/outputs/$CASE/is_empty.flag"
 
-        if [ ! -f $CATALOG ]; then
-            if [ "$DRYRUN" != "d" ]; then
+        if [ ! -f $CATALOG ] ; then
+            if [ "$DRYRUN" != "d" ] && [ ! -f $EMPTY_FLAG ]; then
                 >&2 echo WARNING: Catalog file $CATALOG for case $CASE does not exist
             fi
             continue