Merge pull request #658 from broadinstitute/dp-quote-wdl-file-paths

Quote WDL File variable expansions to handle paths with spaces

Author: dpark01

pipes/WDL/tasks/tasks_assembly.wdl

@@ -71,15 +71,15 @@ task assemble {
         assembly --version | tee VERSION
 
         assembly assemble_spades \
-          ~{reads_unmapped_bam} \
-          ~{trim_clip_db} \
-          ~{sample_name}.assembly1-spades.fasta \
+          "~{reads_unmapped_bam}" \
+          "~{trim_clip_db}" \
+          "~{sample_name}.assembly1-spades.fasta" \
           ~{'--nReads=' + spades_n_reads} \
           ~{true="--alwaysSucceed" false="" always_succeed} \
           ~{'--minContigLen=' + spades_min_contig_len} \
           ~{'--spadesOpts="' + spades_options + '"'} \
           --memLimitGb $mem_in_gb \
-          --outReads=~{sample_name}.subsamp.bam \
+          --outReads="~{sample_name}.subsamp.bam" \
           --loglevel=DEBUG
 
         samtools view -c ~{sample_name}.subsamp.bam | tee subsample_read_count >&2
@@ -595,13 +595,13 @@ task ivar_trim {
         ivar version | head -1 | tee VERSION
         if [ -f "~{trim_coords_bed}" ]; then
           ivar trim -e \
-            ~{'-b ' + trim_coords_bed} \
+            ~{'-b "' + trim_coords_bed + '"'} \
             ~{'-m ' + min_keep_length} \
             ~{'-s ' + sliding_window} \
             ~{'-q ' + min_quality} \
             ~{'-x ' + primer_offset} \
-            -i ~{aligned_bam} -p trim | tee IVAR_OUT
-          samtools sort -@ $(nproc) -m 1000M -o ~{bam_basename}.trimmed.bam trim.bam
+            -i "~{aligned_bam}" -p trim | tee IVAR_OUT
+          samtools sort -@ $(nproc) -m 1000M -o "~{bam_basename}.trimmed.bam" trim.bam
         else
           echo "skipping ivar trim"
           cp "~{aligned_bam}" "~{bam_basename}.trimmed.bam"
@@ -922,16 +922,16 @@ task refine_assembly_with_aligned_reads {
 
         if [ ~{true='true' false='false' mark_duplicates} == "true" ]; then
           read_utils mkdup_picard \
-            ~{reads_aligned_bam} \
+            "~{reads_aligned_bam}" \
             temp_markdup.bam \
             --JVMmemory "$mem_in_mb"m \
             --loglevel=DEBUG
         else
-          ln -s ~{reads_aligned_bam} temp_markdup.bam
+          ln -s "~{reads_aligned_bam}" temp_markdup.bam
         fi
         samtools index -@ $(nproc) temp_markdup.bam temp_markdup.bai
 
-        ln -s ~{reference_fasta} assembly.fasta
+        ln -s "~{reference_fasta}" assembly.fasta
         assembly refine_assembly \
           assembly.fasta \
           temp_markdup.bam \

pipes/WDL/tasks/tasks_demux.wdl

@@ -239,22 +239,22 @@ task illumina_demux {
     fi
     
     # Parse the lane count & run ID from RunInfo.xml file
-    lane_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@LaneCount)" $RUNINFO_FILE)
+    lane_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@LaneCount)" "$RUNINFO_FILE")
     if [ -z "$lane_count" ]; then
         echo "Could not parse LaneCount from RunInfo.xml. Please check RunInfo.xml is properly formatted"
     fi
 
-    surface_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@SurfaceCount)" $RUNINFO_FILE)
+    surface_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@SurfaceCount)" "$RUNINFO_FILE")
     if [ -z "$surface_count" ]; then
         echo "Could not parse SurfaceCount from RunInfo.xml. Please check RunInfo.xml is properly formatted"
     fi
 
-    swath_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@SwathCount)" $RUNINFO_FILE)
+    swath_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@SwathCount)" "$RUNINFO_FILE")
     if [ -z "$swath_count" ]; then
         echo "Could not parse SwathCount from RunInfo.xml. Please check RunInfo.xml is properly formatted"
     fi
 
-    tile_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@TileCount)" $RUNINFO_FILE)
+    tile_count=$(xmllint --xpath "string(//Run/FlowcellLayout/@TileCount)" "$RUNINFO_FILE")
     if [ -z "$tile_count" ]; then
         echo "Could not parse TileCount from RunInfo.xml. Please check RunInfo.xml is properly formatted"
     fi
@@ -366,9 +366,9 @@ task illumina_demux {
       $FLOWCELL_DIR \
       ~{lane} \
       . \
-      ~{'--sampleSheet=' + samplesheet} \
-      ~{'--runInfo=' + runinfo} \
-      ~{'--sequencing_center=' + sequencingCenter} \
+      ~{'--sampleSheet="' + samplesheet + '"'} \
+      ~{'--runInfo="' + runinfo + '"'} \
+      ~{'--sequencing_center="' + sequencingCenter + '"'} \
       --outMetrics=metrics.txt \
       --commonBarcodes=barcodes.txt \
       ~{'--flowcell=' + flowcell} \
@@ -436,7 +436,7 @@ task illumina_demux {
       ~{'--predemux_trim_r1_3prime '  + inner_barcode_predemux_trim_r1_3prime} \
       ~{'--predemux_trim_r2_5prime '  + inner_barcode_predemux_trim_r2_5prime} \
       ~{'--predemux_trim_r2_3prime '  + inner_barcode_predemux_trim_r2_3prime} \
-      ~{'--sampleSheet=' + samplesheet} \
+      ~{'--sampleSheet="' + samplesheet + '"'} \
       "--runInfo=${RUNINFO_FILE}" \
       --illuminaRunDirectory=$FLOWCELL_DIR \
       $demux_threads \
@@ -843,7 +843,7 @@ task get_illumina_run_metadata {
     illumina --version | tee VERSION
 
     illumina illumina_metadata \
-      --runinfo ~{runinfo_xml} \
+      --runinfo "~{runinfo_xml}" \
       ~{'--sequencing_center ' + sequencing_center} \
       --out_runinfo runinfo.json \
       --loglevel=DEBUG
@@ -969,15 +969,15 @@ task demux_fastqs {
     illumina --version | tee VERSION
 
     illumina splitcode_demux_fastqs \
-      --fastq_r1 ~{fastq_r1} \
-      ~{'--fastq_r2 ' + fastq_r2} \
-      --samplesheet ~{samplesheet} \
-      --runinfo ~{runinfo_xml} \
-      ~{'--sequencing_center ' + sequencingCenter} \
+      --fastq_r1 "~{fastq_r1}" \
+      ~{'--fastq_r2 "' + fastq_r2 + '"'} \
+      --samplesheet "~{samplesheet}" \
+      --runinfo "~{runinfo_xml}" \
+      ~{'--sequencing_center "' + sequencingCenter + '"'} \
       --outdir . \
       --append_run_id \
-      --out_meta_by_sample ~{fastq_basename}-meta_by_sample.json \
-      --out_meta_by_filename ~{fastq_basename}-meta_by_filename.json \
+      --out_meta_by_sample "~{fastq_basename}-meta_by_sample.json" \
+      --out_meta_by_filename "~{fastq_basename}-meta_by_filename.json" \
       --loglevel=DEBUG
 
     # Workaround: create empty JSON files if Python code didn't produce them (zero-read FASTQs)

pipes/WDL/tasks/tasks_interhost.wdl

@@ -311,7 +311,7 @@ task beast {
       ~{true="-beagle_double" false="-beagle_single" beagle_double_precision} \
       -beagle_scaling always \
       ~{'-beagle_order ' + beagle_order} \
-      ~{beauti_xml}
+      "~{beauti_xml}"
   >>>
 
   output {

pipes/WDL/tasks/tasks_intrahost.wdl

@@ -82,7 +82,7 @@ task polyphonia_detect_cross_contamination {
     fi
 
     polyphonia cross_contamination \
-      --ref ~{reference_fasta} \
+      --ref "~{reference_fasta}" \
       --vcf ~{sep=' ' lofreq_vcfs} \
       --consensus ~{sep=' ' genome_fastas} \
       --read-depths ~{sep=' ' select_first([read_depths, []])} \
@@ -94,7 +94,7 @@ task polyphonia_detect_cross_contamination {
       ~{'--min-matches-proportion ' + min_matches_proportion} \
       ~{'--min-maf ' + min_maf} \
       ~{'--masked-positions ' + masked_positions} \
-      ~{'--masked-positions-file ' + masked_positions_file} \
+      ~{'--masked-positions-file "' + masked_positions_file + '"'} \
       $PLATE_MAPS_INPUT \
       ~{'--plate-size ' + plate_size} \
       ~{'--plate-columns ' + plate_columns} \
@@ -224,8 +224,8 @@ task isnvs_per_sample {
   command <<<
     intrahost --version | tee VERSION
     intrahost vphaser_one_sample \
-        ~{mapped_bam} \
-        ~{assembly_fasta} \
+        "~{mapped_bam}" \
+        "~{assembly_fasta}" \
         vphaser2.~{sample_name}.txt.gz \
         ~{'--vphaserNumThreads=' + threads} \
         ~{true="--removeDoublyMappedReads" false="" removeDoublyMappedReads} \
@@ -286,7 +286,7 @@ task isnvs_vcf {
     echo "snpRefAccessions: $snpRefAccessions"
 
     intrahost merge_to_vcf \
-        ~{reference_fasta} \
+        "~{reference_fasta}" \
         isnvs.vcf.gz \
         $SAMPLES \
         --isnvs ~{sep=' ' vphaser2Calls} \

pipes/WDL/tasks/tasks_megablast.wdl

@@ -98,12 +98,12 @@ task lca_megablast {
     command <<<
         #Extract BLAST DB tarball
         read_utils extract_tarball \
-          ~{blast_db_tgz} . \
+          "~{blast_db_tgz}" . \
           --loglevel=DEBUG
-        
-        # Extract taxonomy DB tarball 
+
+        # Extract taxonomy DB tarball
         read_utils extract_tarball \
-          ~{taxonomy_db_tgz} . \
+          "~{taxonomy_db_tgz}" . \
           --loglevel=DEBUG
 
         '''
@@ -189,9 +189,9 @@ task ChunkBlastHits {
     command <<<
         #Extract tarball contents
         read_utils extract_tarball \
-          ~{blast_db_tgz} . \
+          "~{blast_db_tgz}" . \
           --loglevel=DEBUG
-        export LOG_DIR=~{log_dir_final}
+        export LOG_DIR="~{log_dir_final}"
         echo "Using $(nproc) CPU cores."
         echo "Asked for ~{machine_mem_gb} memory GB"
         #Adding taxidlist input as optional 
@@ -274,12 +274,12 @@ task blastoff {
     command <<<
         #Extract BLAST DB tarball
         read_utils extract_tarball \
-          ~{blast_db_tgz} . \
+          "~{blast_db_tgz}" . \
           --loglevel=DEBUG
 
         # Extract taxonomy DB tarball (includes nodes.dmp)
         read_utils extract_tarball \
-          ~{taxonomy_db_tgz} . \
+          "~{taxonomy_db_tgz}" . \
           --loglevel=DEBUG
         
         export LOG_DIR=~{log_dir_final}

pipes/WDL/tasks/tasks_utils.wdl

@@ -599,7 +599,7 @@ task md5sum {
   }
   Int disk_size = 100
   command <<<
-    md5sum ~{in_file} | cut -f 1 -d ' ' | tee MD5
+    md5sum "~{in_file}" | cut -f 1 -d ' ' | tee MD5
   >>>
   output {
     String md5 = read_string("MD5")
@@ -1138,12 +1138,16 @@ task s3_copy {
     set -e
     S3_PREFIX=$(echo "~{s3_uri_prefix}" | sed 's|/*$||')
     mkdir -p ~/.aws
-    cp ~{aws_credentials} ~/.aws/credentials
+    cp "~{aws_credentials}" ~/.aws/credentials
     touch OUT_URIS
-    for f in ~{sep=' ' infiles}; do
-      aws s3 cp $f $S3_PREFIX/
-      echo "$S3_PREFIX/$(basename $f)" >> OUT_URIS
-    done
+    INFILES=~{write_lines(infiles)}
+    while IFS= read -r f || [ -n "$f" ]; do
+      # Skip empty or whitespace-only lines
+      if [ -n "$f" ] && [ -n "$(echo "$f" | tr -d '[:space:]')" ]; then
+        aws s3 cp "$f" "$S3_PREFIX/"
+        echo "$S3_PREFIX/$(basename "$f")" >> OUT_URIS
+      fi
+    done < "$INFILES"
   >>>
   output {
     Array[String] out_uris = read_lines("OUT_URIS")