Merge pull request #182 from uclahs-cds/nzeltser-remove-tempdir

Remove temp_dir parameter and replace with work_dir

Author: alkaZeltser

CHANGELOG.md

@@ -14,6 +14,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Update index file extension from all processes to .bam.bai 
 - Standardize config files
 - Remove spark_temp_dir parameter from config template
+- Replace temp_dir parameter with work_dir parameter
 
 ### Added
 - Intermediate file removal

README.md

@@ -160,7 +160,6 @@ After marking dup BAM files, the BAM files are then indexed by utilizing Picard
 | `aligner` | yes | list | Which aligners to use as strings in list format. Current options: `BWA-MEM2, HISAT2`. |
 | `reference_genome_version` | no | string | The genome build version. This is only used when the output files are directly saved to the Boutros Lab data storage registry, by setting `blcds_registered_dataset_output = true`. |
 | `output_dir` | yes | path | Absolute path to the directory where the output files to be saved. This is ignored if the output files are directly saved to the Boutros Lab data storage registry, by setting `blcds_registered_dataset_output = true` |
-| `temp_dir` | yes | path | Absolute path to the directory where the nextflow's intermediate files are saved. If your cluster worker node has the `/scratch` set up, this can be set to it. |
 | `save_intermediate_files` | yes | boolean | Save intermediate files. If yes, not only the final BAM, but also the unmerged, unsorted, and duplicates unmarked BAM files will also be saved. |
 | `cache_intermediate_pipeline_steps` | yes | boolean | Enable cahcing to resume pipeline and the end of the last successful process completion when a pipeline fails (if true the default submission script must be modified). |
 | `mark_duplicates` | no | boolean | Disable processes which mark duplicates. When false, the pipeline stops at the sorting step, outputting a sorted, indexed, unmerged BAM with unmarked duplicates. Recommended for high coverage targeted panel sequencing datasets. Defaults as true to mark duplicates as usual.|

pipeline/align-DNA.nf

@@ -21,7 +21,7 @@ log.info """\
       reference_fasta_index_files_hisat2: ${params.aligner.contains("HISAT2") ? params.reference_fasta_index_files_hisat2 : "None"}
 
    - output: 
-      temp_dir: ${params.temp_dir}
+      work_dir: ${params.work_dir}
       output_dir: ${params.output_dir}
       bam_output_dir: ${params.bam_output_dir}
       bam_output_filename: ${params.bam_output_filename}

pipeline/config/methods.config

@@ -237,10 +237,12 @@ methods {
             // If work_dir was specified as a param and exists or can be created, set workDir. Otherwise, let Nextflow's default behavior dictate workDir
             if (params.containsKey('work_dir') && params.work_dir && methods.check_workdir_permissions(params.work_dir)) {
                 workDir = params.work_dir
+            } else {
+                params.work_dir = "${launchDir}/work"
             }
 
             // If spark_temp_dir was specified as a param and exists or can be created, set as spark tempdir. Otherwise, set as workDir.
-            params.spark_temp_dir = (params.containsKey('spark_temp_dir') && params.spark_temp_dir && methods.check_workdir_permissions(params.spark_temp_dir)) ? params.spark_temp_dir : workDir
+            params.spark_temp_dir = (params.containsKey('spark_temp_dir') && params.spark_temp_dir && methods.check_workdir_permissions(params.spark_temp_dir)) ? params.spark_temp_dir : "${launchDir}/work"
         }
     }
 

pipeline/config/template.config

@@ -26,7 +26,6 @@ params {
 
     // input/output locations
     output_dir = "where/to/save/outputs/${SAMPLE}"
-    temp_dir = "/local/disk/for/temp/file/dir/"
 
     // options
     save_intermediate_files = false

pipeline/modules/align_DNA_BWA_MEM2.nf

@@ -124,7 +124,7 @@ workflow align_DNA_BWA_MEM2_workflow {
       validate_output_file(
          och_bam.mix(
             och_bam_index,
-            Channel.from(params.temp_dir, params.output_dir)
+            Channel.from(params.work_dir, params.output_dir)
             ),
             aligner_log_dir
          )

pipeline/modules/align_DNA_HISAT2.nf

@@ -125,7 +125,7 @@ workflow align_DNA_HISAT2_workflow {
       validate_output_file(
          och_bam.mix(
             och_bam_index,
-            Channel.from(params.temp_dir, params.output_dir)
+            Channel.from(params.work_dir, params.output_dir)
             ),
             aligner_log_dir
          )

pipeline/modules/mark_duplicate_picardtools.nf

@@ -2,7 +2,7 @@
 // mark duplicates with picard
 process run_MarkDuplicate_Picard {
    container params.docker_image_picardtools
-   containerOptions "--volume ${params.temp_dir}:/temp_dir"
+   containerOptions "--volume ${params.work_dir}:/temp_dir"
 
    publishDir path: "${bam_output_dir}",
       pattern: "*.{bam,bai}",

pipeline/modules/mark_duplicates_spark.nf

@@ -9,7 +9,7 @@
 */
 process run_MarkDuplicatesSpark_GATK  {
    container params.docker_image_gatk
-   containerOptions "--volume ${params.temp_dir}:/temp_dir --volume ${params.spark_temp_dir}:/spark_temp_dir -u nobody"
+   containerOptions "--volume ${params.work_dir}:/temp_dir --volume ${params.spark_temp_dir}:/spark_temp_dir -u nobody"
 
    publishDir path: "${bam_output_dir}",
       pattern: "*.bam{,.bai}",
@@ -43,14 +43,14 @@ process run_MarkDuplicatesSpark_GATK  {
 
    //Update tempdir permissions for user 'nobody'
    beforeScript "chmod 777 `pwd`; \
-      if [[ ! -d ${params.temp_dir} ]]; \
+      if [[ ! -d ${params.work_dir} ]]; \
       then \
-         mkdir -p ${params.temp_dir}; \
-         chmod 777 ${params.temp_dir}; \
+         mkdir -p ${params.work_dir}; \
+         chmod 777 ${params.work_dir}; \
       else \
-         if [[ ! `stat -c %a ${params.temp_dir}` == 777 ]]; \
+         if [[ ! `stat -c %a ${params.work_dir}` == 777 ]]; \
          then \
-            chmod 777 ${params.temp_dir}; \
+            chmod 777 ${params.work_dir}; \
          fi; \
       fi; \
       if [[ ! -d ${params.spark_temp_dir} ]]; \

pipeline/modules/sort_bam_picardtools.nf

@@ -2,7 +2,7 @@
 // sort coordinate or queryname order with picard
 process run_SortSam_Picard  {
    container params.docker_image_picardtools
-   containerOptions "--volume ${params.temp_dir}:/temp_dir"
+   containerOptions "--volume ${params.work_dir}:/temp_dir"
 
    publishDir path: "${intermediate_output_dir}/${task.process.split(':')[1].replace('_', '-')}",
       enabled: params.save_intermediate_files && params.mark_duplicates,