Merge pull request #314 from uclahs-cds/sfitz-afterscript

use setup_process_afterscript
uclahs-cds · Oct 30, 2024 · 78aa330 · 78aa330
2 parents 5a7582f + 459a7da
commit 78aa330
Show file tree

Hide file tree

Showing 18 changed files with 137 additions and 211 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Use `methods.setup_process_afterscript()` for process logs
 - Output pipeline parameters to log directory using `store_object_as_json`
 - Add `panel_of_normals_vcf` for MuTect2.
 - Add `a_mini-all-tools-vcf-input` to `nftest`

diff --git a/config/methods.config b/config/methods.config
@@ -156,6 +156,7 @@ methods {
         methods.set_intersect_regions_params()
         methods.set_pipeline_log()
         methods.setup_docker_cpus()
+        methods.setup_process_afterscript()
         json_extractor.store_object_as_json( // must be last
             params,
             new File("${params.log_output_dir}/nextflow-log/params.json")

diff --git a/main.nf b/main.nf
@@ -73,28 +73,24 @@ if (params.input_type == 'bam') {
 
     include { somaticsniper } from './module/somaticsniper' addParams(
         workflow_output_dir: "${params.output_dir_base}/SomaticSniper-${params.somaticsniper_version}",
-        workflow_log_output_dir: "${params.log_output_dir}/process-log/SomaticSniper-${params.somaticsniper_version}",
         output_filename: generate_standard_filename("SomaticSniper-${params.somaticsniper_version}",
             params.dataset_id,
             params.sample_id,
             [:]))
     include { strelka2 } from './module/strelka2' addParams(
         workflow_output_dir: "${params.output_dir_base}/Strelka2-${params.strelka2_version}",
-        workflow_log_output_dir: "${params.log_output_dir}/process-log/Strelka2-${params.strelka2_version}",
         output_filename: generate_standard_filename("Strelka2-${params.strelka2_version}",
             params.dataset_id,
             params.sample_id,
             [:]))
     include { mutect2 } from './module/mutect2' addParams(
         workflow_output_dir: "${params.output_dir_base}/Mutect2-${params.GATK_version}",
-        workflow_log_output_dir: "${params.log_output_dir}/process-log/Mutect2-${params.GATK_version}",
         output_filename: generate_standard_filename("Mutect2-${params.GATK_version}",
             params.dataset_id,
             params.sample_id,
             [:]))
     include { muse } from './module/muse' addParams(
         workflow_output_dir: "${params.output_dir_base}/MuSE-${params.MuSE_version}",
-        workflow_log_output_dir: "${params.log_output_dir}/process-log/MuSE-${params.MuSE_version}",
         output_filename: generate_standard_filename("MuSE-${params.MuSE_version}",
             params.dataset_id,
             params.sample_id,
@@ -134,7 +130,6 @@ if (params.input_type == 'bam') {
 } else if (params.input_type == 'vcf') {
     include { process_vcfs } from './module/process-vcfs' addParams(
         workflow_output_dir: "${params.output_dir_base}/Intersect-BCFtools-${params.BCFtools_version}",
-        workflow_log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}",
         output_filename: generate_standard_filename("BCFtools-${params.BCFtools_version}",
         params.dataset_id,
         params.sample_id,
@@ -151,15 +146,13 @@ if (params.input_type == 'bam') {
 
 include { intersect; getToolName } from './module/intersect' addParams(
     workflow_output_dir: "${params.output_dir_base}/Intersect-BCFtools-${params.BCFtools_version}",
-    workflow_log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}",
     output_filename: generate_standard_filename("BCFtools-${params.BCFtools_version}",
         params.dataset_id,
         params.sample_id,
         [:]))
 
 include { plot_vaf } from './module/plot-vaf' addParams(
     workflow_output_dir: "${params.output_dir_base}/Intersect-BCFtools-${params.BCFtools_version}",
-    workflow_log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}",
     output_filename: generate_standard_filename("BPG-${params.bpg_version}",
         params.dataset_id,
         params.sample_id,

diff --git a/module/common.nf b/module/common.nf
@@ -14,17 +14,13 @@ process filter_VCF_BCFtools {
         mode: "copy",
         pattern: "*.vcf.gz",
         enabled: params.save_intermediate_files
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}-${var_type}/log${file(it).getName()}" }
+    ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${var_type}" }
 
     input:
     tuple val(var_type), path(vcf)
 
     output:
     tuple val(var_type), path("*.vcf.gz"), emit: gzvcf
-    path ".command.*"
 
     script:
     """
@@ -38,17 +34,13 @@ process generate_sha512sum {
     publishDir path: "${params.workflow_output_dir}/output",
         mode: "copy",
         pattern: "${file_for_sha512}.sha512"
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}-${id}/log${file(it).getName()}" }
+    ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${id}" }
 
     input:
     tuple val(id), path (file_for_sha512)
 
     output:
     tuple val(id), path("${file_for_sha512}.sha512"), emit: sha512sum
-    path ".command.*"
 
     script:
     """
@@ -62,18 +54,14 @@ process split_VCF_BCFtools {
     publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}",
         mode: "copy",
         pattern: "*.vcf.gz"
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}_${var_type}/log${file(it).getName()}" }
+    ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}_${var_type}" }
 
     input:
     path vcf
     each var_type
 
     output:
     tuple val(var_type), path("*.vcf.gz"), emit: gzvcf
-    path ".command.*"
 
     script:
     if (params.keep_input_prefix) {
@@ -100,18 +88,14 @@ process rename_samples_BCFtools {
     publishDir path: "${params.workflow_output_dir}/output",
         mode: "copy",
         pattern: "*.vcf.gz"
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}-${var_type}/log${file(it).getName()}" }
+    ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${var_type}" }
 
     input:
     val ids
     tuple val(var_type), path(vcf)
 
     output:
     tuple val(var_type), path("*.vcf.gz"), emit: gzvcf
-    path ".command.*"
 
     script:
     info_replacements = ids
@@ -150,17 +134,13 @@ process compress_file_bzip2 {
         mode: "copy",
         pattern: "*.bz2",
         enabled: params.compress_enabled
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}-${file_type}/log${file(it).getName()}" }
+    ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${file_type}" }
 
     input:
     tuple val(file_type), path(file_to_compress)
 
     output:
     tuple val(file_type), path("*.bz2"), emit: compressed_file
-    path ".command.*"
 
     script:
     """

diff --git a/module/intersect-processes.nf b/module/intersect-processes.nf
@@ -16,10 +16,7 @@ process reorder_samples_BCFtools {
         mode: "copy",
         pattern: "*.vcf.gz",
         enabled: params.save_intermediate_files
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}-${algorithm}/log${file(it).getName()}" }
+    ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}-${algorithm}" }
 
     input:
     tuple val(algorithm), path(gzvcf)
@@ -29,7 +26,6 @@ process reorder_samples_BCFtools {
 
     output:
     path "*-reorder.vcf.gz", emit: gzvcf
-    path ".command.*"
 
     script:
     """
@@ -53,10 +49,7 @@ process intersect_VCFs_BCFtools {
         mode: "copy",
         pattern: "isec-1-or-more/*.txt",
         saveAs: { "${file(it).getParent().getName()}/${params.output_filename}_${file(it).getName()}" }
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" }
+    ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" }
 
     input:
     path gzvcf
@@ -67,7 +60,6 @@ process intersect_VCFs_BCFtools {
     output:
     path "*.vcf.gz", emit: gzvcf
     path "*.vcf.gz.tbi", emit: idx
-    path ".command.*"
     path "isec-2-or-more/*.txt"
     path "isec-1-or-more/*.txt", emit: isec
 
@@ -103,17 +95,13 @@ process plot_VennDiagram_R {
     publishDir path: "${params.workflow_output_dir}/output",
         mode: "copy",
         pattern: "*.tiff"
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" }
+    ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" }
 
     input:
     path script_dir
     path isec
 
     output:
-    path ".command.*"
     path "*.tiff"
 
     script:
@@ -129,18 +117,14 @@ process concat_VCFs_BCFtools {
         mode: "copy",
         pattern: "*concat.vcf",
         enabled: params.save_intermediate_files
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" }
+    ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" }
 
     input:
     path vcfs
     path indices
 
     output:
     path "*concat.vcf", emit: vcf
-    path ".command.*"
 
     script:
     vcf_list = vcfs.join(' ')
@@ -164,10 +148,7 @@ process convert_VCF_vcf2maf {
         mode: "copy",
         pattern: "*.maf",
         enabled: params.save_intermediate_files
-    publishDir path: "${params.workflow_log_output_dir}",
-        mode: "copy",
-        pattern: ".command.*",
-        saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" }
+    ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" }
 
     input:
     path vcf
@@ -177,7 +158,6 @@ process convert_VCF_vcf2maf {
 
     output:
     path "*.maf", emit: maf
-    path ".command.*"
 
     script:
     """

diff --git a/module/intersect.nf b/module/intersect.nf
@@ -1,20 +1,21 @@
 include { compress_file_bzip2; generate_sha512sum} from './common'  addParams(
     compress_publishdir : "${params.workflow_output_dir}/output",
-    compress_enabled : true
+    compress_enabled : true,
+    log_dir_prefix: "Intersect-BCFtools-${params.BCFtools_version}"
     )
 include { reorder_samples_BCFtools; intersect_VCFs_BCFtools; plot_VennDiagram_R; concat_VCFs_BCFtools ; convert_VCF_vcf2maf } from './intersect-processes.nf'
 include { compress_index_VCF as compress_index_VCF_reordered } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams(
     options: [
         output_dir: params.workflow_output_dir,
-        log_output_dir: params.workflow_log_output_dir,
+        log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}",
         bgzip_extra_args: params.bgzip_extra_args,
         tabix_extra_args: params.tabix_extra_args,
         is_output_file: false
         ])
 include { compress_index_VCF as compress_index_VCF_concat } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams(
     options: [
         output_dir: params.workflow_output_dir,
-        log_output_dir: params.workflow_log_output_dir,
+        log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}",
         bgzip_extra_args: params.bgzip_extra_args,
         tabix_extra_args: params.tabix_extra_args
         ])

diff --git a/module/muse-processes.nf b/module/muse-processes.nf
@@ -14,13 +14,10 @@ MuSE Options:
 process call_sSNV_MuSE {
     container params.docker_image_MuSE
     publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}",
-            mode: "copy",
-            pattern: "*.txt",
-            enabled: params.save_intermediate_files
-    publishDir path: "${params.workflow_log_output_dir}",
-            mode: "copy",
-            pattern: ".command.*",
-            saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" }
+        mode: "copy",
+        pattern: "*.txt",
+        enabled: params.save_intermediate_files
+        ext log_dir: { "MuSE-${params.MuSE_version}/${task.process.split(':')[-1]}" }
 
     input:
     path tumor
@@ -32,7 +29,6 @@ process call_sSNV_MuSE {
 
     output:
     path("*.txt"), emit: txt
-    path ".command.*"
 
     script:
     """
@@ -48,13 +44,10 @@ process call_sSNV_MuSE {
 process run_sump_MuSE {
     container params.docker_image_MuSE
     publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}",
-            mode: "copy",
-            pattern: "*.vcf",
-            enabled: params.save_intermediate_files
-    publishDir path: "${params.workflow_log_output_dir}",
-            mode: "copy",
-            pattern: ".command.*",
-            saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" }
+        mode: "copy",
+        pattern: "*.vcf",
+        enabled: params.save_intermediate_files
+        ext log_dir: { "MuSE-${params.MuSE_version}/${task.process.split(':')[-1]}" }
 
     input:
     path MuSE_txt
@@ -63,7 +56,6 @@ process run_sump_MuSE {
 
     output:
     path("*.vcf"), emit: vcf
-    path ".command.*"
 
     script:
     arg_seq_type = params.exome ? "-E" : "-G"

diff --git a/module/muse.nf b/module/muse.nf
@@ -1,9 +1,11 @@
 include { call_sSNV_MuSE; run_sump_MuSE } from './muse-processes'
-include { filter_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common'
+include { filter_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' addParams(
+    log_dir_prefix: "MuSE-${params.MuSE_version}"
+    )
 include { compress_index_VCF } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams(
     options: [
         output_dir: params.workflow_output_dir,
-        log_output_dir: params.workflow_log_output_dir,
+        log_output_dir: "${params.log_output_dir}/process-log/MuSE-${params.MuSE_version}",
         bgzip_extra_args: params.bgzip_extra_args,
         tabix_extra_args: params.tabix_extra_args
         ])