From fed4298fbc38f6b2bb9922ee73eac99343e5385c Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Tue, 15 Oct 2024 16:23:42 -0700 Subject: [PATCH 1/3] update log specifications --- CHANGELOG.md | 1 + config/methods.config | 1 + main.nf | 7 --- module/common.nf | 30 ++------- module/intersect-processes.nf | 30 ++------- module/intersect.nf | 7 ++- module/muse-processes.nf | 24 +++---- module/muse.nf | 6 +- module/mutect2-processes.nf | 36 ++--------- module/mutect2.nf | 6 +- module/plot-vaf-processes.nf | 12 +--- module/process-vcfs.nf | 5 +- module/somaticsniper-processes.nf | 100 ++++++++++-------------------- module/somaticsniper.nf | 11 ++-- module/strelka2-processes.nf | 24 +++---- module/strelka2.nf | 6 +- 16 files changed, 95 insertions(+), 211 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fd302a3..33bae71c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Use `methods.setup_process_afterscript()` for process logs - Add `panel_of_normals_vcf` for MuTect2. - Add `a_mini-all-tools-vcf-input` to `nftest` - Add option to input VCFs for intersection diff --git a/config/methods.config b/config/methods.config index 8d5d3bc7..c8785415 100644 --- a/config/methods.config +++ b/config/methods.config @@ -152,5 +152,6 @@ methods { methods.set_intersect_regions_params() methods.set_pipeline_log() methods.setup_docker_cpus() + methods.setup_process_afterscript() } } diff --git a/main.nf b/main.nf index 342c1086..cec97446 100755 --- a/main.nf +++ b/main.nf @@ -73,28 +73,24 @@ if (params.input_type == 'bam') { include { somaticsniper } from './module/somaticsniper' addParams( workflow_output_dir: "${params.output_dir_base}/SomaticSniper-${params.somaticsniper_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/SomaticSniper-${params.somaticsniper_version}", output_filename: generate_standard_filename("SomaticSniper-${params.somaticsniper_version}", params.dataset_id, params.sample_id, [:])) include { strelka2 } from './module/strelka2' addParams( workflow_output_dir: "${params.output_dir_base}/Strelka2-${params.strelka2_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/Strelka2-${params.strelka2_version}", output_filename: generate_standard_filename("Strelka2-${params.strelka2_version}", params.dataset_id, params.sample_id, [:])) include { mutect2 } from './module/mutect2' addParams( workflow_output_dir: "${params.output_dir_base}/Mutect2-${params.GATK_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/Mutect2-${params.GATK_version}", output_filename: generate_standard_filename("Mutect2-${params.GATK_version}", params.dataset_id, params.sample_id, [:])) include { muse } from './module/muse' addParams( workflow_output_dir: "${params.output_dir_base}/MuSE-${params.MuSE_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/MuSE-${params.MuSE_version}", output_filename: generate_standard_filename("MuSE-${params.MuSE_version}", params.dataset_id, params.sample_id, @@ -134,7 +130,6 @@ if (params.input_type == 'bam') { } else if (params.input_type == 'vcf') { include { process_vcfs } from './module/process-vcfs' addParams( workflow_output_dir: "${params.output_dir_base}/Intersect-BCFtools-${params.BCFtools_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}", output_filename: generate_standard_filename("BCFtools-${params.BCFtools_version}", params.dataset_id, params.sample_id, @@ -151,7 +146,6 @@ if (params.input_type == 'bam') { include { intersect; getToolName } from './module/intersect' addParams( workflow_output_dir: "${params.output_dir_base}/Intersect-BCFtools-${params.BCFtools_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}", output_filename: generate_standard_filename("BCFtools-${params.BCFtools_version}", params.dataset_id, params.sample_id, @@ -159,7 +153,6 @@ include { intersect; getToolName } from './module/intersect' addParams( include { plot_vaf } from './module/plot-vaf' addParams( workflow_output_dir: "${params.output_dir_base}/Intersect-BCFtools-${params.BCFtools_version}", - workflow_log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}", output_filename: generate_standard_filename("BPG-${params.bpg_version}", params.dataset_id, params.sample_id, diff --git a/module/common.nf b/module/common.nf index e7d9f30b..d5708439 100644 --- a/module/common.nf +++ b/module/common.nf @@ -14,17 +14,13 @@ process filter_VCF_BCFtools { mode: "copy", pattern: "*.vcf.gz", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${var_type}/log${file(it).getName()}" } + ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${var_type}" } input: tuple val(var_type), path(vcf) output: tuple val(var_type), path("*.vcf.gz"), emit: gzvcf - path ".command.*" script: """ @@ -38,17 +34,13 @@ process generate_sha512sum { publishDir path: "${params.workflow_output_dir}/output", mode: "copy", pattern: "${file_for_sha512}.sha512" - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${id}/log${file(it).getName()}" } + ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${id}" } input: tuple val(id), path (file_for_sha512) output: tuple val(id), path("${file_for_sha512}.sha512"), emit: sha512sum - path ".command.*" script: """ @@ -62,10 +54,7 @@ process split_VCF_BCFtools { publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", mode: "copy", pattern: "*.vcf.gz" - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}_${var_type}/log${file(it).getName()}" } + ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}_${var_type}" } input: path vcf @@ -73,7 +62,6 @@ process split_VCF_BCFtools { output: tuple val(var_type), path("*.vcf.gz"), emit: gzvcf - path ".command.*" script: if (params.keep_input_prefix) { @@ -100,10 +88,7 @@ process rename_samples_BCFtools { publishDir path: "${params.workflow_output_dir}/output", mode: "copy", pattern: "*.vcf.gz" - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${var_type}/log${file(it).getName()}" } + ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${var_type}" } input: val ids @@ -111,7 +96,6 @@ process rename_samples_BCFtools { output: tuple val(var_type), path("*.vcf.gz"), emit: gzvcf - path ".command.*" script: info_replacements = ids @@ -150,17 +134,13 @@ process compress_file_bzip2 { mode: "copy", pattern: "*.bz2", enabled: params.compress_enabled - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${file_type}/log${file(it).getName()}" } + ext log_dir: { "${params.log_dir_prefix}/${task.process.split(':')[-1]}-${file_type}" } input: tuple val(file_type), path(file_to_compress) output: tuple val(file_type), path("*.bz2"), emit: compressed_file - path ".command.*" script: """ diff --git a/module/intersect-processes.nf b/module/intersect-processes.nf index d2e84f2b..97076464 100644 --- a/module/intersect-processes.nf +++ b/module/intersect-processes.nf @@ -16,10 +16,7 @@ process reorder_samples_BCFtools { mode: "copy", pattern: "*.vcf.gz", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${algorithm}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}-${algorithm}" } input: tuple val(algorithm), path(gzvcf) @@ -29,7 +26,6 @@ process reorder_samples_BCFtools { output: path "*-reorder.vcf.gz", emit: gzvcf - path ".command.*" script: """ @@ -53,10 +49,7 @@ process intersect_VCFs_BCFtools { mode: "copy", pattern: "isec-1-or-more/*.txt", saveAs: { "${file(it).getParent().getName()}/${params.output_filename}_${file(it).getName()}" } - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" } input: path gzvcf @@ -67,7 +60,6 @@ process intersect_VCFs_BCFtools { output: path "*.vcf.gz", emit: gzvcf path "*.vcf.gz.tbi", emit: idx - path ".command.*" path "isec-2-or-more/*.txt" path "isec-1-or-more/*.txt", emit: isec @@ -103,17 +95,13 @@ process plot_VennDiagram_R { publishDir path: "${params.workflow_output_dir}/output", mode: "copy", pattern: "*.tiff" - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" } input: path script_dir path isec output: - path ".command.*" path "*.tiff" script: @@ -129,10 +117,7 @@ process concat_VCFs_BCFtools { mode: "copy", pattern: "*concat.vcf", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" } input: path vcfs @@ -140,7 +125,6 @@ process concat_VCFs_BCFtools { output: path "*concat.vcf", emit: vcf - path ".command.*" script: vcf_list = vcfs.join(' ') @@ -164,10 +148,7 @@ process convert_VCF_vcf2maf { mode: "copy", pattern: "*.maf", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" } input: path vcf @@ -177,7 +158,6 @@ process convert_VCF_vcf2maf { output: path "*.maf", emit: maf - path ".command.*" script: """ diff --git a/module/intersect.nf b/module/intersect.nf index e13e148c..4ad5d4f9 100644 --- a/module/intersect.nf +++ b/module/intersect.nf @@ -1,12 +1,13 @@ include { compress_file_bzip2; generate_sha512sum} from './common' addParams( compress_publishdir : "${params.workflow_output_dir}/output", - compress_enabled : true + compress_enabled : true, + log_dir_prefix: "Intersect-BCFtools-${params.BCFtools_version}" ) include { reorder_samples_BCFtools; intersect_VCFs_BCFtools; plot_VennDiagram_R; concat_VCFs_BCFtools ; convert_VCF_vcf2maf } from './intersect-processes.nf' include { compress_index_VCF as compress_index_VCF_reordered } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args, is_output_file: false @@ -14,7 +15,7 @@ include { compress_index_VCF as compress_index_VCF_reordered } from '../external include { compress_index_VCF as compress_index_VCF_concat } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args ]) diff --git a/module/muse-processes.nf b/module/muse-processes.nf index 33a10cb7..a25ebbc0 100644 --- a/module/muse-processes.nf +++ b/module/muse-processes.nf @@ -14,13 +14,10 @@ MuSE Options: process call_sSNV_MuSE { container params.docker_image_MuSE publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.txt", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.txt", + enabled: params.save_intermediate_files + ext log_dir: { "MuSE-${params.MuSE_version}/${task.process.split(':')[-1]}" } input: path tumor @@ -32,7 +29,6 @@ process call_sSNV_MuSE { output: path("*.txt"), emit: txt - path ".command.*" script: """ @@ -48,13 +44,10 @@ process call_sSNV_MuSE { process run_sump_MuSE { container params.docker_image_MuSE publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.vcf", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.vcf", + enabled: params.save_intermediate_files + ext log_dir: { "MuSE-${params.MuSE_version}/${task.process.split(':')[-1]}" } input: path MuSE_txt @@ -63,7 +56,6 @@ process run_sump_MuSE { output: path("*.vcf"), emit: vcf - path ".command.*" script: arg_seq_type = params.exome ? "-E" : "-G" diff --git a/module/muse.nf b/module/muse.nf index 3e5d5865..57475a49 100644 --- a/module/muse.nf +++ b/module/muse.nf @@ -1,9 +1,11 @@ include { call_sSNV_MuSE; run_sump_MuSE } from './muse-processes' -include { filter_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' +include { filter_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' addParams( + log_dir_prefix: "MuSE-${params.MuSE_version}" + ) include { compress_index_VCF } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/MuSE-${params.MuSE_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args ]) diff --git a/module/mutect2-processes.nf b/module/mutect2-processes.nf index 1c4bdd83..7ee8b323 100644 --- a/module/mutect2-processes.nf +++ b/module/mutect2-processes.nf @@ -24,10 +24,7 @@ process run_SplitIntervals_GATK { mode: "copy", pattern: "interval-files/*-scattered.interval_list", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Mutect2-${params.GATK_version}/${task.process.split(':')[-1]}" } input: path intersect_regions @@ -38,7 +35,6 @@ process run_SplitIntervals_GATK { output: path 'interval-files/*-scattered.interval_list', emit: interval_list - path ".command.*" script: intervals_command = params.use_intersect_regions ? "-L ${intersect_regions}" : "" @@ -61,10 +57,7 @@ process call_sSNV_Mutect2 { mode: "copy", pattern: "${params.output_filename}_unfiltered*", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${interval_id}/log${file(it).getName()}" } + ext log_dir: { "Mutect2-${params.GATK_version}/${task.process.split(':')[-1]}-${interval_id}" } input: path interval @@ -84,7 +77,6 @@ process call_sSNV_Mutect2 { path "*.vcf.gz.tbi", emit: unfiltered_index path "*.vcf.gz.stats", emit: unfiltered_stats path "*-f1r2.tar.gz", emit: f1r2 - path ".command.*" script: tumors = tumor.collect { "-I '$it'" }.join(' ') @@ -116,10 +108,7 @@ process run_MergeVcfs_GATK { mode: "copy", pattern: "*_unfiltered.vcf.gz*", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Mutect2-${params.GATK_version}/${task.process.split(':')[-1]}" } input: path unfiltered_vcf @@ -127,7 +116,6 @@ process run_MergeVcfs_GATK { output: path "*_unfiltered.vcf.gz", emit: unfiltered path "*_unfiltered.vcf.gz.tbi", emit: unfiltered_index - path ".command.*" script: unfiltered_vcfs = unfiltered_vcf.collect { "-I '$it'" }.join(' ') @@ -143,17 +131,13 @@ process run_MergeMutectStats_GATK { mode: "copy", pattern: "*_unfiltered.vcf.gz.stats", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Mutect2-${params.GATK_version}/${task.process.split(':')[-1]}" } input: path unfiltered_stat output: path "*_unfiltered.vcf.gz.stats", emit: merged_stats - path ".command.*" script: unfiltered_stats = unfiltered_stat.collect { "-stats '$it'" }.join(' ') @@ -169,17 +153,13 @@ process run_LearnReadOrientationModel_GATK { mode: "copy", pattern: "read-orientation-model.tar.gz", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Mutect2-${params.GATK_version}/${task.process.split(':')[-1]}" } input: path f1r2 output: path "read-orientation-model.tar.gz", emit: read_orientation_model - path ".command.*" script: f1r2 = f1r2.collect { "-I '$it'" }.join(' ') @@ -198,13 +178,10 @@ process run_FilterMutectCalls_GATK { mode: "copy", pattern: "*_filtered.vcf.gz", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } publishDir path: "${params.workflow_output_dir}/QC/${task.process.split(':')[-1]}", mode: "copy", pattern: "*.tsv" + ext log_dir: { "Mutect2-${params.GATK_version}/${task.process.split(':')[-1]}" } input: path reference @@ -218,7 +195,6 @@ process run_FilterMutectCalls_GATK { output: path "*_filtered.vcf.gz", emit: filtered - path ".command.*" path "*_filteringStats.tsv" script: diff --git a/module/mutect2.nf b/module/mutect2.nf index a2c4195c..c044b3db 100644 --- a/module/mutect2.nf +++ b/module/mutect2.nf @@ -1,9 +1,11 @@ include { run_SplitIntervals_GATK; call_sSNV_Mutect2; run_MergeVcfs_GATK; run_MergeMutectStats_GATK; run_LearnReadOrientationModel_GATK; run_FilterMutectCalls_GATK } from './mutect2-processes' -include { filter_VCF_BCFtools; split_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' +include { filter_VCF_BCFtools; split_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' addParams( + log_dir_prefix: "Mutect2-${params.GATK_version}" + ) include { compress_index_VCF } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/Mutect2-${params.GATK_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args ]) diff --git a/module/plot-vaf-processes.nf b/module/plot-vaf-processes.nf index bd31ee90..9f12c5a8 100644 --- a/module/plot-vaf-processes.nf +++ b/module/plot-vaf-processes.nf @@ -16,17 +16,13 @@ process calculate_adjVAF_Python { mode: "copy", pattern: "adjusted_vafs.tsv", enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" } input: val input_data path input_files output: - path ".command.*" path "adjusted_vafs.tsv", emit: adjusted_vafs script: @@ -48,16 +44,12 @@ process plot_adjVAF_R { mode: "copy", pattern: "stripplot.png", saveAs: { "${params.output_filename}_adjVAF.png" } - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "Intersect-BCFtools-${params.BCFtools_version}/${task.process.split(':')[-1]}" } input: path adjvaf_table output: - path ".command.*" path "stripplot.png" script: diff --git a/module/process-vcfs.nf b/module/process-vcfs.nf index 6ab0b816..274f5d62 100644 --- a/module/process-vcfs.nf +++ b/module/process-vcfs.nf @@ -1,11 +1,12 @@ include { generate_standard_filename } from '../external/pipeline-Nextflow-module/modules/common/generate_standardized_filename/main.nf' include { split_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum} from './common' addParams( - keep_input_prefix: true + keep_input_prefix: true, + log_dir_prefix: "Intersect-BCFtools-${params.BCFtools_version}" ) include { compress_index_VCF } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/Intersect-BCFtools-${params.BCFtools_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args ]) diff --git a/module/somaticsniper-processes.nf b/module/somaticsniper-processes.nf index f00ed166..3310fffd 100644 --- a/module/somaticsniper-processes.nf +++ b/module/somaticsniper-processes.nf @@ -12,13 +12,10 @@ Docker Images: process call_sSNV_SomaticSniper { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.vcf", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.vcf", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path tumor @@ -28,7 +25,6 @@ process call_sSNV_SomaticSniper { output: path "*.vcf", emit: bam_somaticsniper - path ".command.*" """ set -euo pipefail @@ -56,13 +52,10 @@ process call_sSNV_SomaticSniper { process convert_BAM2Pileup_SAMtools { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.pileup", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${type}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.pileup", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}-${type}" } input: tuple val(type), path(bam) @@ -71,7 +64,6 @@ process convert_BAM2Pileup_SAMtools { output: tuple val(type), path("${params.output_filename}_raw-${type}.pileup"), emit: raw_pileup - path ".command.*" """ set -euo pipefail @@ -89,20 +81,16 @@ process convert_BAM2Pileup_SAMtools { process create_IndelCandidate_SAMtools { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.pileup", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}-${type}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.pileup", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}-${type}" } input: tuple val(type), path(raw_pileup) output: tuple val(type), path("*_filtered-${type}.pileup"), emit: filtered_pileup - path ".command.*" """ set -euo pipefail @@ -119,13 +107,10 @@ process create_IndelCandidate_SAMtools { process apply_NormalIndelFilter_SomaticSniper { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*_normal.vcf", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*_normal.vcf", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path snp_file @@ -133,7 +118,6 @@ process apply_NormalIndelFilter_SomaticSniper { output: path "*_normal.vcf", emit: vcf_normal - path ".command.*" """ set -euo pipefail @@ -149,13 +133,10 @@ process apply_NormalIndelFilter_SomaticSniper { process apply_TumorIndelFilter_SomaticSniper { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.SNPfilter", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.SNPfilter", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path snp_file @@ -163,7 +144,6 @@ process apply_TumorIndelFilter_SomaticSniper { output: path "*.SNPfilter", emit: vcf_tumor - path ".command.*" """ set -euo pipefail @@ -179,20 +159,16 @@ process apply_TumorIndelFilter_SomaticSniper { process create_ReadCountPosition_SomaticSniper { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.SNPfilter.pos", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.SNPfilter.pos", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path snp_file output: path "*.SNPfilter.pos", emit: snp_positions - path ".command.*" script: """ @@ -207,10 +183,7 @@ process create_ReadCountPosition_SomaticSniper { // Recommend to use the same mapping quality -q setting as SomaticSniper process generate_ReadCount_bam_readcount { container params.docker_image_bam_readcount - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path reference @@ -221,7 +194,6 @@ process generate_ReadCount_bam_readcount { output: path "*.readcount", emit: readcount - path ".command.*" // tumor index file not explicitly passed to bam-readcount, // but it needs to be in the working directory otherwise bam-readcount will fail @@ -242,13 +214,10 @@ process generate_ReadCount_bam_readcount { process filter_FalsePositive_SomaticSniper { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "*.SNPfilter.*", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "*.SNPfilter.*", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path snp_file @@ -271,13 +240,10 @@ process filter_FalsePositive_SomaticSniper { process call_HighConfidenceSNV_SomaticSniper { container params.docker_image_somaticsniper publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - pattern: "*.vcf", - mode: "copy", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + pattern: "*.vcf", + mode: "copy", + enabled: params.save_intermediate_files + ext log_dir: { "SomaticSniper-${params.somaticsniper_version}/${task.process.split(':')[-1]}" } input: path fp_pass diff --git a/module/somaticsniper.nf b/module/somaticsniper.nf index 865b80c0..b17eebc1 100644 --- a/module/somaticsniper.nf +++ b/module/somaticsniper.nf @@ -1,9 +1,11 @@ include { call_sSNV_SomaticSniper; convert_BAM2Pileup_SAMtools; create_IndelCandidate_SAMtools; apply_NormalIndelFilter_SomaticSniper; apply_TumorIndelFilter_SomaticSniper; create_ReadCountPosition_SomaticSniper; generate_ReadCount_bam_readcount; filter_FalsePositive_SomaticSniper; call_HighConfidenceSNV_SomaticSniper } from './somaticsniper-processes' -include { rename_samples_BCFtools; generate_sha512sum } from './common' +include { rename_samples_BCFtools; generate_sha512sum } from './common' addParams( + log_dir_prefix: "SomaticSniper-${params.somaticsniper_version}" + ) include { compress_index_VCF as compress_index_VCF_hc } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/SomaticSniper-${params.somaticsniper_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args, is_output_file: false @@ -11,13 +13,14 @@ include { compress_index_VCF as compress_index_VCF_hc } from '../external/pipeli include { compress_index_VCF as compress_index_VCF_fix } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/SomaticSniper-${params.somaticsniper_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args ]) include { compress_file_bzip2} from './common' addParams( compress_publishdir : "${params.workflow_output_dir}/intermediate/generate_ReadCount_bam_readcount", - compress_enabled : params.save_intermediate_files + compress_enabled : params.save_intermediate_files, + log_dir_prefix: "SomaticSniper-${params.somaticsniper_version}" ) workflow somaticsniper { diff --git a/module/strelka2-processes.nf b/module/strelka2-processes.nf index 427febfd..7d13bff7 100644 --- a/module/strelka2-processes.nf +++ b/module/strelka2-processes.nf @@ -13,13 +13,10 @@ Strelka2 Options: process call_sIndel_Manta { container params.docker_image_manta publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "MantaWorkflow", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "MantaWorkflow", + enabled: params.save_intermediate_files + ext log_dir: { "Strelka2-${params.strelka2_version}/${task.process.split(':')[-1]}" } input: path tumor @@ -35,7 +32,6 @@ process call_sIndel_Manta { tuple path("MantaWorkflow/results/variants/candidateSmallIndels.vcf.gz"), path("MantaWorkflow/results/variants/candidateSmallIndels.vcf.gz.tbi") path "MantaWorkflow" - path ".command.*" script: exome = params.exome ? "--exome" : "" @@ -56,13 +52,10 @@ process call_sIndel_Manta { process call_sSNV_Strelka2 { container params.docker_image_strelka2 publishDir path: "${params.workflow_output_dir}/intermediate/${task.process.split(':')[-1]}", - mode: "copy", - pattern: "StrelkaSomaticWorkflow", - enabled: params.save_intermediate_files - publishDir path: "${params.workflow_log_output_dir}", - mode: "copy", - pattern: ".command.*", - saveAs: { "${task.process.split(':')[-1]}/log${file(it).getName()}" } + mode: "copy", + pattern: "StrelkaSomaticWorkflow", + enabled: params.save_intermediate_files + ext log_dir: { "Strelka2-${params.strelka2_version}/${task.process.split(':')[-1]}" } input: path tumor @@ -79,7 +72,6 @@ process call_sSNV_Strelka2 { tuple val("SNV"), path("StrelkaSomaticWorkflow/results/variants/somatic.snvs.vcf.gz"), emit: snvs_gzvcf tuple val("Indel"), path("StrelkaSomaticWorkflow/results/variants/somatic.indels.vcf.gz"), emit: indels_gzvcf path "StrelkaSomaticWorkflow" - path ".command.*" script: exome = params.exome ? "--exome" : "" diff --git a/module/strelka2.nf b/module/strelka2.nf index 9d4b7b60..8f8e9c66 100644 --- a/module/strelka2.nf +++ b/module/strelka2.nf @@ -1,10 +1,12 @@ include { call_sSNV_Strelka2; call_sIndel_Manta } from './strelka2-processes' -include { filter_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' +include { filter_VCF_BCFtools; rename_samples_BCFtools; generate_sha512sum } from './common' addParams( + log_dir_prefix: "Strelka2-${params.strelka2_version}" + ) include { compress_index_VCF } from '../external/pipeline-Nextflow-module/modules/common/index_VCF_tabix/main.nf' addParams( options: [ output_dir: params.workflow_output_dir, - log_output_dir: params.workflow_log_output_dir, + log_output_dir: "${params.log_output_dir}/process-log/Strelka2-${params.strelka2_version}", bgzip_extra_args: params.bgzip_extra_args, tabix_extra_args: params.tabix_extra_args ]) From 5d0e1bafd105f4ffa1616594337e5466f42ca04c Mon Sep 17 00:00:00 2001 From: Sorel Fitz-Gibbon Date: Thu, 17 Oct 2024 17:33:10 -0700 Subject: [PATCH 2/3] update module submodule with log fix --- external/pipeline-Nextflow-module | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/pipeline-Nextflow-module b/external/pipeline-Nextflow-module index ef40c100..9f07b9fe 160000 --- a/external/pipeline-Nextflow-module +++ b/external/pipeline-Nextflow-module @@ -1 +1 @@ -Subproject commit ef40c100005466b7d44f2fd60adb2c60ef451c4f +Subproject commit 9f07b9fee6bcd742ef8f461b62782522bb69f65e From 2d7af58220788a5a405126f8a287c90a898f4838 Mon Sep 17 00:00:00 2001 From: nwiltsie Date: Mon, 28 Oct 2024 21:07:42 +0000 Subject: [PATCH 3/3] Autofix Nextflow configuration regression tests --- test/configtest-F16.json | 21 +++++++++++++++++++++ test/configtest-F32.json | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/test/configtest-F16.json b/test/configtest-F16.json index 81d8b4ea..b1be3b96 100644 --- a/test/configtest-F16.json +++ b/test/configtest-F16.json @@ -270,6 +270,12 @@ "work_dir": "/scratch/8543" }, "process": { + "afterScript": { + "1": "", + "2": "", + "3": "", + "closure": "" + }, "cache": false, "commonRetryCodes": [ "104", @@ -298,6 +304,21 @@ "closure": "terminate" }, "executor": "local", + "ext": { + "capture_logs": true, + "commonAfterScript": { + "1": "", + "2": "", + "3": "", + "closure": "" + }, + "log_dir": { + "1": "ext", + "2": "ext", + "3": "ext", + "closure": "ext" + } + }, "maxRetries": "1", "withLabel:process_high": { "cpus": { diff --git a/test/configtest-F32.json b/test/configtest-F32.json index ffbd285b..b2351001 100644 --- a/test/configtest-F32.json +++ b/test/configtest-F32.json @@ -268,6 +268,12 @@ "work_dir": "/scratch/8543" }, "process": { + "afterScript": { + "1": "", + "2": "", + "3": "", + "closure": "" + }, "cache": false, "commonRetryCodes": [ "104", @@ -296,6 +302,21 @@ "closure": "terminate" }, "executor": "local", + "ext": { + "capture_logs": true, + "commonAfterScript": { + "1": "", + "2": "", + "3": "", + "closure": "" + }, + "log_dir": { + "1": "ext", + "2": "ext", + "3": "ext", + "closure": "ext" + } + }, "maxRetries": "1", "withLabel:process_high": { "cpus": {