merge in main

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# Test CODEOWNERS
+# Default owner(s)
+*	@tyamaguchi-ucla @yashpatel6 @maotian06 @sorelfitzgibbon @uclahs-cds/nextflow-wg
+# Folder specific owner

CHANGELOG.md

@@ -6,9 +6,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [7.0.0-rc.2] - 2023-10-05
+
+### Added
+- Add .github/CODEOWNERS
+- Add check for MuSE or Mutect2 on F2 node
+
+### Changed
+- Resource allocations changed for F32 and F72
+- Update `MuSE` to `v2.0.3`
+- Reorder all VCFs before intersection
+- Move `filter_VCF_BCFtools` to `common.nf`
+- Fix blarchive compression log output directory
+- Delay readcount compression until original file is no longer needed
+
+## [7.0.0-rc.1] - 2023-08-28
+
+### Changed
+- Update plot-venn.R to work with all numbers of algorithms greater than two
+
 ### Added
 - Custom resource allocation updates through configuration parameters
-- Add compression of `SomaticSniper` `bam-readcount` QC output
+- Add assertions to `nftest`
+- Add compression of `SomaticSniper` `bam-readcount` output and move to `intermediate` directory
 - Add `ncbi_build` parameter
 - Add conversion of concatenated VCF to MAF
 - Add concatenation of consensus variants to one VCF
@@ -20,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add `split_VCF_bcftools` to `Mutect2` workflow, separating SNVs, MNVs and Indels
 
 ### Changed
+- Fix CPU allocation behavior with Docker
 - Remove redundant directories in Intersect log output directories
 - Change compression of intersect MAF file to bzip2
 - Update `README.md`

config/F32.config

@@ -8,19 +8,37 @@ process {
     }
     withName: call_sSNV_SomaticSniper {
         cpus = 1
-        memory = 10.GB
+        memory = 1.GB
+        retry_strategy {
+            memory {
+                strategy = 'add'
+                operand = 5.GB
+            }
+        }
     }
     withName: convert_BAM2Pileup_SAMtools {
         cpus = 1
-        memory = 10.GB
+        memory = 1.GB
+        retry_strategy {
+            memory {
+                strategy = 'add'
+                operand = 5.GB
+            }
+        }
     }
     withName: create_IndelCandidate_SAMtools {
         cpus = 1
-        memory = 10.GB
+        memory = 1.GB
+        retry_strategy {
+            memory {
+                strategy = 'add'
+                operand = 5.GB
+            }
+        }
     }
     withName: call_sIndel_Manta {
-        cpus = 32
-        memory = 15.GB
+        cpus = 8
+        memory = 6.GB
         retry_strategy {
             memory {
                 strategy = 'add'
@@ -29,8 +47,8 @@ process {
         }
     }
     withName: call_sSNV_Strelka2 {
-        cpus = 32
-        memory = 15.GB
+        cpus = 8
+        memory = 2.GB
         retry_strategy {
             memory {
                 strategy = 'add'
@@ -39,8 +57,8 @@ process {
         }
     }
     withName: call_sSNV_Mutect2 {
-        cpus = 2
-        memory = 5.GB
+        cpus = 1
+        memory = 3.GB
         retry_strategy {
             memory {
                 strategy = 'add'
@@ -59,7 +77,7 @@ process {
         }
     }
     withName: call_sSNV_MuSE {
-        cpus = 20
+        cpus = 12
         memory = 48.GB
         retry_strategy {
             memory {
@@ -69,7 +87,7 @@ process {
         }
     }
     withName: run_sump_MuSE {
-        cpus = 10
+        cpus = 12
         memory = 48.GB
         retry_strategy {
             memory {

config/F72.config

@@ -8,19 +8,37 @@ process {
     }
     withName: call_sSNV_SomaticSniper {
         cpus = 1
-        memory = 45.GB
+        memory = 1.GB
+        retry_strategy {
+            memory {
+                strategy = 'add'
+                operand = 5.GB
+            }
+        }
     }
     withName: convert_BAM2Pileup_SAMtools {
         cpus = 1
-        memory = 45.GB
+        memory = 1.GB
+        retry_strategy {
+            memory {
+                strategy = 'add'
+                operand = 5.GB
+            }
+        }
     }
     withName: create_IndelCandidate_SAMtools {
         cpus = 1
-        memory = 45.GB
+        memory = 1.GB
+        retry_strategy {
+            memory {
+                strategy = 'add'
+                operand = 5.GB
+            }
+        }
     }
     withName: call_sIndel_Manta {
-        cpus = 36
-        memory = 15.GB
+        cpus = 8
+        memory = 6.GB
         retry_strategy {
             memory {
                 strategy = 'add'
@@ -29,8 +47,8 @@ process {
         }
     }
     withName: call_sSNV_Strelka2 {
-        cpus = 36
-        memory = 10.GB
+        cpus = 8
+        memory = 2.GB
         retry_strategy {
             memory {
                 strategy = 'add'
@@ -39,12 +57,12 @@ process {
         }
     }
     withName: call_sSNV_Mutect2 {
-        cpus = 2
-        memory = 5.GB
+        cpus = 1
+        memory = 3.GB
         retry_strategy {
             memory {
                 strategy = 'add'
-                operand = 5.GB
+                operand = 3.GB
             }
         }
     }
@@ -59,7 +77,7 @@ process {
         }
     }
     withName: call_sSNV_MuSE {
-        cpus = 20
+        cpus = 12
         memory = 60.GB
         retry_strategy {
             memory {
@@ -69,8 +87,8 @@ process {
         }
     }
     withName: run_sump_MuSE {
-        cpus = 10
-        memory = 70.GB
+        cpus = 24
+        memory = 80.GB
         retry_strategy {
             memory {
                 strategy = 'add'

config/default.config

@@ -21,10 +21,10 @@ params {
     GATK_version = "4.4.0.0"
     somaticsniper_version = "1.0.5.0"
     bam_readcount_version = "0.8.0"
-    blarchive_version = "dev"
+    blarchive_version = "2.0.0"
     strelka2_version = "2.9.10"
     manta_version = "1.6.0"
-    MuSE_version = "2.0.2"
+    MuSE_version = "2.0.3"
     BCFtools_version = "1.17"
     call_ssnv_r_version = "dev"
     vcf2maf_version = "v1.6.18"

config/methods.config

@@ -195,5 +195,6 @@ methods {
         methods.set_output_directory()
         methods.set_pipeline_log()
         methods.check_valid_algorithms()
+        methods.setup_docker_cpus()
     }
 }

main.nf

@@ -37,6 +37,7 @@ log.info """\
         log_output_dir: ${params.log_output_dir}
 
     - option:
+        ucla_cds: ${params.ucla_cds}
         save_intermediate_files: ${params.save_intermediate_files}
         docker_container_registry: ${params.docker_container_registry}
         bgzip_extra_args = ${params.bgzip_extra_args}
@@ -46,6 +47,17 @@ log.info """\
         tumor_only_mode: ${params.tumor_only_mode}
 """
 
+if (params.max_cpus < 16 || params.max_memory < 30) {
+    if (params.algorithm.contains('muse') || params.algorithm.contains('mutect2')) {
+        error """\
+        ------------------------------------
+        ERROR: Insufficient resources: ${params.max_cpus} CPUs and ${params.max_memory} of memory.
+        ------------------------------------
+        To run Mutect2 or MuSE. this pipeline requires at least 16 CPUs and 32 GB of memory.
+        """
+        }
+    }
+
 include { 
     run_GetSampleName_Mutect2 as run_GetSampleName_Mutect2_normal
     run_GetSampleName_Mutect2 as run_GetSampleName_Mutect2_tumor 
@@ -120,10 +132,10 @@ Channel
         }
     .set { normal_input }
 
-    script_dir_ch = Channel.fromPath(
-        "$projectDir/r-scripts",
-        checkIfExists: true
-        )
+script_dir_ch = Channel.fromPath(
+    "$projectDir/r-scripts",
+    checkIfExists: true
+    )
 
 workflow {
     reference_ch = Channel.from(
@@ -162,10 +174,10 @@ workflow {
         }
 
     // Set empty channels so any unused tools don't cause failure at intersect step
-    Channel.empty().set { somaticsniper_vcf_ch }
-    Channel.empty().set { strelka2_vcf_ch }
-    Channel.empty().set { mutect2_vcf_ch }
-    Channel.empty().set { muse_vcf_ch }
+    Channel.empty().set { somaticsniper_gzvcf_ch }
+    Channel.empty().set { strelka2_gzvcf_ch }
+    Channel.empty().set { mutect2_gzvcf_ch }
+    Channel.empty().set { muse_gzvcf_ch }
 
     Channel.empty().set { somaticsniper_idx_ch }
     Channel.empty().set { strelka2_idx_ch }
@@ -181,7 +193,7 @@ workflow {
             run_GetSampleName_Mutect2_normal.out.name_ch,
             run_GetSampleName_Mutect2_tumor.out.name_ch
             )
-            somaticsniper.out.vcf.set { somaticsniper_vcf_ch }
+            somaticsniper.out.gzvcf.set { somaticsniper_gzvcf_ch }
             somaticsniper.out.idx.set { somaticsniper_idx_ch }
         }
     if ('strelka2' in params.algorithm) {
@@ -193,20 +205,9 @@ workflow {
             run_GetSampleName_Mutect2_normal.out.name_ch,
             run_GetSampleName_Mutect2_tumor.out.name_ch
             )
-            strelka2.out.vcf.set { strelka2_vcf_ch }
+            strelka2.out.gzvcf.set { strelka2_gzvcf_ch }
             strelka2.out.idx.set { strelka2_idx_ch }
         }
-    if ('mutect2' in params.algorithm) {
-        mutect2(
-            tumor_input.tumor_bam.collect(),
-            tumor_input.tumor_index.collect(),
-            normal_input.normal_bam.collect(),
-            normal_input.normal_index.collect(),
-            tumor_input.contamination_est.collect()
-            )
-            mutect2.out.vcf.set { mutect2_vcf_ch }
-            mutect2.out.idx.set { mutect2_idx_ch }
-        }
     if ('muse' in params.algorithm) {
         muse(
             tumor_input.tumor_bam,
@@ -216,30 +217,39 @@ workflow {
             run_GetSampleName_Mutect2_normal.out.name_ch,
             run_GetSampleName_Mutect2_tumor.out.name_ch
             )
-            muse.out.vcf.set { muse_vcf_ch }
+            muse.out.gzvcf.set { muse_gzvcf_ch }
             muse.out.idx.set { muse_idx_ch }
         }
+    if ('mutect2' in params.algorithm) {
+        mutect2(
+            tumor_input.tumor_bam.collect(),
+            tumor_input.tumor_index.collect(),
+            normal_input.normal_bam.collect(),
+            normal_input.normal_index.collect(),
+            tumor_input.contamination_est.collect()
+            )
+            mutect2.out.gzvcf.set { mutect2_gzvcf_ch }
+            mutect2.out.idx.set { mutect2_idx_ch }
+        }
 
     // Intersect all vcf files
     if (params.algorithm.size() > 1) {
-        tool_vcfs = (somaticsniper_vcf_ch
-            .mix(strelka2_vcf_ch)
-            .mix(mutect2_vcf_ch)
-            .mix(muse_vcf_ch))
+        tool_gzvcfs = (somaticsniper_gzvcf_ch
+            .mix(strelka2_gzvcf_ch)
+            .mix(mutect2_gzvcf_ch)
+            .mix(muse_gzvcf_ch))
             .collect()
-
         tool_indices = (somaticsniper_idx_ch
             .mix(strelka2_idx_ch)
             .mix(mutect2_idx_ch)
             .mix(muse_idx_ch))
             .collect()
-
         intersect(
-            tool_vcfs,
+            tool_gzvcfs,
             tool_indices,
             script_dir_ch,
-            run_GetSampleName_Mutect2_normal.out.name_ch,
-            run_GetSampleName_Mutect2_tumor.out.name_ch
+            run_GetSampleName_Mutect2_normal.out.name_ch.first(),
+            run_GetSampleName_Mutect2_tumor.out.name_ch.first()
             )
         }
     }

metadata.yaml

@@ -5,4 +5,4 @@ Maintainers: ['maotian@mednet.ucla.edu']
 Contributors: ['Mao Tian', 'Bugh Caden', 'Helena Winata', 'Yash Patel', 'Sorel Fitz-Gibbon']
 Languages: ['Docker', 'Nextflow']
 Dependencies: ['Docker', 'Nextflow']
-Tools: ['GATK 4.4.0.0', 'SomaticSniper v1.0.5.0', 'SAMtools v1.16.1', 'Strelka2 v2.9.10', 'Manta v1.6.0', 'MuSE v2.0.2', 'BCFtools v1.17', 'R v4.3.1', 'VennDiagram v1.7.3', 'vcf2maf v1.6.18']
+Tools: ['GATK 4.4.0.0', 'SomaticSniper v1.0.5.0', 'SAMtools v1.16.1', 'Strelka2 v2.9.10', 'Manta v1.6.0', 'MuSE v2.0.3', 'BCFtools v1.17', 'R v4.3.1', 'VennDiagram v1.7.3', 'vcf2maf v1.6.18']