Improving configs for OpenMS dev and DIANN versions over 1.8.1

.github/workflows/ci.yml

@@ -48,8 +48,6 @@ jobs:
           - NXF_VER: "latest-everything"
             exec_profile: "conda"
         include:
-          - test_profile: test_latest_dia
-            exec_profile: "singularity"
           - test_profile: test_lfq
             exec_profile: "conda"
           - test_profile: test_dda_id

.github/workflows/diann_private.yml

@@ -0,0 +1,86 @@
+name: Fork-specific CI for test_latest_dia
+
+# Trigger only for your fork and the 'test_latest_dia' test profile
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    branches:
+      - dev
+    paths:
+      - "**.yml"
+
+env:
+  NXF_ANSI_LOG: false
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Run test_latest_dia with Docker
+    runs-on: ubuntu-latest
+    env:
+      NXF_ANSI_LOG: false
+      CAPSULE_LOG: none
+      TEST_PROFILE: test_latest_dia
+      EXEC_PROFILE: docker
+
+    # Ensure this workflow only runs for your fork
+    if: ${{ github.repository == 'ypriverol/quantms' }}
+
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v4
+
+      - name: Set up Nextflow
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: "24.04.2" # Or the Nextflow version you prefer
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Run test_latest_dia with Docker
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,$EXEC_PROFILE --outdir ${TEST_PROFILE}_${EXEC_PROFILE}_results
+
+      - name: Gather failed logs
+        if: failure() || cancelled()
+        run: |
+          mkdir failed_logs
+          failed=$(grep "FAILED" ${TEST_PROFILE}_${EXEC_PROFILE}_results/pipeline_info/execution_trace.txt | cut -f 2)
+          while read -r line ; do cp $(ls work/${line}*/*.log) failed_logs/ | true ; done <<< "$failed"
+
+      - uses: actions/upload-artifact@v4
+        if: failure() || cancelled()
+        name: Upload failed logs
+        with:
+          name: failed_logs_${{ matrix.test_profile }}_${{ matrix.exec_profile }}
+          include-hidden-files: true
+          path: failed_logs
+          overwrite: false
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        name: Upload results
+        with:
+          name: ${{ matrix.test_profile }}_${{ matrix.exec_profile }}_results
+          include-hidden-files: true
+          path: ${{ matrix.test_profile }}_${{ matrix.exec_profile }}_results
+          overwrite: false
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        name: Upload log
+        with:
+          name: nextflow_${{ matrix.test_profile }}_${{ matrix.exec_profile }}.log
+          include-hidden-files: true
+          path: .nextflow.log
+          overwrite: false

conf/dev.config

@@ -18,7 +18,7 @@ params {
 
 process {
     withLabel: openms {
-        conda = "openms::openms-thirdparty=3.2.0"
+        conda = "bioconda::openms-thirdparty=3.2.0" // The conda package is not the nightly version
         container = {"${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'ghcr.io/openms/openms-executables-sif:latest' : 'ghcr.io/openms/openms-executables:latest' }"}
     }
 }

conf/pride_codon_slurm.config

@@ -0,0 +1,60 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Nextflow config file for EMBL-EBI Codon Cluster for the SLURM login nodes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Author: Yasset Perez-Riverol
+Mail: yperez@ebi.ac.uk
+URL: https://www.ebi.ac.uk/
+Basedon: https://github.com/nf-core/configs/blob/master/conf/ebi_codon_slurm.config
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+*/
+
+params {
+    config_profile_contact     = "Yasset Perez-Riverol"
+    config_profile_description = "The European Bioinformatics Institute HPC cluster (codon) profile for the SLURM login nodes"
+    config_profile_url         = "https://www.ebi.ac.uk/"
+}
+
+singularity {
+    enabled     = true
+    // the default is 20 minutes and fails with large images
+    pullTimeout = "3 hours"
+    autoMounts = false
+    runOptions = '-B /hps/nobackup/juan/pride/reanalysis:/hps/nobackup/juan/pride/reanalysis'
+    cacheDir = "/hps/nobackup/juan/pride/reanalysis/singularity/"
+}
+
+process {
+    // this is to avoid errors for missing files due to shared filesystem latency
+    maxRetries    = 30
+    errorStrategy = { task.exitStatus == 0 ? "retry" : "terminate" }
+    cache         = "lenient"
+    afterScript   = "sleep 60"
+
+    withName:PROTEOMICSLFQ{
+        memory = {(mzmls as List).size() < 200 ? 72.GB * task.attempt : 250.GB * task.attempt }
+        cpus   = {(mzmls as List).size() < 200 ? 12 * task.attempt : 24 * task.attempt }
+    }
+
+    withName:ASSEMBLE_EMPIRICAL_LIBRARY{
+        memory = {(ms_files as List).size() < 200 ? 72.GB * task.attempt : 250.GB * task.attempt}
+        cpus   = {(ms_files as List).size() < 200 ? 12 * task.attempt : 24 * task.attempt }
+    }
+
+    withLabel: diann {
+        container = '/hps/nobackup/juan/pride/reanalysis/singularity/ghcr.io-bigbio-diann-1.9.2.sif'
+    }
+}
+
+executor {
+    name            = "slurm"
+    queueSize       = 2000
+    submitRateLimit = "10/1sec"
+    exitReadTimeout = "30 min"
+    jobName         = {
+        task.name
+            .replace("[", "(")
+            .replace("]", ")")
+            .replace(" ", "_")
+    }
+}

conf/test_latest_dia.config

@@ -10,14 +10,6 @@
 ------------------------------------------------------------------------------------------------
 */
 
-process {
-    resourceLimits = [
-        cpus: 4,
-        memory: '12.GB',
-        time: '48.h'
-    ]
-}
-
 params {
     config_profile_name        = 'Test profile for latest DIA'
     config_profile_description = 'Minimal test dataset to check pipeline function for the data-independent acquisition pipeline branch for latest DIA-NN.'
@@ -27,7 +19,6 @@ params {
     // Input data
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/quantms/testdata/dia_ci/PXD026600.sdrf.tsv'
     database = 'https://raw.githubusercontent.com/nf-core/test-datasets/quantms/testdata/dia_ci/REF_EColi_K12_UPS1_combined.fasta'
-    diann_version = '1.9.beta.1'
     min_pr_mz = 350
     max_pr_mz = 950
     min_fr_mz = 500
@@ -47,5 +38,19 @@ process {
     withName: 'NFCORE_QUANTMS:QUANTMS:FILE_PREPARATION:THERMORAWFILEPARSER' {
         publishDir  = [path: { "${params.outdir}/${task.process.tokenize(':')[-1].toLowerCase()}" }, pattern: "*.log" ]
     }
+
+    withLabel: diann {
+        container = 'ghcr.io/bigbio/diann:1.9.2' // This docker container is private in for quantms
+    }
+
+    resourceLimits = [
+        cpus: 4,
+        memory: '12.GB',
+        time: '48.h'
+    ]
+
 }
 
+singularity.enabled = false // Force to use docker
+docker.enabled = true
+

modules/local/add_sage_feat/main.nf

@@ -2,10 +2,10 @@ process SAGEFEATURE {
     tag "$meta.mzml_id"
     label 'process_low'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
     input:
     tuple val(meta), path(id_file), path(extra_feat)

modules/local/assemble_empirical_library/main.nf

@@ -1,15 +1,12 @@
 process ASSEMBLE_EMPIRICAL_LIBRARY {
     tag "$meta.experiment_id"
     label 'process_low'
+    label 'diann'
 
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' :
         'docker.io/biocontainers/diann:v1.8.1_cv1' }"
 
-    if (params.diann_version == "1.9.beta.1") {
-        container 'https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/tools/ghcr.io-bigbio-diann-1.9.1dev.sif'
-    }
-
     input:
     // In this step the real files are passed, and not the names
     path(ms_files)
@@ -18,7 +15,7 @@ process ASSEMBLE_EMPIRICAL_LIBRARY {
     path(lib)
 
     output:
-    path "empirical_library.tsv", emit: empirical_library
+    path "empirical_library.*", emit: empirical_library
     path "assemble_empirical_library.log", emit: log
     path "versions.yml", emit: versions
 
@@ -48,7 +45,7 @@ process ASSEMBLE_EMPIRICAL_LIBRARY {
     diann   --f ${(ms_files as List).join(' --f ')} \\
             --lib ${lib} \\
             --threads ${task.cpus} \\
-            --out-lib empirical_library.tsv \\
+            --out-lib empirical_library \\
             --verbose $params.diann_debug \\
             --rt-profiling \\
             --temp ./quant/ \\

modules/local/diann_preliminary_analysis/main.nf

@@ -1,15 +1,12 @@
 process DIANN_PRELIMINARY_ANALYSIS {
     tag "$ms_file.baseName"
     label 'process_high'
+    label 'diann'
 
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' :
         'docker.io/biocontainers/diann:v1.8.1_cv1' }"
 
-    if (params.diann_version == "1.9.beta.1") {
-        container 'https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/tools/ghcr.io-bigbio-diann-1.9.1dev.sif'
-    }
-
     input:
     tuple val(meta), path(ms_file), path(predict_library)
 

modules/local/diannconvert/main.nf

@@ -2,10 +2,10 @@ process DIANNCONVERT {
     tag "$meta.experiment_id"
     label 'process_medium'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
     input:
     path(report)

modules/local/diannsummary/main.nf

@@ -1,15 +1,12 @@
 process DIANNSUMMARY {
     tag "$meta.experiment_id"
     label 'process_high'
+    label 'diann'
 
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' :
         'docker.io/biocontainers/diann:v1.8.1_cv1' }"
 
-    if (params.diann_version == "1.9.beta.1") {
-        container 'https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/tools/ghcr.io-bigbio-diann-1.9.1dev.sif'
-    }
-
     input:
     // Note that the files are passed as names and not paths, this prevents them from being staged
     // in the directory

modules/local/extract_psm/main.nf

@@ -2,10 +2,10 @@ process PSMCONVERSION {
     tag "$meta.mzml_id"
     label 'process_medium'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
 
     input:

modules/local/extract_sample/main.nf

@@ -2,10 +2,10 @@ process GETSAMPLE {
     tag "$design.Name"
     label 'process_low'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
 
     input:

modules/local/generate_diann_cfg/main.nf

@@ -2,10 +2,10 @@ process GENERATE_DIANN_CFG {
     tag "$meta.experiment_id"
     label 'process_low'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
     input:
     val(meta)

modules/local/individual_final_analysis/main.nf

@@ -1,15 +1,12 @@
 process INDIVIDUAL_FINAL_ANALYSIS {
     tag "$ms_file.baseName"
     label 'process_high'
+    label 'diann'
 
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' :
         'docker.io/biocontainers/diann:v1.8.1_cv1' }"
 
-    if (params.diann_version == "1.9.beta.1") {
-        container 'https://ftp.pride.ebi.ac.uk/pub/databases/pride/resources/tools/ghcr.io-bigbio-diann-1.9.1dev.sif'
-    }
-
     input:
     tuple val(meta), path(ms_file), path(fasta), path(diann_log), path(library)
 

modules/local/ms2rescore/main.nf

@@ -2,10 +2,10 @@ process MS2RESCORE {
     tag "$meta.mzml_id"
     label 'process_high'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
     // userEmulation settings when docker is specified
     containerOptions = (workflow.containerEngine == 'docker') ? '-u $(id -u) -e "HOME=${HOME}" -v /etc/passwd:/etc/passwd:ro -v /etc/shadow:/etc/shadow:ro -v /etc/group:/etc/group:ro -v $HOME:$HOME' : ''

modules/local/mzmlstatistics/main.nf

@@ -3,10 +3,10 @@ process MZMLSTATISTICS {
     label 'process_very_low'
     label 'process_single'
 
-    conda "bioconda::quantms-utils=0.0.11"
+    conda "bioconda::quantms-utils=0.0.15"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.11--pyhdfd78af_0' :
-        'biocontainers/quantms-utils:0.0.11--pyhdfd78af_0' }"
+        'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.15--pyhdfd78af_0' :
+        'biocontainers/quantms-utils:0.0.15--pyhdfd78af_0' }"
 
     input:
     tuple val(meta), path(ms_file)