Bugfixes with task hash and job limiter (#201)

* Fix handling of task hash for WGS jobs * Prevent job submission when query of slurm jobs fails * Update CHANGELOG * Bump version for release * Fix condition
uclahs-cds · May 16, 2024 · e3c5367 · e3c5367
1 parent f306875
commit e3c5367
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ---
 
+## [6.0.0-rc.4] - 2024-05-15
+### Fixed
++ Task hash properly handled between WGS and non-WGS modes
++ Job submission properly restricted when Slurm query fails
+
+---
+
 ## [6.0.0-rc.3] - 2024-05-13
 ### Added
 + More descriptive error messages on failure to identify output files

diff --git a/config/methods.config b/config/methods.config
@@ -121,11 +121,16 @@ methods {
                 hold_submission=true
                 while [ "\$hold_submission" = true ]
                 do
-                    running_metapipeline_jobs=`squeue --noheader --sort=-V --format="%j---%i---%T---%S---%u" | grep ^${job_name_prefix} || true`
+                    running_metapipeline_jobs=`squeue --noheader --sort=-V --format="%j---%i---%T---%S---%u" | grep ^${job_name_prefix} || echo "failed"`
                     if [ -z "\$running_metapipeline_jobs" ]
                     then
                         running_metapipeline_jobs_number=0
                         running_user_metapipeline_jobs_number=0
+                    elif [[ "\$running_metapipeline_jobs" == "failed" ]]
+                    then
+                        echo "squeue command failed, setting to limits to prevent submission"
+                        running_metapipeline_jobs_number=`get_global_limit`
+                        running_user_metapipeline_jobs_number=`get_user_limit`
                     else
                         running_metapipeline_jobs_number=`echo "\$running_metapipeline_jobs" | wc -l`
                         running_user_metapipeline_jobs_number=\$(echo "\$running_metapipeline_jobs" | awk -F"---" {'print \$5'} | grep "`whoami`" | wc -l)

diff --git a/main.nf b/main.nf
@@ -193,8 +193,6 @@ process call_metapipeline_DNA {
     : "\${CURRENT_WORK_DIR:=`pwd`}"
     : "\${SBATCH_RET:=-1}"
 
-    task_hash=\$(pwd | rev | cut -d '/' -f 1,2 | rev | sed 's/\\//_/')
-
     NXF_WORK=${params.resolved_work_dir} \
     nextflow run \
         ${moduleDir}/module/metapipeline_DNA.nf \
@@ -215,7 +213,7 @@ process call_metapipeline_DNA {
         --normal_sample_count ${params.sample_counts[patient]['normal']} \
         --tumor_sample_count ${params.sample_counts[patient]['tumor']} \
         --use_original_intervals ${params.use_original_intervals} \
-        --task_hash \${task_hash} \
+        --task_hash \$(pwd | rev | cut -d '/' -f 1,2 | rev | sed 's/\\//_/') \
         -params-file ${pipeline_params_json} \
         -c ${moduleDir}/config/metapipeline_DNA_base.config
     """ + limiter_wrapper_post

diff --git a/nextflow.config b/nextflow.config
@@ -3,5 +3,5 @@ manifest {
     name = 'metapipeline-DNA'
     author = ['Yash Patel', 'Chenghao Zhu', 'Helena Winata', 'Alfredo Enrique Gonzalez', 'Nicholas Wang', 'Sorel Fitz-Gibbon', 'Mohammed Faizal Eeman Mootor', 'Nicole Zeltser']
     description = 'Nextflow pipeline to convert BAM to FASTQ, align, call gSNP, call sSNV, call gSV, call sSV and call mtSNV'
-    version = '6.0.0-rc.3'
+    version = '6.0.0-rc.4'
 }