feat: Add some useful debug comments

celery_worker_scripts/dev/START_CELERY_WORKERS_SLURM.sh

@@ -38,6 +38,7 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL}
 # are not sent to the batch step (the shell script). With this 
 # option scancel also signals the batch script and its children processes."
 # See https://slurm.schedmd.com/scancel.html#OPT_full
+echo "sending SIGTERM signal to dev celery workers"
 scancel --name=sumstats_service_celery_worker --signal=TERM --full
 
 echo "START spinning up dev celery workers:"

celery_worker_scripts/dev/start_celery_worker_slurm.sh

@@ -21,9 +21,14 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL}
 # are not sent to the batch step (the shell script). With this 
 # option scancel also signals the batch script and its children processes."
 # See https://slurm.schedmd.com/scancel.html#OPT_full
+echo "sending SIGTERM signal to dev celery workers"
 scancel --name=sumstats_service_celery_worker --signal=TERM --full
 
 # Submit new SLURM jobs for celery workers
+echo "START spinning up dev celery workers:"
 for WORKER_ID in {1..2}; do
+    echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
+echo "DONE spinning up dev celery workers"
+

celery_worker_scripts/prod/START_CELERY_WORKERS_SLURM.sh

@@ -44,7 +44,7 @@ singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwa
 # Submit new SLURM jobs for HH celery workers
 echo "spinning up HH celery workers:"
 for WORKER_ID in {4..6}; do
-	echo $WORKER_ID
+    echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
 echo "DONE spinning up HH celery workers"

celery_worker_scripts/prod/start_celery_worker_slurm.sh

@@ -21,18 +21,25 @@ celery_cmd="celery -A sumstats_service.app.celery worker --loglevel=${LOG_LEVEL}
 # are not sent to the batch step (the shell script). With this 
 # option scancel also signals the batch script and its children processes."
 # See https://slurm.schedmd.com/scancel.html#OPT_full
+echo "sending SIGTERM signal to prod celery workers"
 scancel --name=sumstats_service_celery_worker_prod --signal=TERM --full
 
 # Submit new SLURM jobs for HX celery workers
+echo "START spinning up HX celery workers:"
 for WORKER_ID in {1..3}; do
+    echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
+echo "DONE spinning up HX celery workers"
 
 # spin up the workers for hl rabbitmq
 ENV_FILE="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/cel_envs_hh"
 singularity_cmd="singularity exec --env-file $ENV_FILE $SINGULARITY_CACHEDIR/gwas-sumstats-service_latest.sif"
 
 # Submit new SLURM jobs for HH celery workers
+echo "START spinning up HH celery workers:"
 for WORKER_ID in {4..6}; do
+    echo $WORKER_ID
     sbatch --parsable --output="cel_${WORKER_ID}.o" --error="cel_${WORKER_ID}.e" --mem=${MEM} --time=7-00:00:00 --job-name=sumstats_service_celery_worker_prod --wrap="${lmod_cmd}; ${singularity_cmd} ${celery_cmd}"
 done
+echo "DONE spinning up HH celery workers"

celery_worker_scripts/scrontab

@@ -27,4 +27,4 @@
 0 10 */3 * * sbatch --mem=1200M --time=00:30:00 -o /hps/software/users/parkinso/spot/gwas/dev/scripts/cron/sumstats_service/deploy.out -e /hps/software/users/parkinso/spot/gwas/dev/scripts/cron/sumstats_service/deploy.err  --wrap="/hps/software/users/parkinso/spot/gwas/dev/scripts/cron/sumstats_service/start_celery_worker_slurm.sh"
 
 # Celery worker start for sumstats service (PROD):
-0 10 */3 * * sbatch --mem=1200M --time=02:00:00 -o /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.out -e /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.err  --wrap="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/start_celery_worker_slurm.sh"
+0 10 */3 * * sbatch --mem=1200M --time=00:30:00 -o /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.out -e /hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/deploy.err  --wrap="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/sumstats_service/start_celery_worker_slurm.sh"