diff --git a/backend_scripts/rha.slurm b/backend_scripts/rha.slurm new file mode 100644 index 0000000..65b61b1 --- /dev/null +++ b/backend_scripts/rha.slurm @@ -0,0 +1,41 @@ +#!/bin/bash + +#SBATCH -p gpu +#SBATCH --mem=8gb +#SBATCH --gres=gpu:1#!/bin/bash +#SBATCH -N 1 +#SBATCH -c 1 +#SBATCH --mem=8gb +#SBATCH --output=test.o%j +#SBATCH --partition=gpu +#SBATCH -C gpu2v100|gpu4v100 +#SBATCH --gres=gpu:1 # Request one gpu out of 2 (Max) + +#SBATCH --account=mbt8 # substitute appropriate group here + +module load singularity +module load CUDA/11.1.1-GCC-10.2.0 + +export PATH="/usr/local/software/singularity/3.10.4/bin:/usr/local/easybuild_allnodes/software/CUDAcore/11.1.1/nvvm/bin:/usr/local/easybuild_allnodes/software/CUDAcore/11.1.1/bin:/usr/local/easybuild_allnodes/software/binutils/2.35-GCCcore-10.2.0/bin:/usr/local/easybuild_allnodes/software/GCCcore/10.2.0/bin:/home/sxg1373/miniconda3/bin:/home/sxg1373/miniconda3/condabin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/dell/srvadmin/bin:/home/sxg1373/.local/bin:/home/sxg1373/bin" +echo "hello" +echo $PFSDIR +scp -r /scratch/users/sxg1373/input_files/$1 /scratch/users/sxg1373/1.jpg /home/sxg1373/rha_sing.sif $PFSDIR +ls "$PFSDIR" +final_output_dir=$6 +input_dir=$PFSDIR/$1 +mkdir $PFSDIR/output_files +output_dir=$PFSDIR/output_files +img_dir=$PFSDIR/1.jpg + +for file in "$input_dir"/* +do + echo "$file" + file_name=`echo "${file##*/}"` + echo "This is the file name $file_name" + singularity run --nv --bind "$PFSDIR" $PFSDIR/rha_sing.sif "$2" "audiovideo" $3 $4 $5 "$file" "$output_dir/output_$file_name.mp4" "$img_dir" +done +echo "saving files" + +scp -r $output_dir/* $final_output_dir +rm -rf "$PFSDIR"/* +rm -r /scratch/users/sxg1373/input_files/$1 diff --git a/backend_scripts/scheduler_script.sh b/backend_scripts/scheduler_script.sh new file mode 100755 index 0000000..2366cac --- /dev/null +++ b/backend_scripts/scheduler_script.sh @@ -0,0 +1,43 @@ +#!/bin/bash +#this script will setup the files in correct directories before scheduling the +#batch script + +#read files of all the directories in input directory one by one and transfer +#them to some input directory in scratch folder + +#set the parameters of anonymization according to the config.json file for batch +#batch job script + +module load Python/3.10.8-GCCcore-12.2.0-bare +export PATH="/usr/local/easybuild_allnodes/software/Python/3.10.8-GCCcore-12.2.0-bare/bin:/usr/local/easybuild_allnodes/software/OpenSSL/1.1/bin:/usr/local/easybuild_allnodes/software/XZ/5.2.7-GCCcore-12.2.0/bin:/usr/local/easybuild_allnodes/software/SQLite/3.39.4-GCCcore-12.2.0/bin:/usr/local/easybuild_allnodes/software/Tcl/8.6.12-GCCcore-12.2.0/bin:/usr/local/easybuild_allnodes/software/ncurses/6.3-GCCcore-12.2.0/bin:/usr/local/easybuild_allnodes/software/bzip2/1.0.8-GCCcore-12.2.0/bin:/usr/local/easybuild_allnodes/software/binutils/2.39-GCCcore-12.2.0/bin:/usr/local/easybuild_allnodes/software/GCCcore/12.2.0/bin:/usr/local/software/singularity/3.10.4/bin:/usr/local/easybuild_allnodes/software/CUDAcore/11.1.1/nvvm/bin:/usr/local/easybuild_allnodes/software/CUDAcore/11.1.1/bin:/home/sxg1373/miniconda3/bin:/home/sxg1373/miniconda3/condabin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/dell/srvadmin/bin:/home/sxg1373/.local/bin:/home/sxg1373/bin" +scratch_dir="/scratch/users/sxg1373/input_files" +for f in /home/sxg1373/sxg1373gallinahome/anonymizer_storage/input/*; do +if [ -d "$f" ]; then +echo $f +status=`grep -o '"status": "[^"]*' $f/config.json | grep -o '[^"]*$'` + +#python /home/sxg1373/change_status.py $f/config.json +#change the status to 1 by default the status is 0 +cp -r $f/ $scratch_dir/ + +ls $scratch_dir +project_name=`grep -o '"project_name": "[^"]*' $f/config.json | grep -o '[^"]*$'` +visual_anonymization=`grep -o '"visual_anonymization": "[^"]*' $scratch_dir/$project_name/config.json | grep -o '[^"]*$'` +pitch=`grep -o '"pitch": "[^"]*' $scratch_dir/$project_name/config.json | grep -o '[^"]*$'` +echo_=`grep -o '"echo": "[^"]*' $scratch_dir/$project_name/config.json | grep -o '[^"]*$'` +distortion=`grep -o '"distortion": "[^"]*' $scratch_dir/$project_name/config.json | grep -o '[^"]*$'` +rm $scratch_dir/$project_name/config.json +mkdir /home/sxg1373/sxg1373gallinahome/anonymizer_storage/output/$project_name +output_dir="sxg1373@hpc8:/home/sxg1373/sxg1373gallinahome/anonymizer_storage/output/$project_name" +echo $project_name $visual_anonymization $pitch $echo_ $distortion $output_dir +#call the batch script and pass the parameters to it +job_id_str=`sbatch rha.slurm $project_name $visual_anonymization $pitch $echo_ $distortion $output_dir` +echo "batch job scheduled $job_id_str" +rm -r $f +echo "removed file from anonymizer_storage" + +#else +#echo "No files found" +fi + +done