diff --git a/env/GAEA.env b/env/GAEA.env index 7736e0f1ea..90e7b553fa 100755 --- a/env/GAEA.env +++ b/env/GAEA.env @@ -15,8 +15,11 @@ export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 -ulimit -s unlimited -ulimit -a +# Setting stacksize to unlimited on login nodes is prohibited +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + ulimit -s unlimited + ulimit -a +fi # Calculate common variables # Check first if the dependent variables are set @@ -71,7 +74,28 @@ elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" -elif [[ "${step}" = "fcst" ]]; then +elif [[ "${step}" = "eobs" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "eupd" ]]; then + + export NTHREADS_ENKF=${NTHREADSmax} + export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) @@ -93,6 +117,24 @@ elif [[ "${step}" = "oceanice_products" ]]; then export NTHREADS_OCNICEPOST=${NTHREADS1} export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" +elif [[ "${step}" = "ecen" ]]; then + + export NTHREADS_ECEN=${NTHREADSmax} + export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}" + + export NTHREADS_CHGRES=${threads_per_task_chgres:-12} + [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} + export APRUN_CHGRES="time" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}" + +elif [[ "${step}" = "epos" ]]; then + + export NTHREADS_EPOS=${NTHREADSmax} + export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}" + elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} diff --git a/parm/config/gfs/config.resources.GAEA b/parm/config/gfs/config.resources.GAEA index c50601da00..79dd593c72 100644 --- a/parm/config/gfs/config.resources.GAEA +++ b/parm/config/gfs/config.resources.GAEA @@ -4,7 +4,6 @@ case ${step} in "prep") - # Run on two nodes (requires ~400GB total) tasks_per_node=7 ;; @@ -21,6 +20,65 @@ case ${step} in esac ;; + "eupd") + # update ntasks to 80 and threads_per_task to 20 + case ${CASE} in + "C768") + export ntasks=80 + export threads_per_task=20 + ;; + *) + ;; + esac + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "analcalc") + # decrease tasks_per_node 127 to 64 + case ${CASE} in + "C768") + export tasks_per_node=64 + ;; + *) + ;; + esac + ;; + + "upp") + # decrease tasks_per_node 120 to 60 + case ${CASE} in + "C768") + export tasks_per_node=60 + ;; + *) + ;; + esac + ;; + + "fcst") + # increase WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_{GDAS,GFS} + case ${CASE} in + "C768") + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=20 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=25 + (( WRTTASK_PER_GROUP_PER_THREAD_GDAS = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS * 6 )) + (( WRTTASK_PER_GROUP_PER_THREAD_GFS = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS * 6 )) + export WRTTASK_PER_GROUP_PER_THREAD_GDAS + export WRTTASK_PER_GROUP_PER_THREAD_GFS + (( ntasks_quilt_gdas = WRITE_GROUP_GDAS * WRTTASK_PER_GROUP_PER_THREAD_GDAS )) + (( ntasks_quilt_gfs = WRITE_GROUP_GFS * WRTTASK_PER_GROUP_PER_THREAD_GFS )) + export ntasks_quilt_gdas + export ntasks_quilt_gfs + if [[ "${gaea_sourced_resources:-false}" == false ]]; then + export gaea_sourced_resources=true + source "${EXPDIR}/config.resources" "${step}" + fi + ;; + *) + ;; + esac + ;; + *) ;;