[develop] Update SRW with spack-stack version 1.5.0 (from 1.4.1) (#969)

* Update SRW with spack-stack 1.5.0
* Remove Gaea-C4 (gaea) from SRW App
* Update run_vx.local.lua on Orion to allow verifications tests to run successfully
* Temporarily comment out Gaea-C5 (gaeac5) from SRW App

---------

Co-authored-by: Parallel Works app-run user <Natalie.Perlin@mgmt-natalieperlin-gclusternoaav2usc1-00070.us-central1-c.c.noaa-users.internal>
Co-authored-by: Natalie Perlin <Natalie.Perlin@noaa.gov>
Co-authored-by: michael.lueken <Michael.Lueken@noaa.gov>
Co-authored-by: Natalie Perlin <perlin.natalie@gmail.com>

.cicd/Jenkinsfile

@@ -10,15 +10,14 @@ pipeline {
     parameters {
         // Allow job runner to filter based on platform
         // Use the line below to enable all PW clusters
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaeac5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
         // Use the line below to enable the PW AWS cluster
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaeac5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaeac5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
-        // Uncomment the following line to re-enable comprehensive tests
-        // booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
+        booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
     }
 
     stages {
@@ -88,8 +87,8 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        // values 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
-                        values 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        // values 'derecho', 'gaeac5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        values 'derecho', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -103,7 +102,8 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'gaea', 'gaea-c5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            // values 'derecho', 'gaeac5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            values 'derecho', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {
@@ -184,11 +184,6 @@ pipeline {
                             dir ("${env.SRW_PLATFORM}") {
                             echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})"
 
-                            // Remove the following line to re-enable comprehensive tests
-                            sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"'
-
-                            // Uncomment the following block to re-enable comprehensive tests
-                            /*
                             // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set,
                             // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter
                             script {
@@ -205,7 +200,6 @@ pipeline {
 
                                 sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"'
                             }
-                            */
                             }
                         }
 

.cicd/scripts/srw_ftest.sh

@@ -85,7 +85,12 @@ module load build_${platform,,}_${SRW_COMPILER}
 module load wflow_${platform,,}
 
 [[ ${FORGIVE_CONDA} == true ]] && set +e +u    # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests
-conda activate srw_app
+# Gaea-C5 special case missing jinja2
+if [ "${platform}" == "gaea-c5" ]; then
+  conda activate workflow_tools
+else
+  conda activate srw_app
+fi
 set -e -u
 
 # Adjust for strict limitation of stack size 

.cicd/scripts/wrapper_srw_ftest.sh

@@ -23,11 +23,6 @@ else
 fi
 
 # Customize wrapper scripts
-if [[ "${SRW_PLATFORM}" == gaea ]]; then
-    sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
-    sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
-fi
-
 if [[ "${SRW_PLATFORM}" == gaea-c5 ]]; then
     sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
     sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh

devbuild.sh

@@ -212,6 +212,11 @@ printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2
 if [ "${PLATFORM}" = "wcoss2" ]; then
     BUILD_CONDA="off"
 fi
+# Conda is not used on Gaea-c5 F2 filesystem 
+# it needs to be reevaluated when moved to F2 filesystem
+if [ "${PLATFORM}" = "gaea-c5" ]; then
+    BUILD_CONDA="off"
+fi
 
 # build conda and conda environments, if requested.
 if [ "${BUILD_CONDA}" = "on" ] ; then
@@ -244,8 +249,10 @@ if [ "${BUILD_CONDA}" = "on" ] ; then
   fi
 
 else
-  source ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh
-  conda activate
+  if [ -d "${CONDA_BUILD_DIR}" ] ; then
+    source ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh
+    conda activate
+  fi
 fi
 
 # Conda environment should have linux utilities to perform these tasks on macos.
@@ -281,7 +288,7 @@ set -eu
 # automatically determine compiler
 if [ -z "${COMPILER}" ] ; then
   case ${PLATFORM} in
-    jet|hera|gaea|gaea-c5) COMPILER=intel ;;
+    jet|hera|gaea-c5) COMPILER=intel ;;
     orion) COMPILER=intel ;;
     wcoss2) COMPILER=intel ;;
     cheyenne) COMPILER=intel ;;

devclean.sh

@@ -135,12 +135,13 @@ elif [ "${CLEAN}" = true ]; then
     "${INSTALL_DIR}/include" \
     "${INSTALL_DIR}/lib" \
     "${INSTALL_DIR}/lib64" \
-    "${SRW_DIR}/manage_externals/manic" \
   )
-  for directory in $directories; do
-    [[ -d $directory ]] && rm -rfv $directory
-  done
+  if [ ${#directories[@]} -ge 1 ]; then
+    for dir in ${directories[@]}; do 
+     [[ -d "${dir}" ]] &&  rm -rfv ${dir} 
+    done
   echo " "
+  fi
 fi
 # Clean all the submodules if requested. Note: Need to check out them again before attempting subsequent builds, by sourcing ${SRW_DIR}/manage_externals/checkout_externals
 if [ ${INCLUDE_SUB_MODULES} == true ]; then

etc/lmod-setup.csh

@@ -37,14 +37,11 @@ else if ( "$L_MACHINE" == singularity ) then
 
    module purge
 
-else if ( "$L_MACHINE" == gaea ) then
-   source /lustre/f2/dev/role.epic/contrib/Lmod_init.csh
-
 else if ( "$L_MACHINE" == gaea-c5 ) then
-   source /lustre/f2/dev/role.epic/contrib/Lmod_init_C5.csh
+   module reset
 
 else if ( "$L_MACHINE" == derecho ) then
-   module reset
+   module purge
 
 else if ( "$L_MACHINE" == odin ) then
    module unload modules

etc/lmod-setup.sh

@@ -44,14 +44,11 @@ elif [ "$L_MACHINE" = singularity ]; then
 
    module purge
 
-elif [ "$L_MACHINE" = gaea ]; then
-   source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
-
 elif [ "$L_MACHINE" = gaea-c5 ]; then
-   source /lustre/f2/dev/role.epic/contrib/Lmod_init_C5.sh
+   module reset 
 
 elif [ "$L_MACHINE" = derecho ]; then
-   module reset
+   module purge
 
 elif [ "$L_MACHINE" = odin ]; then
    module unload modules

jobs/JREGIONAL_AQM_ICS

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_AQM_LBCS

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_BIAS_CORRECTION_O3

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_BIAS_CORRECTION_PM25

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_CHECK_POST_OUTPUT

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_FIRE_EMISSION

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_MAKE_GRID

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_MAKE_ICS

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_MAKE_LBCS

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_MAKE_OROG

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_MAKE_SFC_CLIMO

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_NEXUS_EMISSION

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_NEXUS_GFS_SFC

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_NEXUS_POST_SPLIT

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_PLOT_ALLVARS

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_POINT_SOURCE

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_POST_STAT_O3

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_POST_STAT_PM25

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_PRE_POST_STAT

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_FCST

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_MET_GENENSPROD_OR_ENSEMBLESTAT

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSMEAN

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENSPROB

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_MET_PB2NC_OBS

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_MET_PCPCOMBINE

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_POST

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_RUN_PRDGEN

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 #-----------------------------------------------------------------------

modulefiles/build_derecho_intel.lua

@@ -1,35 +1,18 @@
 help([[
 This module loads libraries for building the UFS SRW App on
-the CISL machine Derecho (Cray) using Intel-classic-2023.0.0
+the CISL machine Derecho (Cray) using Intel@2021.10.0
 ]])
 
-whatis([===[Loads libraries needed for building the UFS SRW App on Cheyenne ]===])
+whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===])
 
-load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3"))
-load(pathJoin("ncarenv", os.getenv("ncarenv_ver") or "23.06"))
-load(pathJoin("craype", os.getenv("craype_ver") or "2.7.20"))
-
-unload("netcdf")
-unload("hdf5")
-load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2023.0.0"))
-load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "8.1.25"))
-
-prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/derecho/hpc-stack/intel-classic-2023.0.0/modulefiles/stack")
-load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
-load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2023.0.0"))
-load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "8.1.25"))
+prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
-load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "1.0.0"))
-load(pathJoin("mkl", os.getenv("mkl_ver") or "2023.0.0"))
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.10.0"))
+load(pathJoin("stack-cray-mpich", os.getenv("stack_cray_mpich_ver") or "8.1.25"))
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3"))
 
 load("srw_common")
 
-setenv("CC","cc")
-setenv("FC","ftn")
-setenv("CXX","CC")
-
-setenv("CMAKE_C_COMPILER","cc")
-setenv("CMAKE_CXX_COMPILER","CC")
-setenv("CMAKE_Fortran_COMPILER","ftn")
 setenv("CMAKE_Platform","derecho.intel")