Move machine file definitions of *_OBS_DIRs to TEST_*_OBS_DIR for WE2…

…E testing purposes, similar to TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS
ufs-community · Aug 4, 2023 · 23dc10f · 23dc10f
1 parent 3bc1125
commit 23dc10f
Show file tree

Hide file tree

Showing 10 changed files with 38 additions and 28 deletions.
diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py
@@ -188,7 +188,17 @@ def run_we2e_tests(homedir, args) -> None:
                                                                        config_defaults,"lbcs")
 
         if 'verification' in test_cfg:
-            logging.debug(test_cfg['verification'])
+            # This section checks if we are doing verification on a machine with staged verification
+            # obs. If so, and if the config file does not explicitly set the observation locations,
+            # fill these in with defaults from the machine files
+            obs_vars = ['CCPA_OBS_DIR','MRMS_OBS_DIR','NDAS_OBS_DIR']
+            if 'platform' not in test_cfg:
+                test_cfg['platform'] = {}
+            for obvar in obs_vars:
+                mach_path = machine_defaults['platform'].get('TEST_'+obvar)
+                if not test_cfg['platform'].get(obvar) and mach_path:
+                    logging.debug(f'Setting CCPA_OBS_DIR = {mach_path} from machine file')
+                    test_cfg['platform'][obvar] = mach_path
 
         if 'cpl_aqm_parm' in test_cfg:
             test_aqm_input_basedir = machine_defaults['platform']['TEST_AQM_INPUT_BASEDIR']

diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 36
   SCHED: pbspro
-  CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
   METPLUS_PATH: /glade/p/ral/jntp/MET/METplus/METplus-4.1.1
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /glade/p/ral/jntp/MET/MET_releases/10.1.1

diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 32
   SCHED: slurm
-  CCPA_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
   METPLUS_PATH: /usw/met/METplus/METplus-4.1.3
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /usw/met/10.1.2

diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 40
   SCHED: slurm
-  CCPA_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc
   METPLUS_PATH: /contrib/METplus/METplus-4.1.1
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /contrib/met/10.1.1

diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 24
   SCHED: slurm
-  CCPA_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc
   METPLUS_PATH: /contrib/met/METplus/METplus-4.1.1
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /contrib/met/10.1.1

diff --git a/ush/machine/linux.yaml b/ush/machine/linux.yaml
@@ -3,9 +3,9 @@ platform:
   NCORES_PER_NODE: 8
   TASKTHROTTLE: 1
   SCHED: slurm
-  CCPA_OBS_DIR: /home/username/DATA/UFS/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /home/username/DATA/UFS/obs_data/mrms/proc
-  NDAS_OBS_DIR: /home/username/DATA/UFS/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /Users/username/DATA/UFS/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc
   METPLUS_PATH: ""
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: ""

diff --git a/ush/machine/macos.yaml b/ush/machine/macos.yaml
@@ -3,9 +3,9 @@ platform:
   NCORES_PER_NODE: 8
   TASKTHROTTLE: 1
   SCHED: slurm
-  CCPA_OBS_DIR: /Users/username/DATA/UFS/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc
-  NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /Users/username/DATA/UFS/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc
   DOMAIN_PREGEN_BASEDIR: /Users/username/DATA/UFS/FV3LAM_pregen
   METPLUS_PATH: ""
   MET_BIN_EXEC: bin

diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 36
   SCHED: slurm
-  CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc
   METPLUS_PATH: /contrib/EPIC/spack-stack/envs/srw-develop-intel/install/intel/2021.3.0/metplus
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /contrib/EPIC/spack-stack/envs/srw-develop-intel/install/intel/2021.3.0/met

diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 40
   SCHED: slurm
-  CCPA_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ndas/proc
   METPLUS_PATH: /apps/contrib/MET/METplus/METplus-4.1.1
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /apps/contrib/MET/10.1.1

diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml
@@ -2,9 +2,9 @@ platform:
   WORKFLOW_MANAGER: rocoto
   NCORES_PER_NODE: 128
   SCHED: pbspro
-  CCPA_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc
-  MRMS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc
-  NDAS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc
+  TEST_CCPA_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc
   METPLUS_PATH: /apps/ops/para/libs/intel/19.1.3.304/metplus/4.1.1
   MET_BIN_EXEC: bin
   MET_INSTALL_DIR: /apps/ops/para/libs/intel/19.1.3.304/met/10.1.1