diff --git a/.gitignore b/.gitignore index 254b87cf..5fcfc6ce 100644 --- a/.gitignore +++ b/.gitignore @@ -488,6 +488,8 @@ AtmOcnTransferGridProto/fields_ocn_init_export.tile3.nc AtmOcnTransferGridProto/fields_ocn_init_export.tile4.nc AtmOcnTransferGridProto/fields_ocn_init_export.tile5.nc AtmOcnTransferGridProto/fields_ocn_init_export.tile6.nc +AtmOcnTransferGridProto/fort.100 +AtmOcnTransferGridProto/fort.200 AtmOcnTransferGridProto/ocn.mod AtmOcnTransferGridProto/ocn.o AtmOcnTransferLocStreamProto/PET0.ESMF_LogFile @@ -504,6 +506,12 @@ AtmOcnTransferLocStreamProto/esmApp.stdout AtmOcnTransferLocStreamProto/field_pmsl_init.nc AtmOcnTransferLocStreamProto/ocn.mod AtmOcnTransferLocStreamProto/ocn.o +AtmOcnTransferMeshProto/Atm-MeshIn_corners.vtk +AtmOcnTransferMeshProto/Atm-MeshOut_corners.vtk +AtmOcnTransferMeshProto/OCN-MeshIn.vtk +AtmOcnTransferMeshProto/OCN-MeshOut.vtk +AtmOcnTransferMeshProto/fort.100 +AtmOcnTransferMeshProto/fort.200 AtmOcnTransferMeshProto/Atm-MeshIn_corners.2.0.vtk AtmOcnTransferMeshProto/Atm-MeshIn_corners.2.1.vtk AtmOcnTransferMeshProto/Atm-MeshOut_corners.2.0.vtk @@ -686,6 +694,52 @@ ExternalDriverAPIProto/field_ocn_import_rsns.nc ExternalDriverAPIProto/field_ocn_import_sdhf.nc ExternalDriverAPIProto/ocn.mod ExternalDriverAPIProto/ocn.o +ExternalDriverAPIProto/field_atm_export_pmsl.nc +ExternalDriverAPIProto/field_atm_export_rsnl.nc +ExternalDriverAPIProto/field_atm_export_rsns.nc +ExternalDriverAPIProto/field_atm_export_sdhf.nc +ExternalDriverAPIProto/field_atm_import_precip.nc +ExternalDriverAPIProto/field_atm_import_sst.nc +ExternalDriverAPIProto/field_externalExportState_init_precipitation_flux.nc +ExternalDriverAPIProto/field_externalExportState_precipitation_flux.nc +ExternalDriverAPIProto/field_externalImportState_init_surface_net_downward_shortwave_flux.nc +ExternalDriverAPIProto/field_externalImportState_surface_net_downward_shortwave_flux.nc +ExternalDriverAPIWeakCplDAProto/PET0.ESMF_LogFile +ExternalDriverAPIWeakCplDAProto/PET1.ESMF_LogFile +ExternalDriverAPIWeakCplDAProto/PET2.ESMF_LogFile +ExternalDriverAPIWeakCplDAProto/PET3.ESMF_LogFile +ExternalDriverAPIWeakCplDAProto/atm.mod +ExternalDriverAPIWeakCplDAProto/atm.o +ExternalDriverAPIWeakCplDAProto/atmDA.o +ExternalDriverAPIWeakCplDAProto/atmda.mod +ExternalDriverAPIWeakCplDAProto/esm.mod +ExternalDriverAPIWeakCplDAProto/esm.o +ExternalDriverAPIWeakCplDAProto/externalApp +ExternalDriverAPIWeakCplDAProto/externalApp.o +ExternalDriverAPIWeakCplDAProto/externalApp.stdout +ExternalDriverAPIWeakCplDAProto/field_atm_export_pmsl.nc +ExternalDriverAPIWeakCplDAProto/field_atm_export_rsnl.nc +ExternalDriverAPIWeakCplDAProto/field_atm_export_rsns.nc +ExternalDriverAPIWeakCplDAProto/field_atm_export_sdhf.nc +ExternalDriverAPIWeakCplDAProto/field_atm_import_precip.nc +ExternalDriverAPIWeakCplDAProto/field_atm_import_sst.nc +ExternalDriverAPIWeakCplDAProto/field_fmNuopcTop_init_sea_surface_temperature.nc +ExternalDriverAPIWeakCplDAProto/field_fmNuopcTop_init_surface_net_downward_shortwave_flux.nc +ExternalDriverAPIWeakCplDAProto/field_fmNuopcTop_sea_surface_temperature.nc +ExternalDriverAPIWeakCplDAProto/field_fmNuopcTop_surface_net_downward_shortwave_flux.nc +ExternalDriverAPIWeakCplDAProto/field_ocn_export_sst.nc +ExternalDriverAPIWeakCplDAProto/field_ocn_import_pmsl.nc +ExternalDriverAPIWeakCplDAProto/field_ocn_import_rsnl.nc +ExternalDriverAPIWeakCplDAProto/field_ocn_import_rsns.nc +ExternalDriverAPIWeakCplDAProto/field_ocn_import_sdhf.nc +ExternalDriverAPIWeakCplDAProto/field_toNuopcTop_init_precipitation_flux.nc +ExternalDriverAPIWeakCplDAProto/field_toNuopcTop_precipitation_flux.nc +ExternalDriverAPIWeakCplDAProto/nuopc_da.mod +ExternalDriverAPIWeakCplDAProto/nuopc_da.o +ExternalDriverAPIWeakCplDAProto/ocn.mod +ExternalDriverAPIWeakCplDAProto/ocn.o +ExternalDriverAPIWeakCplDAProto/ocnDA.o +ExternalDriverAPIWeakCplDAProto/ocnda.mod GenericMediatorProto/PET0.ESMF_LogFile GenericMediatorProto/PET1.ESMF_LogFile GenericMediatorProto/PET2.ESMF_LogFile @@ -802,6 +856,17 @@ SingleModelOpenMPProto/mainApp.o SingleModelOpenMPProto/mainApp.stdout SingleModelOpenMPProto/model.mod SingleModelOpenMPProto/model.o +SingleModelOpenMPUnawareProto/PET0.ESMF_LogFile +SingleModelOpenMPUnawareProto/PET1.ESMF_LogFile +SingleModelOpenMPUnawareProto/PET2.ESMF_LogFile +SingleModelOpenMPUnawareProto/PET3.ESMF_LogFile +SingleModelOpenMPUnawareProto/driver.mod +SingleModelOpenMPUnawareProto/driver.o +SingleModelOpenMPUnawareProto/mainApp +SingleModelOpenMPUnawareProto/mainApp.o +SingleModelOpenMPUnawareProto/mainApp.stdout +SingleModelOpenMPUnawareProto/model.mod +SingleModelOpenMPUnawareProto/model.o SingleModelProto/PET0.ESMF_LogFile SingleModelProto/PET1.ESMF_LogFile SingleModelProto/PET2.ESMF_LogFile diff --git a/AsyncIOBlockingProto/Makefile b/AsyncIOBlockingProto/Makefile index 0ded8323..a3cda116 100644 --- a/AsyncIOBlockingProto/Makefile +++ b/AsyncIOBlockingProto/Makefile @@ -70,4 +70,4 @@ edit: nedit asyncIOApp.F90 asyncIODriver.F90 model.F90 io.F90 & run: - mpirun -np 4 ./asyncIOApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./asyncIOApp diff --git a/AsyncIONonblockingProto/Makefile b/AsyncIONonblockingProto/Makefile index e242c0bf..b82e6d13 100644 --- a/AsyncIONonblockingProto/Makefile +++ b/AsyncIONonblockingProto/Makefile @@ -70,4 +70,4 @@ edit: nedit asyncIOApp.F90 asyncIODriver.F90 model.F90 io.F90 conn.F90 & run: - mpirun -np 4 ./asyncIOApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./asyncIOApp diff --git a/AtmOcnConOptsProto/Makefile b/AtmOcnConOptsProto/Makefile index 9ab63796..26d5a182 100644 --- a/AtmOcnConOptsProto/Makefile +++ b/AtmOcnConOptsProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnConProto/Makefile b/AtmOcnConProto/Makefile index 1deea26a..798befaf 100644 --- a/AtmOcnConProto/Makefile +++ b/AtmOcnConProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 conn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnCplListProto/Makefile b/AtmOcnCplListProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnCplListProto/Makefile +++ b/AtmOcnCplListProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnCplListProto/esm.F90 b/AtmOcnCplListProto/esm.F90 index 0cfce629..964df9b7 100644 --- a/AtmOcnCplListProto/esm.F90 +++ b/AtmOcnCplListProto/esm.F90 @@ -267,8 +267,13 @@ subroutine ModifyCplLists(driver, rc) .or. trim(cplList(j))=="air_pressure_at_sea_level") then tempString = trim(cplList(j))//":REMAPMETHOD=redist" else +#ifdef ESMF_NETCDF tempString = trim(cplList(j))//":REMAPMETHOD=bilinear"//& ":SrcTermProcessing=1:DUMPWEIGHTS=true:TermOrder=SrcSeq" +#else + tempString = trim(cplList(j))//":REMAPMETHOD=bilinear"//& + ":SrcTermProcessing=1:TermOrder=SrcSeq" +#endif endif cplList(j) = trim(tempString) write (msg,*) "Modified: "//trim(cplList(j)) diff --git a/AtmOcnCplSetProto/Makefile b/AtmOcnCplSetProto/Makefile index 71375b5b..beb0f3dd 100644 --- a/AtmOcnCplSetProto/Makefile +++ b/AtmOcnCplSetProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnFDSynoProto/Makefile b/AtmOcnFDSynoProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnFDSynoProto/Makefile +++ b/AtmOcnFDSynoProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnIceSimpleImplicitProto/Makefile b/AtmOcnIceSimpleImplicitProto/Makefile index 295dd793..2e5bbb63 100644 --- a/AtmOcnIceSimpleImplicitProto/Makefile +++ b/AtmOcnIceSimpleImplicitProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 ice.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnImplicitProto/Makefile b/AtmOcnImplicitProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnImplicitProto/Makefile +++ b/AtmOcnImplicitProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnLndProto/Makefile b/AtmOcnLndProto/Makefile index 468c4824..963e1e3d 100644 --- a/AtmOcnLndProto/Makefile +++ b/AtmOcnLndProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 lnd.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnLogNoneProto/Makefile b/AtmOcnLogNoneProto/Makefile index 9ab63796..26d5a182 100644 --- a/AtmOcnLogNoneProto/Makefile +++ b/AtmOcnLogNoneProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMedIngestFromConfigProto/Makefile b/AtmOcnMedIngestFromConfigProto/Makefile index 47e0fc2e..e32259e6 100644 --- a/AtmOcnMedIngestFromConfigProto/Makefile +++ b/AtmOcnMedIngestFromConfigProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 med.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMedIngestFromInternalProto/Makefile b/AtmOcnMedIngestFromInternalProto/Makefile index 47e0fc2e..e32259e6 100644 --- a/AtmOcnMedIngestFromInternalProto/Makefile +++ b/AtmOcnMedIngestFromInternalProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 med.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMedIngestFromInternalProto/esm.F90 b/AtmOcnMedIngestFromInternalProto/esm.F90 index d0ac5807..6bb9cd69 100644 --- a/AtmOcnMedIngestFromInternalProto/esm.F90 +++ b/AtmOcnMedIngestFromInternalProto/esm.F90 @@ -94,8 +94,8 @@ subroutine SetModelServices(driver, rc) ! set up free format driver attributes attrFF = NUOPC_FreeFormatCreate(stringList=(/ & - "Verbosity = 1", & - "Profiling = 0" /), & + "Verbosity = high", & + "Profiling = 0 " /), & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -273,10 +273,16 @@ subroutine SetRunSequence(driver, rc) " MED ", & " MED -> ATM ", & " MED -> OCN ", & - " ATM ", & - " OCN ", & - " ATM -> MED ", & + " @@3600 ", & + " ATM ", & + " OCN ", & + " @1800 ", & + " ATM -> MED ", & + " @ ", & + " @@ ", & + " @@3600 ", & " OCN -> MED ", & + " @@ ", & " MED ", & " @ " /), & rc=rc) @@ -298,7 +304,7 @@ subroutine SetRunSequence(driver, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out -#if 0 +#if 1 ! Diagnostic output call NUOPC_DriverPrint(driver, orderflag=.true., rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & diff --git a/AtmOcnMedPetListProto/Makefile b/AtmOcnMedPetListProto/Makefile index 47e0fc2e..e32259e6 100644 --- a/AtmOcnMedPetListProto/Makefile +++ b/AtmOcnMedPetListProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 med.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMedPetListTimescalesProto/Makefile b/AtmOcnMedPetListTimescalesProto/Makefile index 47e0fc2e..e32259e6 100644 --- a/AtmOcnMedPetListTimescalesProto/Makefile +++ b/AtmOcnMedPetListTimescalesProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 med.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMedPetListTimescalesSplitFastProto/Makefile b/AtmOcnMedPetListTimescalesSplitFastProto/Makefile index 47e0fc2e..e32259e6 100644 --- a/AtmOcnMedPetListTimescalesSplitFastProto/Makefile +++ b/AtmOcnMedPetListTimescalesSplitFastProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 med.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMedProto/Makefile b/AtmOcnMedProto/Makefile index 47e0fc2e..e32259e6 100644 --- a/AtmOcnMedProto/Makefile +++ b/AtmOcnMedProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 med.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnMirrorFieldsProto/Makefile b/AtmOcnMirrorFieldsProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnMirrorFieldsProto/Makefile +++ b/AtmOcnMirrorFieldsProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnPetListProto/Makefile b/AtmOcnPetListProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnPetListProto/Makefile +++ b/AtmOcnPetListProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnPetListProto/atm.F90 b/AtmOcnPetListProto/atm.F90 index be719494..24c88264 100644 --- a/AtmOcnPetListProto/atm.F90 +++ b/AtmOcnPetListProto/atm.F90 @@ -86,8 +86,8 @@ subroutine Advertise(model, rc) ! Disabling the following macro, e.g. renaming to WITHIMPORTFIELDS_disable, ! will result in a model component that does not advertise any importable ! Fields. Use this if you want to drive the model independently. -#define WITHIMPORTFIELDS -#ifdef WITHIMPORTFIELDS +#define WITHIMPORTFIELDS_GRID +#ifdef WITHIMPORTFIELDS_GRID ! importable field: sea_surface_temperature call NUOPC_Advertise(importState, & StandardName="sea_surface_temperature", name="sst", rc=rc) @@ -97,6 +97,28 @@ subroutine Advertise(model, rc) return ! bail out #endif +#define WITHIMPORTFIELDS_MESH +#ifdef WITHIMPORTFIELDS_MESH + ! importable field: sea_surface_salinity + call NUOPC_Advertise(importState, & + StandardName="sea_surface_salinity", name="sss", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + +#define WITHIMPORTFIELDS_LOCSTREAM +#ifdef WITHIMPORTFIELDS_LOCSTREAM + ! importable field: sea_surface_height_above_sea_level + call NUOPC_Advertise(importState, & + StandardName="sea_surface_height_above_sea_level", name="ssh", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + ! exportable field: air_pressure_at_sea_level call NUOPC_Advertise(exportState, & StandardName="air_pressure_at_sea_level", name="pmsl", rc=rc) @@ -113,6 +135,14 @@ subroutine Advertise(model, rc) file=__FILE__)) & return ! bail out + ! exportable field: precipitation_flux + call NUOPC_Advertise(exportState, & + StandardName="precipitation_flux", name="precip", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end subroutine !----------------------------------------------------------------------------- @@ -124,8 +154,16 @@ subroutine Realize(model, rc) ! local variables type(ESMF_State) :: importState, exportState type(ESMF_Field) :: field - type(ESMF_Grid) :: gridIn - type(ESMF_Grid) :: gridOut + type(ESMF_Grid) :: gridIn, gridOut + type(ESMF_Mesh) :: meshIn, meshOut + type(ESMF_LocStream) :: locsIn, locsOut + + integer, parameter :: totalNumPoints=100 + integer(ESMF_KIND_I4), pointer :: mask(:) + real(ESMF_KIND_R8), pointer :: lon(:), lat(:) + real(ESMF_KIND_R8), pointer :: fptr(:) + integer :: clb(1), cub(1), i + type(ESMF_VM) :: vm rc = ESMF_SUCCESS @@ -137,11 +175,12 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out - ! create a Grid object for Fields + ! create Grid objects for Fields gridIn = ESMF_GridCreateNoPeriDimUfrm(maxIndex=(/10, 100/), & minCornerCoord=(/10._ESMF_KIND_R8, 20._ESMF_KIND_R8/), & maxCornerCoord=(/100._ESMF_KIND_R8, 200._ESMF_KIND_R8/), & - coordSys=ESMF_COORDSYS_CART, staggerLocList=(/ESMF_STAGGERLOC_CENTER/), & + coordSys=ESMF_COORDSYS_CART, & + staggerLocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -149,8 +188,87 @@ subroutine Realize(model, rc) return ! bail out gridOut = gridIn ! for now out same as in -#ifdef WITHIMPORTFIELDS - ! importable field: sea_surface_temperature + ! create Mesh objects for Fields + meshIn = ESMF_MeshCreate(grid=gridIn, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + meshOut = ESMF_MeshCreate(grid=gridOut, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create LocStream objects for Fields + locsIn=ESMF_LocStreamCreate(name="Equatorial Measurements", & + maxIndex=totalNumPoints, coordSys=ESMF_COORDSYS_SPH_DEG, & + indexFlag=ESMF_INDEX_GLOBAL, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Add key data (internally allocating memory). + call ESMF_LocStreamAddKey(locsIn, & + keyName="ESMF:Lat", & + KeyTypeKind=ESMF_TYPEKIND_R8, & + keyUnits="Degrees", & + keyLongName="Latitude", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LocStreamAddKey(locsIn, & + keyName="ESMF:Lon", & + KeyTypeKind=ESMF_TYPEKIND_R8, & + keyUnits="Degrees", & + keyLongName="Longitude", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LocStreamAddKey(locsIn, & + keyName="ESMF:Mask", & + KeyTypeKind=ESMF_TYPEKIND_I4, & + keyUnits="none", & + keyLongName="mask values", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Get coordinate memory + call ESMF_LocStreamGetKey(locsIn, & + localDE=0, & + keyName="ESMF:Lat", & + farray=lat, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LocStreamGetKey(locsIn, & + localDE=0, & + keyName="ESMF:Lon", & + farray=lon, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Get mask memory + call ESMF_LocStreamGetKey(locsIn, & + localDE=0, & + keyName="ESMF:Mask", & + farray=mask, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + locsOut = locsIn ! for now out same as in + +#ifdef WITHIMPORTFIELDS_GRID + ! importable field on Grid: sea_surface_temperature field = ESMF_FieldCreate(name="sst", grid=gridIn, & typekind=ESMF_TYPEKIND_R8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -164,7 +282,52 @@ subroutine Realize(model, rc) return ! bail out #endif - ! exportable field: air_pressure_at_sea_level +#ifdef WITHIMPORTFIELDS_MESH + ! importable field on Mesh: sea_surface_salinity + field = ESMF_FieldCreate(name="sss", mesh=meshIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + +#ifdef WITHIMPORTFIELDS_LOCSTREAM + ! importable field on LocStream: sea_surface_height_above_sea_level + field = ESMF_FieldCreate(name="ssh", locstream=locsIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Get Field memory + call ESMF_FieldGet(field, localDe=0, farrayPtr=fptr, & + computationalLBound=clb, computationalUBound=cub, rc=rc) + ! Set coordinate data and field data + do i=clb(1),cub(1) + lon(i)=(i-1)*360.0/REAL(totalNumPoints) + lat(i)=0.0 + fptr(i)=0.0 ! Init to 0.0 + mask(i)=0 + ! Mask out range + ! (Same range as in ocn.F90) + if ((lon(i) > 10.0) .and. (lon(i) < 20.0)) then + mask(i)=2 + endif + enddo +#endif + + ! exportable field on Grid: air_pressure_at_sea_level field = ESMF_FieldCreate(name="pmsl", grid=gridOut, & typekind=ESMF_TYPEKIND_R8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -177,7 +340,7 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out - ! exportable field: surface_net_downward_shortwave_flux + ! exportable field on Grid: surface_net_downward_shortwave_flux field = ESMF_FieldCreate(name="rsns", grid=gridOut, & typekind=ESMF_TYPEKIND_R8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -190,6 +353,19 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out + ! exportable field on Mesh: precipitation_flux + field = ESMF_FieldCreate(name="precip", mesh=meshOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end subroutine !----------------------------------------------------------------------------- @@ -223,6 +399,12 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out + call ESMF_VMLog(vm, "ATM Advance(): ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Now can use OpenMP for fine grained parallelism... ! Here just write info about the PET-local OpenMP threads to Log. !$omp parallel private(msgString, currentSsiPe) diff --git a/AtmOcnPetListProto/esm.F90 b/AtmOcnPetListProto/esm.F90 index 94f7d084..8ad92e74 100644 --- a/AtmOcnPetListProto/esm.F90 +++ b/AtmOcnPetListProto/esm.F90 @@ -55,6 +55,12 @@ subroutine SetServices(driver, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + call NUOPC_CompSpecialize(driver, specLabel=label_ModifyCplLists, & + specRoutine=ModifyCplLists, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out ! set driver verbosity call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc) @@ -81,8 +87,11 @@ subroutine SetModelServices(driver, rc) type(ESMF_GridComp) :: child type(ESMF_CplComp) :: connector integer :: petCount, i + character(len=160) :: petListString integer, allocatable :: petList(:) type(ESMF_Info) :: info + type(ESMF_Config) :: config + type(NUOPC_FreeFormat) :: ff ! - diagnostics - type(ESMF_VM) :: vm @@ -100,25 +109,41 @@ subroutine SetModelServices(driver, rc) return ! bail out ! get the petCount - call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) + call ESMF_GridCompGet(driver, config=config, petCount=petCount, vm=vm, & + rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out - ! SetServices for ATM with petList on first half of PETs - allocate(petList(petCount/2)) - do i=1, petCount/2 - petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 - enddo - call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & + call ESMF_VMLog(vm, "ESM SetModelServices() enter: ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! ATM + ! - set up petList + ff = NUOPC_FreeFormatCreate(config, label="ATM_petlist:", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_IngestPetList(petList, ff, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! - set /NUOPC/Hint/PePerPet/MaxCount + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=1, & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out + ! - add the ATM component to Driver call NUOPC_DriverAddComp(driver, "ATM", atmSS, atmSVM, info=info, & - petList=petList, comp=child, rc=rc) + petList=petList, config=config, comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -195,17 +220,26 @@ subroutine SetModelServices(driver, rc) #endif #endif - ! SetServices for OCN with petList on second half of PETs - allocate(petList(petCount/2)) - do i=1, petCount/2 - petList(i) = petCount/2 + i-1 ! PET labeling goes from 0 to petCount-1 - enddo - call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & + ! OCN + ! - set up petList + ff = NUOPC_FreeFormatCreate(config, label="OCN_petlist:", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_IngestPetList(petList, ff, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! - set /NUOPC/Hint/PePerPet/MaxCount + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=1, & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out + ! - add the OCN component to Driver call NUOPC_DriverAddComp(driver, "OCN", ocnSS, ocnSVM, info=info, & petList=petList, comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -323,6 +357,92 @@ subroutine SetModelServices(driver, rc) file=__FILE__)) & return ! bail out + call ESMF_GridCompGet(driver, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_VMLog(vm, "ESM SetModelServices() exit: ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine ModifyCplLists(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + ! local variables + character(len=160) :: msg + type(ESMF_CplComp), pointer :: connectorList(:) + integer :: i, j, cplListSize + character(len=160), allocatable :: cplList(:) + character(len=160) :: tempString + + rc = ESMF_SUCCESS + + call ESMF_LogWrite("Driver is in ModifyCplLists()", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + nullify(connectorList) + call NUOPC_DriverGetComp(driver, compList=connectorList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + write (msg,*) "Found ", size(connectorList), " Connectors."// & + " Modifying CplList Attribute...." + call ESMF_LogWrite(trim(msg), ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + do i=1, size(connectorList) + ! query the cplList for connector i + call NUOPC_CompAttributeGet(connectorList(i), name="CplList", & + itemCount=cplListSize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + if (cplListSize>0) then + allocate(cplList(cplListSize)) + call NUOPC_CompAttributeGet(connectorList(i), name="CplList", & + valueList=cplList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! go through all of the entries in the cplList + do j=1, cplListSize + if (trim(cplList(j))=="sea_surface_height_above_sea_level") then + ! switch to ESMF_REGRIDMETHOD_NEAREST_STOD + cplList(j) = trim(cplList(j))//":REMAPMETHOD=nearest_stod"// & + ":srcMaskValues=1:dstMaskValues=2" + endif + enddo + ! store the modified cplList in CplList attribute of connector i + call NUOPC_CompAttributeSet(connectorList(i), & + name="CplList", valueList=cplList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(cplList) + endif + enddo + + deallocate(connectorList) + end subroutine !----------------------------------------------------------------------------- diff --git a/AtmOcnPetListProto/esmApp.F90 b/AtmOcnPetListProto/esmApp.F90 index 2aaba819..16bae9a2 100644 --- a/AtmOcnPetListProto/esmApp.F90 +++ b/AtmOcnPetListProto/esmApp.F90 @@ -21,10 +21,16 @@ program esmApp integer :: rc, urc type(ESMF_GridComp) :: esmComp + type(ESMF_Config) :: config + type(ESMF_VM) :: vm ! Initialize ESMF - call ESMF_Initialize(logkindflag=ESMF_LOGKIND_MULTI, & - defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) + call ESMF_Initialize(& + configFileName="nuopc.configure", & + defaultGlobalResourceControl=.true., & + defaultCalKind=ESMF_CALKIND_GREGORIAN, & + config=config, & + vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -36,8 +42,14 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_VMLog(vm, "esmApp VM init: ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + ! Create the earth system Component - esmComp = ESMF_GridCompCreate(name="esm", rc=rc) + esmComp = ESMF_GridCompCreate(name="esm", config=config, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -65,6 +77,12 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_VMLog(vm, "esmApp VM: ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + ! Call Run for earth the system Component call ESMF_GridCompRun(esmComp, userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -94,6 +112,12 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_VMLog(vm, "esmApp VM final: ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_LogWrite("esmApp FINISHED", ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & diff --git a/AtmOcnPetListProto/nuopc.configure b/AtmOcnPetListProto/nuopc.configure new file mode 100644 index 00000000..fd64c8e6 --- /dev/null +++ b/AtmOcnPetListProto/nuopc.configure @@ -0,0 +1,6 @@ +globalResourceControl: .false. +logKindFlag: ESMF_LOGKIND_MULTI +logAppendFlag: .false. + +ATM_petlist: 0-1 +OCN_petlist: 2-3 diff --git a/AtmOcnPetListProto/ocn.F90 b/AtmOcnPetListProto/ocn.F90 index dffddccf..cb56083f 100644 --- a/AtmOcnPetListProto/ocn.F90 +++ b/AtmOcnPetListProto/ocn.F90 @@ -109,6 +109,14 @@ subroutine Advertise(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + + ! importable field: precipitation_flux + call NUOPC_Advertise(importState, & + StandardName="precipitation_flux", name="precip", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out #endif ! exportable field: sea_surface_temperature @@ -119,6 +127,22 @@ subroutine Advertise(model, rc) file=__FILE__)) & return ! bail out + ! exportable field: sea_surface_salinity + call NUOPC_Advertise(exportState, & + StandardName="sea_surface_salinity", name="sss", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_height_above_sea_level + call NUOPC_Advertise(exportState, & + StandardName="sea_surface_height_above_sea_level", name="ssh", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + end subroutine !----------------------------------------------------------------------------- @@ -131,8 +155,16 @@ subroutine Realize(model, rc) type(ESMF_State) :: importState, exportState type(ESMF_TimeInterval) :: stabilityTimeStep type(ESMF_Field) :: field - type(ESMF_Grid) :: gridIn - type(ESMF_Grid) :: gridOut + type(ESMF_Grid) :: gridIn, gridOut + type(ESMF_Mesh) :: meshIn, meshOut + type(ESMF_LocStream) :: locsIn, locsOut + + integer, parameter :: totalNumPoints=100 + integer(ESMF_KIND_I4), pointer :: mask(:) + real(ESMF_KIND_R8), pointer :: lon(:), lat(:) + real(ESMF_KIND_R8), pointer :: fptr(:) + integer :: clb(1), cub(1), i + type(ESMF_VM) :: vm rc = ESMF_SUCCESS @@ -144,11 +176,12 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out - ! create a Grid object for Fields + ! create Grid objects for Fields gridIn = ESMF_GridCreateNoPeriDimUfrm(maxIndex=(/100, 20/), & minCornerCoord=(/10._ESMF_KIND_R8, 20._ESMF_KIND_R8/), & maxCornerCoord=(/100._ESMF_KIND_R8, 200._ESMF_KIND_R8/), & - coordSys=ESMF_COORDSYS_CART, staggerLocList=(/ESMF_STAGGERLOC_CENTER/), & + coordSys=ESMF_COORDSYS_CART, & + staggerLocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -156,8 +189,87 @@ subroutine Realize(model, rc) return ! bail out gridOut = gridIn ! for now out same as in + ! create Mesh objects for Fields + meshIn = ESMF_MeshCreate(grid=gridIn, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + meshOut = ESMF_MeshCreate(grid=gridOut, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create LocStream objects for Fields + locsIn=ESMF_LocStreamCreate(name="Equatorial Measurements", & + maxIndex=totalNumPoints, coordSys=ESMF_COORDSYS_SPH_DEG, & + indexFlag=ESMF_INDEX_GLOBAL, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Add key data (internally allocating memory). + call ESMF_LocStreamAddKey(locsIn, & + keyName="ESMF:Lat", & + KeyTypeKind=ESMF_TYPEKIND_R8, & + keyUnits="Degrees", & + keyLongName="Latitude", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LocStreamAddKey(locsIn, & + keyName="ESMF:Lon", & + KeyTypeKind=ESMF_TYPEKIND_R8, & + keyUnits="Degrees", & + keyLongName="Longitude", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LocStreamAddKey(locsIn, & + keyName="ESMF:Mask", & + KeyTypeKind=ESMF_TYPEKIND_I4, & + keyUnits="none", & + keyLongName="mask values", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Get coordinate memory + call ESMF_LocStreamGetKey(locsIn, & + localDE=0, & + keyName="ESMF:Lat", & + farray=lat, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LocStreamGetKey(locsIn, & + localDE=0, & + keyName="ESMF:Lon", & + farray=lon, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Get mask memory + call ESMF_LocStreamGetKey(locsIn, & + localDE=0, & + keyName="ESMF:Mask", & + farray=mask, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + locsOut = locsIn ! for now out same as in + #ifdef WITHIMPORTFIELDS - ! importable field: air_pressure_at_sea_level + ! importable field on Grid: air_pressure_at_sea_level field = ESMF_FieldCreate(name="pmsl", grid=gridIn, & typekind=ESMF_TYPEKIND_R8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -170,7 +282,7 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out - ! importable field: surface_net_downward_shortwave_flux + ! importable field on Grid: surface_net_downward_shortwave_flux field = ESMF_FieldCreate(name="rsns", grid=gridIn, & typekind=ESMF_TYPEKIND_R8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -182,9 +294,22 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + + ! importable field on Mesh: precipitation_flux + field = ESMF_FieldCreate(name="precip", mesh=meshIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out #endif - ! exportable field: sea_surface_temperature + ! exportable field on Grid: sea_surface_temperature field = ESMF_FieldCreate(name="sst", grid=gridOut, & typekind=ESMF_TYPEKIND_R8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -197,6 +322,48 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out + ! exportable field on Mesh: sea_surface_salinity + field = ESMF_FieldCreate(name="sss", mesh=meshOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field on LocStream: sea_surface_height_above_sea_level + field = ESMF_FieldCreate(name="ssh", locstream=locsOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Get Field memory + call ESMF_FieldGet(field, localDe=0, farrayPtr=fptr, & + computationalLBound=clb, computationalUBound=cub, rc=rc) + ! Set coordinate data and field data + do i=clb(1),cub(1) + lon(i)=(i-1)*360.0/REAL(totalNumPoints) + lat(i)=0.0 + fptr(i)=lon(i)/360.0 ! Just set it to this for testing + mask(i)=0 + ! Mask out range and make data bad + ! (Same range as in atm.F90) + if ((lon(i) > 10.0) .and. (lon(i) < 20.0)) then + mask(i)=1 + fptr(i)=-10000.0 ! Bad value to check that mask works + endif + enddo + end subroutine !----------------------------------------------------------------------------- @@ -268,6 +435,12 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out + call ESMF_VMLog(vm, "OCN Advance(): ", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! Now can use OpenMP for fine grained parallelism... ! Here just write info about the PET-local OpenMP threads to Log. !$omp parallel private(msgString, currentSsiPe) diff --git a/AtmOcnProto/Makefile b/AtmOcnProto/Makefile index 9ab63796..26d5a182 100644 --- a/AtmOcnProto/Makefile +++ b/AtmOcnProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnRtmTwoTimescalesProto/Makefile b/AtmOcnRtmTwoTimescalesProto/Makefile index fbed929f..216de5b8 100644 --- a/AtmOcnRtmTwoTimescalesProto/Makefile +++ b/AtmOcnRtmTwoTimescalesProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 rtm.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnScalarProto/Makefile b/AtmOcnScalarProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnScalarProto/Makefile +++ b/AtmOcnScalarProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnSelectExternalProto/Makefile b/AtmOcnSelectExternalProto/Makefile index 464ba877..208c639a 100644 --- a/AtmOcnSelectExternalProto/Makefile +++ b/AtmOcnSelectExternalProto/Makefile @@ -170,4 +170,4 @@ edit: nedit esmApp.F90 esm.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnSelectExternalProto/buildSubs.csh b/AtmOcnSelectExternalProto/buildSubs.csh index 6518109e..6804b9ce 100755 --- a/AtmOcnSelectExternalProto/buildSubs.csh +++ b/AtmOcnSelectExternalProto/buildSubs.csh @@ -1,4 +1,4 @@ -#!/bin/csh +#!/bin/sh cd ATM-A gmake distclean; gmake diff --git a/AtmOcnSelectExternalProto/cleanSubs.csh b/AtmOcnSelectExternalProto/cleanSubs.csh index 097e3780..2c94dc32 100755 --- a/AtmOcnSelectExternalProto/cleanSubs.csh +++ b/AtmOcnSelectExternalProto/cleanSubs.csh @@ -1,4 +1,4 @@ -#!/bin/csh +#!/bin/sh cd ATM-A gmake distclean diff --git a/AtmOcnSelectProto/Makefile b/AtmOcnSelectProto/Makefile index 41adebcb..60d8e384 100644 --- a/AtmOcnSelectProto/Makefile +++ b/AtmOcnSelectProto/Makefile @@ -85,4 +85,4 @@ edit: nedit esmApp.F90 esm.F90 atmA.F90 atmB.F90 ocnA.F90 ocnB.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnSimpleImplicitProto/Makefile b/AtmOcnSimpleImplicitProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnSimpleImplicitProto/Makefile +++ b/AtmOcnSimpleImplicitProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnTransferGridProto/Makefile b/AtmOcnTransferGridProto/Makefile index 565d4938..a3ea92a6 100644 --- a/AtmOcnTransferGridProto/Makefile +++ b/AtmOcnTransferGridProto/Makefile @@ -54,7 +54,7 @@ esm.o: atm.o ocn.o # ----------------------------------------------------------------------------- .PHONY: dust clean distclean info edit dust: - rm -f PET*.ESMF_LogFile *.nc *.vtk *.stdout + rm -f PET*.ESMF_LogFile *.nc *.vtk *.stdout fort.* clean: rm -f esmApp *.o *.mod distclean: dust clean @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnTransferGridProto/atm.F90 b/AtmOcnTransferGridProto/atm.F90 index 1aeba592..edba1d78 100644 --- a/AtmOcnTransferGridProto/atm.F90 +++ b/AtmOcnTransferGridProto/atm.F90 @@ -28,7 +28,7 @@ module ATM private public SetVM, SetServices - + !----------------------------------------------------------------------------- contains !----------------------------------------------------------------------------- @@ -407,7 +407,7 @@ subroutine AcceptTransfer(model, rc) ! create the new DistGrid with the same minIndexPTile and maxIndexPTile, ! but use default multi-tile regDecomp - ! If the default regDecomp is not suitable, a custome one could be set + ! If the default regDecomp is not suitable, a custom one could be set ! up here and used. distgrid = ESMF_DistGridCreate(minIndexPTile=minIndexPTile, & maxIndexPTile=maxIndexPTile, connectionList=connectionList, rc=rc) @@ -419,7 +419,7 @@ subroutine AcceptTransfer(model, rc) deallocate(minIndexPTile, maxIndexPTile, connectionList) ! Create a new Grid on the new DistGrid - grid = ESMF_GridCreate(distgrid, rc=rc) + grid = ESMF_GridCreate(distgrid, name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -539,7 +539,7 @@ subroutine AcceptTransfer(model, rc) file=__FILE__)) & return ! bail out ! Create default regDecomp Grid - grid = ESMF_GridCreate(distgrid, rc=rc) + grid = ESMF_GridCreate(distgrid, name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -1046,21 +1046,25 @@ subroutine RealizeAccepted(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_center_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(grid, coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_center_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif #ifdef TEST_GRID_EDGE_WIDTHS ! corner: call ESMF_GridGetCoord(grid, staggerloc=ESMF_STAGGERLOC_CORNER, & @@ -1069,22 +1073,26 @@ subroutine RealizeAccepted(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_corner_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(grid, staggerloc=ESMF_STAGGERLOC_CORNER, & coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_corner_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif ! edge1: call ESMF_GridGetCoord(grid, staggerloc=ESMF_STAGGERLOC_EDGE1, & coordDim=1, array=array, rc=rc) @@ -1092,22 +1100,26 @@ subroutine RealizeAccepted(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_edge1_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(grid, staggerloc=ESMF_STAGGERLOC_EDGE1, & coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_edge1_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif ! edge2: call ESMF_GridGetCoord(grid, staggerloc=ESMF_STAGGERLOC_EDGE2, & coordDim=1, array=array, rc=rc) @@ -1115,17 +1127,20 @@ subroutine RealizeAccepted(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_edge2_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(grid, staggerloc=ESMF_STAGGERLOC_EDGE2, & coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_ATM-grid_edge2_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -1133,6 +1148,7 @@ subroutine RealizeAccepted(model, rc) return ! bail out #endif #endif +#endif #if 1 ! write out the Grid into VTK file for inspection @@ -1365,6 +1381,7 @@ subroutine DataInitialize(model, rc) !----------------------------------------------------------------------------- subroutine Advance(model, rc) +!$ use omp_lib type(ESMF_GridComp) :: model integer, intent(out) :: rc @@ -1372,6 +1389,8 @@ subroutine Advance(model, rc) type(ESMF_Clock) :: clock type(ESMF_State) :: importState, exportState integer, save :: slice=1 + type(ESMF_VM) :: vm + integer :: currentSsiPe, i, tid, unit, localPet character(len=160) :: msgString rc = ESMF_SUCCESS @@ -1383,6 +1402,49 @@ subroutine Advance(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + call ESMF_GridCompGet(model, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_VMGet(vm, localPet=localPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Now can use OpenMP for fine grained parallelism... + ! Here just write info about the PET-local OpenMP threads to Log. +!$omp parallel private(msgString, currentSsiPe) +!$omp critical +!$ call ESMF_VMGet(vm, currentSsiPe=currentSsiPe) +!$ write(msgString,'(A,I4,A,I4,A,I4,A,I4,A,I4)') & +!$ "thread_num=", omp_get_thread_num(), & +!$ " currentSsiPe=", currentSsiPe, & +!$ " num_threads=", omp_get_num_threads(), & +!$ " max_threads=", omp_get_max_threads(), & +!$ " num_procs=", omp_get_num_procs() +!$ call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) +!$omp end critical +!$omp end parallel + +#define FILEOUT_off +! Activating FILEOUT triggers a hang for Intel < 19.1.1 when multi-threaded +! due to an Intel bug +#ifdef FILEOUT + unit = localPet + 100 +#else + unit = 6 +#endif +!$omp parallel private(tid) + tid = -1 ! initialize to obvious value if building without OpenMP +!$ tid = omp_get_thread_num() +!$omp do + do i=1, 100 + write(unit,*) "ATM test write, localPet=", localPet, " tid=", tid, & + " slice=", slice, " i=", i + enddo +!$omp end parallel ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep diff --git a/AtmOcnTransferGridProto/esm.F90 b/AtmOcnTransferGridProto/esm.F90 index f2ca4706..f548c7cb 100644 --- a/AtmOcnTransferGridProto/esm.F90 +++ b/AtmOcnTransferGridProto/esm.F90 @@ -96,10 +96,24 @@ subroutine SetModelServices(driver, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + + ! get the petCount + call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! split up the PETs between ATM and OCN + petCountOCN = min(2,petCount/2) ! don't give OCN more than 2 PETs + petCountATM = petCount - petCountOCN + #define MORE_THAN_ONE_PE_PER_PET_WORKING_ATM #ifdef MORE_THAN_ONE_PE_PER_PET_WORKING_ATM call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & rc=rc) +! call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/ForceChildPthreads", & +! value=.true., rc=rc) #else call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=1, & rc=rc) @@ -109,24 +123,14 @@ subroutine SetModelServices(driver, rc) file=__FILE__)) & return ! bail out - ! get the petCount - call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - - ! split up the PETs between ATM and OCN - petCountOCN = min(2,petCount/2) ! don't give OCN more than 2 PETs - petCountATM = petCount - petCountOCN - ! SetServices for ATM with petList on first half of PETs allocate(petList(petCountATM)) do i=1, petCountATM petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 enddo call NUOPC_DriverAddComp(driver, "ATM", atmSS, atmSVM, info=info, & - petList=petList, comp=child, rc=rc) + petList=petList, & + comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -142,6 +146,8 @@ subroutine SetModelServices(driver, rc) #ifdef MORE_THAN_ONE_PE_PER_PET_WORKING_OCN call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & rc=rc) +! call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/ForceChildPthreads", & +! value=.false., rc=rc) #else call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=1, & rc=rc) @@ -157,7 +163,8 @@ subroutine SetModelServices(driver, rc) petList(i) = petCountATM + i-1 ! PET labeling goes from 0 to petCount-1 enddo call NUOPC_DriverAddComp(driver, "OCN", ocnSS, ocnSVM, info=info, & - petList=petList, comp=child, rc=rc) + petList=petList, & + comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & diff --git a/AtmOcnTransferGridProto/esmApp.F90 b/AtmOcnTransferGridProto/esmApp.F90 index 2aaba819..b717175f 100644 --- a/AtmOcnTransferGridProto/esmApp.F90 +++ b/AtmOcnTransferGridProto/esmApp.F90 @@ -101,6 +101,6 @@ program esmApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Finalize ESMF - call ESMF_Finalize() + call ESMF_Finalize(rc=rc) end program diff --git a/AtmOcnTransferGridProto/ocn.F90 b/AtmOcnTransferGridProto/ocn.F90 index cffcd676..4032d7ed 100644 --- a/AtmOcnTransferGridProto/ocn.F90 +++ b/AtmOcnTransferGridProto/ocn.F90 @@ -347,21 +347,25 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_center_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(gridIn, coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_center_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif #ifdef TEST_GRID_EDGE_WIDTHS ! corner: call ESMF_GridGetCoord(gridIn, staggerloc=ESMF_STAGGERLOC_CORNER, & @@ -370,22 +374,26 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_corner_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(gridIn, staggerloc=ESMF_STAGGERLOC_CORNER, & coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_corner_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif ! edge1: call ESMF_GridGetCoord(gridIn, staggerloc=ESMF_STAGGERLOC_EDGE1, & coordDim=1, array=array, rc=rc) @@ -393,22 +401,26 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_edge1_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(gridIn, staggerloc=ESMF_STAGGERLOC_EDGE1, & coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_edge1_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif ! edge2: call ESMF_GridGetCoord(gridIn, staggerloc=ESMF_STAGGERLOC_EDGE2, & coordDim=1, array=array, rc=rc) @@ -416,17 +428,20 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_edge2_coord1.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif call ESMF_GridGetCoord(gridIn, staggerloc=ESMF_STAGGERLOC_EDGE2, & coordDim=2, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_ArrayWrite(array, "array_OCN-gridIn_edge2_coord2.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -434,6 +449,7 @@ subroutine Realize(model, rc) return ! bail out #endif #endif +#endif #if 1 ! write out the Grid into VTK file for inspection @@ -726,11 +742,13 @@ subroutine Realize(model, rc) return ! bail out ! write the field out to file, which will be 1d and with the arbitrary order +#ifdef ESMF_NETCDF call ESMF_FieldWrite(fieldArb, fileName="field_gridArb.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif ! create an auxiliary regDecomp grid with the same index space as gridArb ! periodic along i (must set here explicitly to match the gridArb @@ -780,11 +798,13 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMF_FieldWrite(fieldAux, fileName="field_gridAux.nc", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif #endif ! pull out the sst field from exportState for testing regridding to it @@ -828,6 +848,23 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out + ! clean-up + call ESMF_FieldDestroy(fieldArb, noGarbage=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldDestroy(fieldAux, noGarbage=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_RouteHandleDestroy(rh, noGarbage=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + #ifdef TEST_MULTI_TILE_GRID ioComp = ESMFIO_Create(gridOut, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -839,12 +876,14 @@ subroutine Realize(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#ifdef ESMF_NETCDF call ESMFIO_Write(ioComp, "fields_ocn_init_export.nc", & (/field/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out +#endif #if 0 ! Currently do not destroy the ioComp here, because it will trigger ! an issue in StateReconcile(), which looks like a bug to me. @@ -927,6 +966,7 @@ subroutine SetClock(model, rc) !----------------------------------------------------------------------------- subroutine Advance(model, rc) +!$ use omp_lib type(ESMF_GridComp) :: model integer, intent(out) :: rc @@ -938,6 +978,8 @@ subroutine Advance(model, rc) type(ESMF_Time) :: currTime type(ESMF_TimeInterval) :: timeStep integer, save :: slice=1 + type(ESMF_VM) :: vm + integer :: currentSsiPe, i, tid, unit, localPet character(len=160) :: msgString character(80) :: fieldName, gridName @@ -950,6 +992,49 @@ subroutine Advance(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + call ESMF_GridCompGet(model, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_VMGet(vm, localPet=localPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Now can use OpenMP for fine grained parallelism... + ! Here just write info about the PET-local OpenMP threads to Log. +!$omp parallel private(msgString, currentSsiPe) +!$omp critical +!$ call ESMF_VMGet(vm, currentSsiPe=currentSsiPe) +!$ write(msgString,'(A,I4,A,I4,A,I4,A,I4,A,I4)') & +!$ "thread_num=", omp_get_thread_num(), & +!$ " currentSsiPe=", currentSsiPe, & +!$ " num_threads=", omp_get_num_threads(), & +!$ " max_threads=", omp_get_max_threads(), & +!$ " num_procs=", omp_get_num_procs() +!$ call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) +!$omp end critical +!$omp end parallel + +#define FILEOUT_off +! Activating FILEOUT triggers a hang for Intel < 19.1.1 when multi-threaded +! due to an Intel bug +#ifdef FILEOUT + unit = localPet + 200 +#else + unit = 6 +#endif +!$omp parallel private(tid) + tid = -1 ! initialize to obvious value if building without OpenMP +!$ tid = omp_get_thread_num() +!$omp do + do i=1, 100 + write(unit,*) "OCN test write, localPet=", localPet, " tid=", tid, & + " slice=", slice, " i=", i + enddo +!$omp end parallel ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep diff --git a/AtmOcnTransferLocStreamProto/Makefile b/AtmOcnTransferLocStreamProto/Makefile index cd34e9e8..2630ec0a 100644 --- a/AtmOcnTransferLocStreamProto/Makefile +++ b/AtmOcnTransferLocStreamProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnTransferMeshProto/Makefile b/AtmOcnTransferMeshProto/Makefile index 14b92566..d3d696d7 100644 --- a/AtmOcnTransferMeshProto/Makefile +++ b/AtmOcnTransferMeshProto/Makefile @@ -54,7 +54,7 @@ esm.o: atm.o ocn.o # ----------------------------------------------------------------------------- .PHONY: dust clean distclean info edit dust: - rm -f PET*.ESMF_LogFile field*.nc *.vtk *.stdout + rm -f PET*.ESMF_LogFile field*.nc *.vtk *.stdout fort.* clean: rm -f esmApp *.o *.mod distclean: dust clean @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/AtmOcnTransferMeshProto/atm.F90 b/AtmOcnTransferMeshProto/atm.F90 index aceb506b..dccc7547 100644 --- a/AtmOcnTransferMeshProto/atm.F90 +++ b/AtmOcnTransferMeshProto/atm.F90 @@ -23,7 +23,7 @@ module ATM private - public SetServices + public SetVM, SetServices !----------------------------------------------------------------------------- contains @@ -230,6 +230,12 @@ subroutine RealizeProvided(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + ! destroy the grid actually not needed + call ESMF_GridDestroy(gridIn, noGarbage=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out endif !NOTE: The air_pressure_at_sea_level (pmsl) Field is not realized here @@ -387,21 +393,23 @@ subroutine AcceptTransfer(model, rc) #if (defined USE_NODAL_DG && defined USE_ELEMENT_DG) ! Create a new Mesh on both new DistGrid mesh = ESMF_MeshEmptyCreate(nodalDistGrid=newNodalDG, & - elementDistGrid=newElementDG, name=name, rc=rc) + elementDistGrid=newElementDG, name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out #elif (defined USE_NODAL_DG) ! Create a new Mesh on new nodal DistGrid - mesh = ESMF_MeshEmptyCreate(nodalDistGrid=newNodalDG, name=name, rc=rc) + mesh = ESMF_MeshEmptyCreate(nodalDistGrid=newNodalDG, & + name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out #elif (defined USE_ELEMENT_DG) ! Create a new Mesh on new element DistGrid - mesh = ESMF_MeshEmptyCreate(elementDistGrid=newElementDG, name=name, rc=rc) + mesh = ESMF_MeshEmptyCreate(elementDistGrid=newElementDG, & + name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -452,7 +460,7 @@ subroutine AcceptTransfer(model, rc) #undef USE_NODAL_DG #undef USE_ELEMENT_DG -#define USE_NODAL_DG_off +#define USE_NODAL_DG #define USE_ELEMENT_DG #ifdef USE_NODAL_DG @@ -532,21 +540,23 @@ subroutine AcceptTransfer(model, rc) #if (defined USE_NODAL_DG && defined USE_ELEMENT_DG) ! Create a new Mesh on both new DistGrid mesh = ESMF_MeshEmptyCreate(nodalDistGrid=newNodalDG, & - elementDistGrid=newElementDG, name=name, rc=rc) + elementDistGrid=newElementDG, name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out #elif (defined USE_NODAL_DG) ! Create a new Mesh on new nodal DistGrid - mesh = ESMF_MeshEmptyCreate(nodalDistGrid=newNodalDG, name=name, rc=rc) + mesh = ESMF_MeshEmptyCreate(nodalDistGrid=newNodalDG, & + name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out #elif (defined USE_ELEMENT_DG) ! Create a new Mesh on new element DistGrid - mesh = ESMF_MeshEmptyCreate(elementDistGrid=newElementDG, name=name, rc=rc) + mesh = ESMF_MeshEmptyCreate(elementDistGrid=newElementDG, & + name="ATM-custom-"//trim(name), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -727,6 +737,13 @@ subroutine RealizeAccepted(model, rc) file=__FILE__)) & return ! bail out + ! release the RouteHandle + call ESMF_FieldRegridRelease(rh, noGarbage=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! realize "sss" field in the importState, with transferred Mesh ! test the option here to specify an explicit typekind call NUOPC_Realize(importState, fieldName="sss", & @@ -805,12 +822,16 @@ subroutine DataInitialize(model, rc) !----------------------------------------------------------------------------- subroutine Advance(model, rc) +!$ use omp_lib type(ESMF_GridComp) :: model integer, intent(out) :: rc ! local variables type(ESMF_Clock) :: clock type(ESMF_State) :: importState, exportState + integer, save :: slice=1 + type(ESMF_VM) :: vm + integer :: currentSsiPe, i, tid, unit, localPet character(len=160) :: msgString rc = ESMF_SUCCESS @@ -822,6 +843,49 @@ subroutine Advance(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + call ESMF_GridCompGet(model, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_VMGet(vm, localPet=localPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Now can use OpenMP for fine grained parallelism... + ! Here just write info about the PET-local OpenMP threads to Log. +!$omp parallel private(msgString, currentSsiPe) +!$omp critical +!$ call ESMF_VMGet(vm, currentSsiPe=currentSsiPe) +!$ write(msgString,'(A,I4,A,I4,A,I4,A,I4,A,I4)') & +!$ "thread_num=", omp_get_thread_num(), & +!$ " currentSsiPe=", currentSsiPe, & +!$ " num_threads=", omp_get_num_threads(), & +!$ " max_threads=", omp_get_max_threads(), & +!$ " num_procs=", omp_get_num_procs() +!$ call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) +!$omp end critical +!$omp end parallel + +#define FILEOUT_off +! Activating FILEOUT triggers a hang for Intel < 19.1.1 when multi-threaded +! due to an Intel bug +#ifdef FILEOUT + unit = localPet + 100 +#else + unit = 6 +#endif +!$omp parallel private(tid) + tid = -1 ! initialize to obvious value if building without OpenMP +!$ tid = omp_get_thread_num() +!$omp do + do i=1, 100 + write(unit,*) "ATM test write, localPet=", localPet, " tid=", tid, & + " slice=", slice, " i=", i + enddo +!$omp end parallel ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep @@ -854,6 +918,8 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out + slice = slice+1 + end subroutine !----------------------------------------------------------------------------- diff --git a/AtmOcnTransferMeshProto/esm.F90 b/AtmOcnTransferMeshProto/esm.F90 index 2332fceb..4dfbc8df 100644 --- a/AtmOcnTransferMeshProto/esm.F90 +++ b/AtmOcnTransferMeshProto/esm.F90 @@ -19,8 +19,8 @@ module ESM use NUOPC_Driver, & driverSS => SetServices - use ATM, only: atmSS => SetServices - use OCN, only: ocnSS => SetServices + use ATM, only: atmSVM => SetVM, atmSS => SetServices + use OCN, only: ocnSVM => SetVM, ocnSS => SetServices use NUOPC_Connector, only: cplSS => SetServices @@ -86,11 +86,19 @@ subroutine SetModelServices(driver, rc) integer, allocatable :: petList(:) type(ESMF_GridComp) :: child type(ESMF_CplComp) :: conn + type(ESMF_Info) :: info integer :: verbosity character(len=10) :: attrStr rc = ESMF_SUCCESS + ! Create and set the info object that is used to pass hints into methods + info = ESMF_InfoCreate(rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! get the petCount call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -102,12 +110,28 @@ subroutine SetModelServices(driver, rc) petCountOCN = min(2,petCount/2) ! don't give OCN more than 2 PETs petCountATM = petCount - petCountOCN - ! SetServices for ATM with petList on first half of PETs +#define MORE_THAN_ONE_PE_PER_PET_WORKING_ATM +#ifdef MORE_THAN_ONE_PE_PER_PET_WORKING_ATM + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=4, & + rc=rc) +! call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/ForceChildPthreads", & +! value=.true., rc=rc) +#else + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=1, & + rc=rc) +#endif + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ATM with petList on first half of PETs allocate(petList(petCountATM)) do i=1, petCountATM petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 enddo - call NUOPC_DriverAddComp(driver, "ATM", atmSS, petList=petList, & + call NUOPC_DriverAddComp(driver, "ATM", atmSS, atmSVM, info=info, & + petList=petList, & comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & @@ -123,12 +147,28 @@ subroutine SetModelServices(driver, rc) file=__FILE__)) & return ! bail out +#define MORE_THAN_ONE_PE_PER_PET_WORKING_OCN +#ifdef MORE_THAN_ONE_PE_PER_PET_WORKING_OCN + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & + rc=rc) +! call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/ForceChildPthreads", & +! value=.false., rc=rc) +#else + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=1, & + rc=rc) +#endif + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! SetServices for OCN with petList on second half of PETs allocate(petList(petCountOCN)) do i=1, petCountOCN petList(i) = petCountATM + i-1 ! PET labeling goes from 0 to petCount-1 enddo - call NUOPC_DriverAddComp(driver, "OCN", ocnSS, petList=petList, & + call NUOPC_DriverAddComp(driver, "OCN", ocnSS, ocnSVM, info=info, & + petList=petList, & comp=child, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & diff --git a/AtmOcnTransferMeshProto/esmApp.F90 b/AtmOcnTransferMeshProto/esmApp.F90 index 2aaba819..55d5c558 100644 --- a/AtmOcnTransferMeshProto/esmApp.F90 +++ b/AtmOcnTransferMeshProto/esmApp.F90 @@ -22,6 +22,11 @@ program esmApp integer :: rc, urc type(ESMF_GridComp) :: esmComp +#define WITH_EXTERNAL_STATES__off +#ifdef WITH_EXTERNAL_STATES + type(ESMF_State) :: toESM, fmESM +#endif + ! Initialize ESMF call ESMF_Initialize(logkindflag=ESMF_LOGKIND_MULTI, & defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) @@ -54,8 +59,27 @@ program esmApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) +#ifdef WITH_EXTERNAL_STATES + ! Created external states + toESM = ESMF_StateCreate(name="toESM", stateintent=ESMF_STATEINTENT_EXPORT, rc=rc) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + fmESM = ESMF_StateCreate(name="fmESM", stateintent=ESMF_STATEINTENT_IMPORT, rc=rc) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) +#endif + ! Call Initialize for the earth system Component - call ESMF_GridCompInitialize(esmComp, userRc=urc, rc=rc) + call ESMF_GridCompInitialize(esmComp, & +#ifdef WITH_EXTERNAL_STATES + importState=toESM, exportState=fmESM, & +#endif + userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -66,7 +90,11 @@ program esmApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Call Run for earth the system Component - call ESMF_GridCompRun(esmComp, userRc=urc, rc=rc) + call ESMF_GridCompRun(esmComp, & +#ifdef WITH_EXTERNAL_STATES + importState=toESM, exportState=fmESM, & +#endif + userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -77,7 +105,11 @@ program esmApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Call Finalize for the earth system Component - call ESMF_GridCompFinalize(esmComp, userRc=urc, rc=rc) + call ESMF_GridCompFinalize(esmComp, & +#ifdef WITH_EXTERNAL_STATES + importState=toESM, exportState=fmESM, & +#endif + userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -101,6 +133,6 @@ program esmApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Finalize ESMF - call ESMF_Finalize() + call ESMF_Finalize(rc=rc) end program diff --git a/AtmOcnTransferMeshProto/ocn.F90 b/AtmOcnTransferMeshProto/ocn.F90 index ca643640..6c6b687e 100644 --- a/AtmOcnTransferMeshProto/ocn.F90 +++ b/AtmOcnTransferMeshProto/ocn.F90 @@ -23,7 +23,7 @@ module OCN private - public SetServices + public SetVM, SetServices !----------------------------------------------------------------------------- contains @@ -148,7 +148,7 @@ subroutine Realize(model, rc) ! --- IMPORT ------------------------------------------------------------- #define READ_MESHIN_FROM_FILE -#ifdef READ_MESHIN_FROM_FILE +#if (defined READ_MESHIN_FROM_FILE && defined ESMF_NETCDF) ! create from file meshIn = ESMF_MeshCreate("./gx3v7_unstructured.nc", & fileformat=ESMF_FILEFORMAT_ESMFMESH, name="OCN-MeshIn", rc=rc) @@ -263,10 +263,10 @@ subroutine Realize(model, rc) ! --- EXPORT ------------------------------------------------------------- #define READ_MESHOUT_FROM_FILE -#ifdef READ_MESHOUT_FROM_FILE +#if (defined READ_MESHOUT_FROM_FILE && defined ESMF_NETCDF) ! create from file meshOut = ESMF_MeshCreate("./fv1.9x2.5_unstructured.nc", & - fileformat=ESMF_FILEFORMAT_ESMFMESH, name="OCN-GridOut", rc=rc) + fileformat=ESMF_FILEFORMAT_ESMFMESH, name="OCN-MeshOut", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & @@ -426,6 +426,7 @@ subroutine SetClock(model, rc) !----------------------------------------------------------------------------- subroutine Advance(model, rc) +!$ use omp_lib type(ESMF_GridComp) :: model integer, intent(out) :: rc @@ -434,6 +435,9 @@ subroutine Advance(model, rc) type(ESMF_State) :: importState, exportState type(ESMF_Time) :: currTime type(ESMF_TimeInterval) :: timeStep + integer, save :: slice=1 + type(ESMF_VM) :: vm + integer :: currentSsiPe, i, tid, unit, localPet character(len=160) :: msgString rc = ESMF_SUCCESS @@ -445,6 +449,49 @@ subroutine Advance(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + call ESMF_GridCompGet(model, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_VMGet(vm, localPet=localPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Now can use OpenMP for fine grained parallelism... + ! Here just write info about the PET-local OpenMP threads to Log. +!$omp parallel private(msgString, currentSsiPe) +!$omp critical +!$ call ESMF_VMGet(vm, currentSsiPe=currentSsiPe) +!$ write(msgString,'(A,I4,A,I4,A,I4,A,I4,A,I4)') & +!$ "thread_num=", omp_get_thread_num(), & +!$ " currentSsiPe=", currentSsiPe, & +!$ " num_threads=", omp_get_num_threads(), & +!$ " max_threads=", omp_get_max_threads(), & +!$ " num_procs=", omp_get_num_procs() +!$ call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) +!$omp end critical +!$omp end parallel + +#define FILEOUT_off +! Activating FILEOUT triggers a hang for Intel < 19.1.1 when multi-threaded +! due to an Intel bug +#ifdef FILEOUT + unit = localPet + 200 +#else + unit = 6 +#endif +!$omp parallel private(tid) + tid = -1 ! initialize to obvious value if building without OpenMP +!$ tid = omp_get_thread_num() +!$omp do + do i=1, 100 + write(unit,*) "OCN test write, localPet=", localPet, " tid=", tid, & + " slice=", slice, " i=", i + enddo +!$omp end parallel ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep @@ -485,6 +532,8 @@ subroutine Advance(model, rc) file=__FILE__)) & return ! bail out + slice = slice+1 + end subroutine !----------------------------------------------------------------------------- diff --git a/ComponentExplorer/Makefile b/ComponentExplorer/Makefile index a4b1957e..b80ffa32 100644 --- a/ComponentExplorer/Makefile +++ b/ComponentExplorer/Makefile @@ -97,4 +97,4 @@ edit: nedit nuopcExplorerScript nuopcExplorerApp.F90 nuopcExplorerDriver.F90 explorer.config & run: - mpirun -np 4 ./nuopcExplorerApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./nuopcExplorerApp diff --git a/CustomFieldDictionaryProto/Makefile b/CustomFieldDictionaryProto/Makefile index 2f2a89ed..cd8d2029 100644 --- a/CustomFieldDictionaryProto/Makefile +++ b/CustomFieldDictionaryProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driver.F90 model.F90 util.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/DriverInDriverDataDepProto/Makefile b/DriverInDriverDataDepProto/Makefile index 61763a8f..a84c31ed 100644 --- a/DriverInDriverDataDepProto/Makefile +++ b/DriverInDriverDataDepProto/Makefile @@ -71,4 +71,4 @@ edit: nedit mainApp.F90 driverParentComp.F90 driverChildComp.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/DriverInDriverProto/Makefile b/DriverInDriverProto/Makefile index 61763a8f..a84c31ed 100644 --- a/DriverInDriverProto/Makefile +++ b/DriverInDriverProto/Makefile @@ -71,4 +71,4 @@ edit: nedit mainApp.F90 driverParentComp.F90 driverChildComp.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/DynPhyProto/Makefile b/DynPhyProto/Makefile index d283df2e..fd393ae9 100644 --- a/DynPhyProto/Makefile +++ b/DynPhyProto/Makefile @@ -70,4 +70,4 @@ edit: nedit esmApp.F90 atm.F90 dyn.F90 phy.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/ExternalDriverAPIProto/Makefile b/ExternalDriverAPIProto/Makefile index b21c5b36..46f593ad 100644 --- a/ExternalDriverAPIProto/Makefile +++ b/ExternalDriverAPIProto/Makefile @@ -70,4 +70,4 @@ edit: nedit externalApp.F90 esm.F90 atm.F90 ocn.F90 & run: - mpirun -np 4 ./externalApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./externalApp diff --git a/ExternalDriverAPIProto/atm.F90 b/ExternalDriverAPIProto/atm.F90 index 45661559..22914385 100644 --- a/ExternalDriverAPIProto/atm.F90 +++ b/ExternalDriverAPIProto/atm.F90 @@ -61,6 +61,12 @@ subroutine SetServices(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_CheckImport, & + specRoutine=NUOPC_NoOp, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out end subroutine @@ -83,6 +89,7 @@ subroutine Advertise(model, rc) file=__FILE__)) & return ! bail out +#if 1 ! importable field: sea_surface_temperature call NUOPC_Advertise(importState, & StandardName="sea_surface_temperature", name="sst", rc=rc) @@ -90,7 +97,9 @@ subroutine Advertise(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#endif +#if 1 ! importable field: precipitation_flux call NUOPC_Advertise(importState, & StandardName="precipitation_flux", name="precip", rc=rc) @@ -98,6 +107,7 @@ subroutine Advertise(model, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#endif ! exportable field: air_pressure_at_sea_level call NUOPC_Advertise(exportState, & @@ -158,7 +168,8 @@ subroutine Realize(model, rc) return ! bail out ! create a Grid object for Fields -#if 0 +#define LAT_LON_GRID +#ifdef LAT_LON_GRID gridIn = ESMF_GridCreate1PeriDimUfrm(maxIndex=(/100, 50/), & minCornerCoord=(/0._ESMF_KIND_R8, -60._ESMF_KIND_R8/), & maxCornerCoord=(/360._ESMF_KIND_R8, 80._ESMF_KIND_R8/), & @@ -354,8 +365,7 @@ subroutine Advance(model, rc) enddo endif -#if 0 -! cannot use NUOPC_Write() for fields on cubed sphere grid right now +#ifdef LAT_LON_GRID ! write out the Fields in the importState call NUOPC_Write(importState, fileNamePrefix="field_atm_import_", & timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) diff --git a/ExternalDriverAPIProto/externalApp.F90 b/ExternalDriverAPIProto/externalApp.F90 index 60a7a30c..e75107c1 100644 --- a/ExternalDriverAPIProto/externalApp.F90 +++ b/ExternalDriverAPIProto/externalApp.F90 @@ -26,6 +26,8 @@ program externalApp type(ESMF_TimeInterval) :: timeStep type(ESMF_Clock) :: clock type(ESMF_State) :: externalExportState, externalImportState + type(ESMF_StateItem_Flag):: itemType + type(ESMF_Field) :: field integer :: phase character(40) :: sharePolicy integer, save :: slice=1 @@ -111,6 +113,7 @@ program externalApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) +#if 1 ! Advertise field(s) in external import state in order to receive from ESM #define SHARED #ifdef SHARED @@ -123,15 +126,41 @@ program externalApp sharePolicy = "not share" #endif call NUOPC_Advertise(externalImportState, & - StandardNames=(/"sea_surface_temperature"/), & + StandardNames=(/"sea_surface_temperature ", & + "surface_net_downward_shortwave_flux"/), & TransferOfferGeomObject="cannot provide", SharePolicyField=sharePolicy, & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) +#endif + +#if 1 + ! Advertise field(s) in external export state in order to send to ESM +#define SHARED +#ifdef SHARED + ! request reference sharing for field and geomobject + ! -> still the child component may not allow sharing and we end up with copy + sharePolicy = "share" +#else + ! prevent reference sharing for field and geomobject + ! -> definitely create local copy of the field here + sharePolicy = "not share" +#endif + call NUOPC_Advertise(externalExportState, & + StandardNames=(/"precipitation_flux"/), & + TransferOfferGeomObject="cannot provide", SharePolicyField=sharePolicy, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) +#endif ! Call "ExternalAdvertise" Initialize for the earth system Component + ! -> This method removes any fields previously advertised in the states, + ! but ended up not connected. call NUOPC_CompSearchPhaseMap(esmComp, methodflag=ESMF_METHOD_INITIALIZE, & phaseLabel=label_ExternalAdvertise, phaseIndex=phase, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -151,6 +180,9 @@ program externalApp call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Call "ExternalRealize" Initialize for the earth system Component + ! -> This method realizes any fields previously advertised in the states, + ! for which enough information is available to realize (e.g. due to sharing, + ! or transfers). call NUOPC_CompSearchPhaseMap(esmComp, methodflag=ESMF_METHOD_INITIALIZE, & phaseLabel=label_ExternalRealize, phaseIndex=phase, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -169,6 +201,27 @@ program externalApp file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) + ! Fill fields provided from this level with data + call ESMF_StateGet(externalExportState, itemName="precipitation_flux", & + itemType=itemType, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (itemType /= ESMF_STATEITEM_NOTFOUND) then + call ESMF_StateGet(externalExportState, itemName="precipitation_flux", & + field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + call ESMF_FieldFill(field, dataFillScheme="sincos", member=1, rc=rc) + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif + ! Call "ExternalDataInit" Initialize for the earth system Component call NUOPC_CompSearchPhaseMap(esmComp, methodflag=ESMF_METHOD_INITIALIZE, & phaseLabel=label_ExternalDataInit, phaseIndex=phase, rc=rc) @@ -207,6 +260,12 @@ program externalApp ! Explicit time stepping loop on the external level, here based on ESMF_Clock do while (.not.ESMF_ClockIsStopTime(clock, rc=rc)) + ! Timestamp the fields in the externalExportState + call NUOPC_SetTimestamp(externalExportState, clock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) ! Run the earth system Component: i.e. step ESM forward by timestep=15min call ESMF_GridCompRun(esmComp, clock=clock, & importState=externalExportState, exportState=externalImportState, & diff --git a/ExternalDriverAPIProto/ocn.F90 b/ExternalDriverAPIProto/ocn.F90 index eafeb595..8cdd521d 100644 --- a/ExternalDriverAPIProto/ocn.F90 +++ b/ExternalDriverAPIProto/ocn.F90 @@ -143,8 +143,6 @@ subroutine Realize(model, rc) type(ESMF_Field) :: field type(ESMF_Grid) :: gridIn type(ESMF_Grid) :: gridOut - real(ESMF_KIND_R8), pointer :: dataPtr(:,:), lonPtr(:), latPtr(:) - integer :: i,j rc = ESMF_SUCCESS @@ -219,40 +217,14 @@ subroutine Realize(model, rc) file=__FILE__)) & return ! bail out - ! exportable field: sea_surface_temperature - field = ESMF_FieldCreate(name="sst", grid=gridOut, & - typekind=ESMF_TYPEKIND_R8, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - call NUOPC_Realize(exportState, field=field, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - - ! fill export with some data - call ESMF_GridGetCoord(gridOut, coordDim=1, farrayPtr=lonPtr, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - call ESMF_GridGetCoord(gridOut, coordDim=2, farrayPtr=latPtr, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out - call ESMF_FieldGet(field, farrayPtr=dataPtr, rc=rc) + ! exportable field: sea_surface_temperature, realize conditionally and fill + call NUOPC_Realize(exportState, grid=gridOut, fieldName="sst", & + typekind=ESMF_TYPEKIND_R8, selection="realize_connected_remove_others", & + dataFillScheme="sincos", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out - do j=lbound(dataPtr,2),ubound(dataPtr,2) - do i=lbound(dataPtr,1),ubound(dataPtr,1) - dataPtr(i,j) = sin(3.1416*lonPtr(i)/180.) * cos(3.1416*latPtr(j)/180.) - enddo - enddo end subroutine diff --git a/ExternalDriverAPIWeakCplDAProto/Makefile b/ExternalDriverAPIWeakCplDAProto/Makefile new file mode 100644 index 00000000..d8bb2e88 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/Makefile @@ -0,0 +1,75 @@ +# GNU Makefile template for user ESMF application + +################################################################################ +################################################################################ +## This Makefile must be able to find the "esmf.mk" Makefile fragment in the ## +## 'include' line below. Following the ESMF User's Guide, a complete ESMF ## +## installation should ensure that a single environment variable "ESMFMKFILE" ## +## is made available on the system. This variable should point to the ## +## "esmf.mk" file. ## +## ## +## This example Makefile uses the "ESMFMKFILE" environment variable. ## +## ## +## If you notice that this Makefile cannot find variable ESMFMKFILE then ## +## please contact the person responsible for the ESMF installation on your ## +## system. ## +## As a work-around you can simply hardcode the path to "esmf.mk" in the ## +## include line below. However, doing so will render this Makefile a lot less ## +## flexible and non-portable. ## +################################################################################ + +ifneq ($(origin ESMFMKFILE), environment) +$(error Environment variable ESMFMKFILE was not set.) +endif + +include $(ESMFMKFILE) + +################################################################################ +################################################################################ + +.SUFFIXES: .f90 .F90 .c .C + +%.o : %.f90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREENOCPP) $< + +%.o : %.F90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $< + +%.o : %.c + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + +%.o : %.C + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + + +# ----------------------------------------------------------------------------- +externalApp: externalApp.o atmDA.o ocnDA.o nuopc_da.o esm.o atm.o ocn.o + $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS) + +# module dependencies: +externalApp.o: esm.o atmDA.o ocnDA.o nuopc_da.o +atmDA.o: nuopc_da.o +ocnDA.o: nuopc_da.o +esm.o: atm.o ocn.o + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +.PHONY: dust clean distclean info edit +dust: + rm -f PET*.ESMF_LogFile *.nc *.stdout +clean: + rm -f externalApp *.o *.mod +distclean: dust clean + +info: + @echo ================================================================== + @echo ESMFMKFILE=$(ESMFMKFILE) + @echo ================================================================== + @cat $(ESMFMKFILE) + @echo ================================================================== + +edit: + nedit externalApp.F90 atmDA.F90 ocnDA.F90 nuopc_da.F90 esm.F90 atm.F90 ocn.F90 & + +run: + $(ESMF_INTERNAL_MPIRUN) -np 4 ./externalApp diff --git a/ExternalDriverAPIWeakCplDAProto/README b/ExternalDriverAPIWeakCplDAProto/README new file mode 100644 index 00000000..b81e3342 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/README @@ -0,0 +1,49 @@ +README for External Driver API NUOPC for weakly coupled DA prototype +-------------------------------------------------------------------- + +A simple two model coupled system with a driver, layered with the nuopc-da +interface, allowing access by two DA components (for ATM and OCN). +The time step method implemented by the nuopc-da layer supports independent +time steps on different MPI ranks. +The nuopc-da interface is using the "External" Driver API to drive the coupled +system. Fields are connected between the coupled system and the DA components. + +Description: + + A two-way coupled system with a single driver (ESM) and two model components + (ATM, OCN). + + The ESM driver uses explicitly constructed petLists when adding the two + model components. + + The ESM driver component uses the default run sequence to implement coupling + between ATM and OCN components. + + The application layer splits the MPI_COMM_WORLD communicator for ATM DA (lower + half of ranks) and OCN DA (upper half of ranks). Both DA components access the + coupled system through the nuopc-da interface layer. + +Build: + - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF + installation. + - gmake + +Execution: + - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. + - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) + +Output: + - PET*.Log files containing compliance checker output if turned on. + - The prototype outputs time stepping information to stdout. + +Code structure: + - Makefile - Makefile that is based on the standard esmf.mk mechanism. + - atm.F90 - The ATM component, specializing generic NUOPC_Model. + - ocn.F90 - The OCN component, specializing generic NUOPC_Model. + - esm.F90 - The Earth System Model (ESM) component, specializing generic + NUOPC_Driver. Define partial petLists for ATM and OCN. + - nuopc_da.F90- NUOPC-DA interface layer. + - atmDA.F90 - ATM DA component on a subset of ranks. + - ocnDA.F90 - OCN DA component on a subset of ranks. + - externalApp.F90 - Application layer setting up the NUOPC-DA interface, and + both DA components. diff --git a/ExternalDriverAPIWeakCplDAProto/atm.F90 b/ExternalDriverAPIWeakCplDAProto/atm.F90 new file mode 100644 index 00000000..22914385 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/atm.F90 @@ -0,0 +1,391 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module ATM + + !----------------------------------------------------------------------------- + ! ATM Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_CheckImport, & + specRoutine=NUOPC_NoOp, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + +#if 1 + ! importable field: sea_surface_temperature + call NUOPC_Advertise(importState, & + StandardName="sea_surface_temperature", name="sst", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + +#if 1 + ! importable field: precipitation_flux + call NUOPC_Advertise(importState, & + StandardName="precipitation_flux", name="precip", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + + ! exportable field: air_pressure_at_sea_level + call NUOPC_Advertise(exportState, & + StandardName="air_pressure_at_sea_level", name="pmsl", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_net_downward_shortwave_flux + call NUOPC_Advertise(exportState, & + StandardName="surface_net_downward_longwave_flux", name="rsnl", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_net_downward_shortwave_flux + call NUOPC_Advertise(exportState, & + StandardName="surface_net_downward_shortwave_flux", name="rsns", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_downward_heat_flux_in_air + call NUOPC_Advertise(exportState, & + StandardName="surface_downward_heat_flux_in_air", name="sdhf", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_Grid) :: gridIn + type(ESMF_Grid) :: gridOut + real(ESMF_KIND_R8), pointer :: dataPtr(:,:), lonPtr(:), latPtr(:) + integer :: i,j + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create a Grid object for Fields +#define LAT_LON_GRID +#ifdef LAT_LON_GRID + gridIn = ESMF_GridCreate1PeriDimUfrm(maxIndex=(/100, 50/), & + minCornerCoord=(/0._ESMF_KIND_R8, -60._ESMF_KIND_R8/), & + maxCornerCoord=(/360._ESMF_KIND_R8, 80._ESMF_KIND_R8/), & + staggerLocList=(/ESMF_STAGGERLOC_CENTER/), rc=rc) +#else + gridIn = ESMF_GridCreateCubedSphere(tileSize=16, name="CubedSphere", & + staggerLocList=(/ESMF_STAGGERLOC_CENTER/), rc=rc) +#endif + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + gridOut = gridIn ! for now out same as in + + ! conditionally realize the fields in the importState + call NUOPC_Realize(importState, grid=gridIn, typekind=ESMF_TYPEKIND_R8, & + selection="realize_connected_remove_others", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: air_pressure_at_sea_level + field = ESMF_FieldCreate(name="pmsl", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! fill export with some data + call ESMF_FieldFill(field, dataFillScheme="sincos", member=1, rc=rc) + + ! exportable field: surface_net_downward_longwave_flux + field = ESMF_FieldCreate(name="rsnl", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! fill export with some data + call ESMF_FieldFill(field, dataFillScheme="sincos", member=2, rc=rc) + + ! exportable field: surface_net_downward_shortwave_flux + field = ESMF_FieldCreate(name="rsns", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! fill export with some data + call ESMF_FieldFill(field, dataFillScheme="sincos", member=3, rc=rc) + + ! exportable field: surface_downward_heat_flux_in_air + field = ESMF_FieldCreate(name="sdhf", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! fill export with some data + call ESMF_FieldFill(field, dataFillScheme="sincos", member=4, rc=rc) + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_StateItem_Flag) :: itemType + real(ESMF_KIND_R8), pointer :: dataPtr(:,:) + integer :: i,j + character(len=160) :: msgString + integer, save :: slice=1 + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set by default, + ! its timeStep is equal to the parent timeStep. As a consequence the + ! currTime + timeStep is equal to the stopTime of the internal Clock + ! for this call of the Advance() routine. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="------>Advancing ATM from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockPrint(clock, options="stopTime", & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! update the "pmsl" field + call ESMF_StateGet(exportState, itemName="pmsl", itemType=itemType, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + if (itemType /= ESMF_STATEITEM_NOTFOUND) then + call ESMF_StateGet(exportState, itemName="pmsl", field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldGet(field, farrayPtr=dataPtr, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! dummy calculation to make the export field more interesting + do j=lbound(dataPtr,2),ubound(dataPtr,2) + do i=lbound(dataPtr,1),ubound(dataPtr,1) + dataPtr(i,j) = dataPtr(i,j) * 1.1 + enddo + enddo + endif + + ! update the "rsns" field + call ESMF_StateGet(exportState, itemName="rsns", itemType=itemType, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + if (itemType /= ESMF_STATEITEM_NOTFOUND) then + call ESMF_StateGet(exportState, itemName="rsns", field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldGet(field, farrayPtr=dataPtr, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! dummy calculation to make the export field more interesting + do j=lbound(dataPtr,2),ubound(dataPtr,2) + do i=lbound(dataPtr,1),ubound(dataPtr,1) + dataPtr(i,j) = dataPtr(i,j) * 1.1 + enddo + enddo + endif + +#ifdef LAT_LON_GRID + ! write out the Fields in the importState + call NUOPC_Write(importState, fileNamePrefix="field_atm_import_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! write out the Fields in the exportState + call NUOPC_Write(exportState, fileNamePrefix="field_atm_export_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + ! advance the time slice counter + slice = slice + 1 + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/ExternalDriverAPIWeakCplDAProto/atmDA.F90 b/ExternalDriverAPIWeakCplDAProto/atmDA.F90 new file mode 100644 index 00000000..d57f5a54 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/atmDA.F90 @@ -0,0 +1,90 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module atmDA + + !----------------------------------------------------------------------------- + ! ATM DA Code + !----------------------------------------------------------------------------- + + use MPI + use ESMF + use NUOPC + + use nuopc_da, only: & + nuopc_da_connect => connect, & + nuopc_da_step => step + + implicit none + + private + + public exec + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine exec(comm) + integer :: comm + + integer :: rc + + call ESMF_LogWrite("Starting into ATM DA code", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Connect DA code with NUOPC system top component + call nuopc_da_connect(toNuopcTopStandardNames=(/"precipitation_flux"/), & + fmNuopcTopStandardNames=(/"surface_net_downward_shortwave_flux"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !--> explicitly unrolled time loop to demonstrate stepping more clearly + + ! take 15min timestep + call nuopc_da_step(tStart="2010-06-01T00:00:00", & + tFinal="2010-06-01T00:15:00", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! take 30min timestep + call nuopc_da_step(tStart="2010-06-01T00:15:00", & + tFinal="2010-06-01T00:45:00", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! take 15min timestep + call nuopc_da_step(tStart="2010-06-01T00:45:00", & + tFinal="2010-06-01T01:00:00", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !<-- end of explicitly unrolled time loop + + call ESMF_LogWrite("Finished with ATM DA code", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + end subroutine + +end module diff --git a/ExternalDriverAPIWeakCplDAProto/esm.F90 b/ExternalDriverAPIWeakCplDAProto/esm.F90 new file mode 100644 index 00000000..91fd57c9 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/esm.F90 @@ -0,0 +1,220 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module ESM + + !----------------------------------------------------------------------------- + ! Code that specializes generic ESM Component code. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Driver, & + driverSS => SetServices + + use ATM, only: atmSS => SetServices + use OCN, only: ocnSS => SetServices + + use NUOPC_Connector, only: cplSS => SetServices + + implicit none + + private + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Driver + call NUOPC_CompDerive(driver, driverSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize driver + call NUOPC_CompSpecialize(driver, specLabel=label_SetModelServices, & + specRoutine=SetModelServices, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set driver verbosity + call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetModelServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + ! local variables + type(ESMF_Grid) :: grid + type(ESMF_Field) :: field + type(ESMF_Time) :: startTime + type(ESMF_Time) :: stopTime + type(ESMF_TimeInterval) :: timeStep + logical :: clockIsPresent + type(ESMF_Clock) :: internalClock + type(ESMF_GridComp) :: child + type(ESMF_CplComp) :: connector + integer :: petCount, i + integer, allocatable :: petList(:) + + rc = ESMF_SUCCESS + + ! get the petCount + call ESMF_GridCompGet(driver, petCount=petCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ATM with petList on first half of PETs + allocate(petList(petCount/2)) + do i=1, petCount/2 + petList(i) = i-1 ! PET labeling goes from 0 to petCount-1 + enddo + call NUOPC_DriverAddComp(driver, "ATM", atmSS, & + petList=petList, comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="low", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(petList) + + ! SetServices for OCN with petList on second half of PETs + allocate(petList(petCount/2)) + do i=1, petCount/2 + petList(i) = petCount/2 + i-1 ! PET labeling goes from 0 to petCount-1 + enddo + call NUOPC_DriverAddComp(driver, "OCN", ocnSS, & + petList=petList, comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="low", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + deallocate(petList) + + ! SetServices for atm2ocn + call NUOPC_DriverAddComp(driver, srcCompLabel="ATM", dstCompLabel="OCN", & + compSetServicesRoutine=cplSS, comp=connector, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(connector, name="Verbosity", value="low", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! SetServices for ocn2atm + call NUOPC_DriverAddComp(driver, srcCompLabel="OCN", dstCompLabel="ATM", & + compSetServicesRoutine=cplSS, comp=connector, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(connector, name="Verbosity", value="low", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! conditionally set the driver clock + call ESMF_GridCompGet(driver, clockIsPresent=clockIsPresent, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + if (.not.clockIsPresent) then + ! set the driver clock here + call ESMF_TimeIntervalSet(timeStep, m=15, rc=rc) ! 15 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimeSet(startTime, yy=2010, mm=6, dd=1, h=0, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimeSet(stopTime, yy=2010, mm=6, dd=1, h=1, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + internalClock = ESMF_ClockCreate(name="Application Clock", & + timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_GridCompSet(driver, clock=internalClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + else + call ESMF_TimeIntervalSet(timeStep, m=5, rc=rc) ! 5 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_GridCompGet(driver, clock=internalClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_ClockSet(internalClock, timeStep=timeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + endif + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/ExternalDriverAPIWeakCplDAProto/externalApp.F90 b/ExternalDriverAPIWeakCplDAProto/externalApp.F90 new file mode 100644 index 00000000..9d7f9127 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/externalApp.F90 @@ -0,0 +1,64 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +program externalApp + + !----------------------------------------------------------------------------- + ! Generic external application driver + !----------------------------------------------------------------------------- + + use MPI + + use ESMF + use NUOPC + use ESM, only: esmSS => SetServices + + use nuopc_da, only: & + nuopc_da_init => init, & + nuopc_da_final => final + + use atmDA, only: atmDAexec => exec + use ocnDA, only: ocnDAexec => exec + + implicit none + + integer :: rc + integer :: size, rank + integer :: splitComm + + ! Initialize the NUOPC-DA interface + call nuopc_da_init(nuopcTopSetServices=esmSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Split up the MPI_COMM_WORLD into atmDA (first half) and ocnDA (second half) + ! of MPI ranks. Call into the respective DA routine + call MPI_Comm_size(MPI_COMM_WORLD, size, ierror=rc) + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierror=rc) + if (rank < size/2) then + ! atm DA processes + call MPI_Comm_split(MPI_COMM_WORLD, 1, rank, splitComm, ierror=rc) + call atmDAexec(comm=splitComm) + else + ! ocn DA processes + call MPI_Comm_split(MPI_COMM_WORLD, 2, rank, splitComm, ierror=rc) + call ocnDAexec(comm=splitComm) + endif + + ! Finalize the NUOPC-DA interface + call nuopc_da_final(rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + +end program diff --git a/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 new file mode 100644 index 00000000..5f0edbe2 --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/nuopc_da.F90 @@ -0,0 +1,413 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module nuopc_da + + !----------------------------------------------------------------------------- + ! NUOPC - DA interface code + !----------------------------------------------------------------------------- + + use MPI + use ESMF + use NUOPC + + implicit none + + private + + type(ESMF_GridComp) :: nuopcTop + type(ESMF_State) :: toNuopcTop, fmNuopcTop + type(ESMF_Clock) :: clock + + public init, connect, step, final + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine init(nuopcTopSetServices, rc) + interface + recursive subroutine nuopcTopSetServices(gridcomp, rc) + use ESMF + implicit none + type(ESMF_GridComp) :: gridcomp ! must not be optional + integer, intent(out) :: rc ! must not be optional + end subroutine + end interface + integer, intent(out) :: rc + + integer :: urc + type(ESMF_Time) :: startTime, stopTime + type(ESMF_TimeInterval) :: timeStep + + ! Initialize ESMF + call ESMF_Initialize(logkindflag=ESMF_LOGKIND_MULTI, & + defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + call ESMF_LogSet(flush=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + call ESMF_LogWrite("INIT...", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + ! Create the application Clock + call ESMF_TimeIntervalSet(timeStep, m=15, rc=rc) ! 15 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + call ESMF_TimeSet(startTime, yy=2010, mm=6, dd=1, h=0, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + call ESMF_TimeSet(stopTime, yy=2010, mm=6, dd=1, h=1, m=0, & + calkindflag=ESMF_CALKIND_GREGORIAN, rc=rc) + clock = ESMF_ClockCreate(name="Application Clock", & + timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + ! Create the external level import/export States + ! NOTE: The "stateintent" must be specified, and it must be set from the + ! perspective of the external level: + ! -> state holding fields exported by the external level to the ESM component + toNuopcTop = ESMF_StateCreate(stateintent=ESMF_STATEINTENT_EXPORT, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + ! -> state holding fields imported by the external level from the ESM component + fmNuopcTop = ESMF_StateCreate(stateintent=ESMF_STATEINTENT_IMPORT, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + ! Create the earth system Component + nuopcTop = ESMF_GridCompCreate(name="nuopcTop", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + ! SetServices for the earth system Component + call ESMF_GridCompSetServices(nuopcTop, nuopcTopSetServices, userRc=urc, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine connect(toNuopcTopStandardNames, fmNuopcTopStandardNames, rc) + character(*), intent(in), optional :: toNuopcTopStandardNames(:) + character(*), intent(in), optional :: fmNuopcTopStandardNames(:) + integer, intent(out) :: rc + + integer :: urc, phase, i + type(ESMF_Field) :: field + type(ESMF_StateItem_Flag) :: itemType + + if (present(toNuopcTopStandardNames)) then + call NUOPC_Advertise(toNuopcTop, StandardNames=toNuopcTopStandardNames, & + TransferOfferGeomObject="cannot provide", SharePolicyField="share", & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + endif + + if (present(fmNuopcTopStandardNames)) then + call NUOPC_Advertise(fmNuopcTop, StandardNames=fmNuopcTopStandardNames, & + TransferOfferGeomObject="cannot provide", SharePolicyField="share", & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + endif + + ! Call "ExternalAdvertise" Initialize for the earth system Component + ! -> This method removes any fields previously advertised in the states, + ! but ended up not connected. + call NUOPC_CompSearchPhaseMap(nuopcTop, methodflag=ESMF_METHOD_INITIALIZE, & + phaseLabel=label_ExternalAdvertise, phaseIndex=phase, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + call ESMF_GridCompInitialize(nuopcTop, phase=phase, & + importState=toNuopcTop, exportState=fmNuopcTop, & + userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Call "ExternalRealize" Initialize for the earth system Component + ! -> This method realizes any fields previously advertised in the states, + ! for which enough information is available to realize (e.g. due to sharing, + ! or transfers). + call NUOPC_CompSearchPhaseMap(nuopcTop, methodflag=ESMF_METHOD_INITIALIZE, & + phaseLabel=label_ExternalRealize, phaseIndex=phase, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + call ESMF_GridCompInitialize(nuopcTop, phase=phase, & + importState=toNuopcTop, exportState=fmNuopcTop, & + userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Fill fields provided from this level to nuopcTop with data + if (present(toNuopcTopStandardNames)) then + do i=1, size(toNuopcTopStandardNames) + call ESMF_StateGet(toNuopcTop, itemName=toNuopcTopStandardNames(i), & + itemType=itemType, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (itemType /= ESMF_STATEITEM_NOTFOUND) then + call ESMF_StateGet(toNuopcTop, itemName=toNuopcTopStandardNames(i), & + field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + call ESMF_FieldFill(field, dataFillScheme="sincos", member=1, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + endif + enddo + endif + + ! Call "ExternalDataInit" Initialize for the earth system Component + call NUOPC_CompSearchPhaseMap(nuopcTop, methodflag=ESMF_METHOD_INITIALIZE, & + phaseLabel=label_ExternalDataInit, phaseIndex=phase, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + call ESMF_GridCompInitialize(nuopcTop, phase=phase, & + importState=toNuopcTop, exportState=fmNuopcTop, & + userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Write out the Fields in the toNuopcTop after initialize + call NUOPC_Write(toNuopcTop, & + fileNamePrefix="field_toNuopcTop_init_", & + overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + ! Write out the Fields in the fmNuopcTop after initialize + call NUOPC_Write(fmNuopcTop, & + fileNamePrefix="field_fmNuopcTop_init_", & + overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine step(tStart, tFinal, rc) + ! Take a step forward from start time to final time. + ! Different MPI ranks are allowed to call with different values for times. + ! On each MPI rank the method will block until tFinal has been reached by + ! the underlying coupled NUOPC system. + character(*), intent(in) :: tStart ! start time + character(*), intent(in) :: tFinal ! final time + integer, intent(out) :: rc + + type(ESMF_Time) :: startTime, stopTime, currTime + type(ESMF_TimeInterval) :: timeStep + integer :: urc, i + integer(ESMF_KIND_I8) :: sec(1), secMin(1) + type(ESMF_VM) :: vm + integer, save :: slice=1 + + call ESMF_VMGetCurrent(vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Convert string to ESMF_Time + call ESMF_TimeSet(startTime, timeString=tStart, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Convert string to ESMF_Time + call ESMF_TimeSet(stopTime, timeString=tFinal, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Set the currTime and stopTime on Clock + call ESMF_ClockSet(clock, currTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Loop until the current MPI rank has reached the local stopTime, taking + ! time steps that are coordinated across all MPI ranks. + do while (.not.ESMF_ClockIsStopTime(clock, rc=rc)) + + ! Get the currTime as the basis for the next step + call ESMF_ClockGet(clock, currTime=currTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Check for consistency of currTime across all MPI ranks + call ESMF_TimeGet(currTime, s_i8=sec(1), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + call ESMF_VMAllReduce(vm, sendData=sec, recvData=secMin, & + count=1, reduceflag=ESMF_REDUCE_MIN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + if (sec(1) > secMin(1)) then + call ESMF_LogSetError(ESMF_RC_INTNRL_INCONS, & + msg="Inconsistent currTime detected arcoss MPI ranks!", & + line=__LINE__, & + file=__FILE__, rcToReturn=rc) + return ! bail out + endif + + ! Determine local timeStep + timeStep = stopTime - currTime + + ! Determine consistent timeStep across all MPI ranks + call ESMF_TimeIntervalGet(timeStep, s_i8=sec(1), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + call ESMF_VMAllReduce(vm, sendData=sec, recvData=secMin, & + count=1, reduceflag=ESMF_REDUCE_MIN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + call ESMF_TimeIntervalSet(timeStep, s_i8=secMin(1), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Set the consistent timeStep in Clock for the next step + call ESMF_ClockSet(clock, timeStep=timeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Timestamp the fields in the toNuopcTop state + call NUOPC_SetTimestamp(toNuopcTop, clock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Run the earth system Component: i.e. step nuopcTop forward by timestep + call ESMF_GridCompRun(nuopcTop, clock=clock, & + importState=toNuopcTop, exportState=fmNuopcTop, & + userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Write out the Fields in the toNuopcTop + call NUOPC_Write(toNuopcTop, & + fileNamePrefix="field_toNuopcTop_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Write out the Fields in the fmNuopcTop after initialize + call NUOPC_Write(fmNuopcTop, & + fileNamePrefix="field_fmNuopcTop_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Increment the time slice counter + slice = slice + 1 + + ! Advance the clock + call ESMF_ClockAdvance(clock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + enddo + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine final(rc) + integer, intent(out) :: rc + + integer :: urc, i + + ! Finalize the nuopcTop Component + call ESMF_GridCompFinalize(nuopcTop, & + importState=toNuopcTop, exportState=fmNuopcTop, & + userRc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__, rcToReturn=rc)) return + + ! Destroy the nuopcTop Component + call ESMF_GridCompDestroy(nuopcTop, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + call ESMF_LogWrite("...FINAL", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) return + + ! Finalize ESMF + call ESMF_Finalize(rc=rc) + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/ExternalDriverAPIWeakCplDAProto/ocn.F90 b/ExternalDriverAPIWeakCplDAProto/ocn.F90 new file mode 100644 index 00000000..8cdd521d --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/ocn.F90 @@ -0,0 +1,380 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module OCN + + !----------------------------------------------------------------------------- + ! OCN Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_SetClock, & + specRoutine=SetClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: air_pressure_at_sea_level + call NUOPC_Advertise(importState, & + StandardName="air_pressure_at_sea_level", name="pmsl", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: surface_net_downward_shortwave_flux + call NUOPC_Advertise(importState, & + StandardName="surface_net_downward_longwave_flux", name="rsnl", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: surface_net_downward_shortwave_flux + call NUOPC_Advertise(importState, & + StandardName="surface_net_downward_shortwave_flux", name="rsns", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: surface_downward_heat_flux_in_air + call NUOPC_Advertise(importState, & + StandardName="surface_downward_heat_flux_in_air", name="sdhf", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_temperature + call NUOPC_Advertise(exportState, & + StandardName="sea_surface_temperature", name="sst", & + SharePolicyField="share", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_Grid) :: gridIn + type(ESMF_Grid) :: gridOut + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create a Grid object for Fields + gridIn = ESMF_GridCreate1PeriDimUfrm(maxIndex=(/100, 50/), & + minCornerCoord=(/0._ESMF_KIND_R8, -60._ESMF_KIND_R8/), & + maxCornerCoord=(/360._ESMF_KIND_R8, 80._ESMF_KIND_R8/), & + staggerLocList=(/ESMF_STAGGERLOC_CENTER/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + gridOut = gridIn ! for now out same as in + + ! importable field: air_pressure_at_sea_level + field = ESMF_FieldCreate(name="pmsl", grid=gridIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: surface_net_downward_longwave_flux + field = ESMF_FieldCreate(name="rsnl", grid=gridIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: surface_net_downward_shortwave_flux + field = ESMF_FieldCreate(name="rsns", grid=gridIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! importable field: surface_downward_heat_flux_in_air + field = ESMF_FieldCreate(name="sdhf", grid=gridIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: sea_surface_temperature, realize conditionally and fill + call NUOPC_Realize(exportState, grid=gridOut, fieldName="sst", & + typekind=ESMF_TYPEKIND_R8, selection="realize_connected_remove_others", & + dataFillScheme="sincos", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetClock(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_TimeInterval) :: stabilityTimeStep + + rc = ESMF_SUCCESS + + ! query for clock + call NUOPC_ModelGet(model, modelClock=clock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! initialize internal clock + ! here: parent Clock and stability timeStep determine actual model timeStep + !TODO: stabilityTimeStep should be read in from configuation + !TODO: or computed from internal Grid information + call ESMF_TimeIntervalSet(stabilityTimeStep, m=5, rc=rc) ! 5 minute steps + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSetClock(model, clock, stabilityTimeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_Time) :: currTime + type(ESMF_TimeInterval) :: timeStep + type(ESMF_Field) :: field + type(ESMF_StateItem_Flag) :: itemType + real(ESMF_KIND_R8), pointer :: dataPtr(:,:) + integer :: i,j + character(len=160) :: msgString + integer, save :: slice=1 + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call NUOPC_ModelGet(model, modelClock=clock, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set in SetClock(), + ! its timeStep is likely smaller than the parent timeStep. As a consequence + ! the time interval covered by a single parent timeStep will result in + ! multiple calls to the Advance() routine. Every time the currTime + ! will come in by one internal timeStep advanced. This goes until the + ! stopTime of the internal Clock has been reached. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="------>Advancing OCN from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockGet(clock, currTime=currTime, timeStep=timeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_TimePrint(currTime + timeStep, & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! update the "sst" field + call ESMF_StateGet(exportState, itemName="sst", itemType=itemType, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + if (itemType /= ESMF_STATEITEM_NOTFOUND) then + call ESMF_StateGet(exportState, itemName="sst", field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_FieldGet(field, farrayPtr=dataPtr, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! dummy calculation to make the export field more interesting + do j=lbound(dataPtr,2),ubound(dataPtr,2) + do i=lbound(dataPtr,1),ubound(dataPtr,1) + dataPtr(i,j) = dataPtr(i,j) * 1.1 + enddo + enddo + endif + + ! write out the Fields in the importState + call NUOPC_Write(importState, fileNamePrefix="field_ocn_import_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! write out the Fields in the exportState + call NUOPC_Write(exportState, fileNamePrefix="field_ocn_export_", & + timeslice=slice, overwrite=.true., relaxedFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + ! advance the time slice counter + slice = slice + 1 + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 b/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 new file mode 100644 index 00000000..f858372c --- /dev/null +++ b/ExternalDriverAPIWeakCplDAProto/ocnDA.F90 @@ -0,0 +1,82 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module ocnDA + + !----------------------------------------------------------------------------- + ! OCN DA Code + !----------------------------------------------------------------------------- + + use MPI + use ESMF + use NUOPC + + use nuopc_da, only: & + nuopc_da_connect => connect, & + nuopc_da_step => step + + implicit none + + private + + public exec + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine exec(comm) + integer :: comm + + integer :: rc + + call ESMF_LogWrite("Starting into OCN DA code", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Connect DA code with NUOPC system top component + call nuopc_da_connect(fmNuopcTopStandardNames=(/"sea_surface_temperature"/), & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !--> explicitly unrolled time loop to demonstrate stepping more clearly + + ! take 30min timestep + call nuopc_da_step(tStart="2010-06-01T00:00:00", & + tFinal="2010-06-01T00:30:00", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! take 30min timestep + call nuopc_da_step(tStart="2010-06-01T00:30:00", & + tFinal="2010-06-01T01:00:00", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !<-- end of explicitly unrolled time loop + + call ESMF_LogWrite("Finished with OCN DA code", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + end subroutine + +end module diff --git a/GenericMediatorProto/Makefile b/GenericMediatorProto/Makefile index 931b4f9a..d3ae0fd4 100644 --- a/GenericMediatorProto/Makefile +++ b/GenericMediatorProto/Makefile @@ -70,4 +70,4 @@ edit: nedit app.F90 driver.F90 mediator.F90 modelA.F90 modelB.F90 & run: - mpirun -np 4 ./app + $(ESMF_INTERNAL_MPIRUN) -np 4 ./app diff --git a/HierarchyProto/Makefile b/HierarchyProto/Makefile index 496c7a4f..7a1eff0e 100644 --- a/HierarchyProto/Makefile +++ b/HierarchyProto/Makefile @@ -71,4 +71,4 @@ edit: nedit esmApp.F90 esm.F90 atm.F90 dyn.F90 phy.F90 ocn.F90 & run: - mpirun -np 4 ./esmApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./esmApp diff --git a/NamespaceProto/Makefile b/NamespaceProto/Makefile index 9bbb9217..27bf4c1a 100644 --- a/NamespaceProto/Makefile +++ b/NamespaceProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driver.F90 atm.F90 med.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/NestingMultipleProto/Makefile b/NestingMultipleProto/Makefile index 18dcf1e2..ad7b7bcf 100644 --- a/NestingMultipleProto/Makefile +++ b/NestingMultipleProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driverComp.F90 advectDiffComp.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/NestingSingleProto/Makefile b/NestingSingleProto/Makefile index 18dcf1e2..ad7b7bcf 100644 --- a/NestingSingleProto/Makefile +++ b/NestingSingleProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driverComp.F90 advectDiffComp.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/NestingTelescopeMultipleProto/Makefile b/NestingTelescopeMultipleProto/Makefile index 18dcf1e2..ad7b7bcf 100644 --- a/NestingTelescopeMultipleProto/Makefile +++ b/NestingTelescopeMultipleProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driverComp.F90 advectDiffComp.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/SingleModelOpenMPProto/Makefile b/SingleModelOpenMPProto/Makefile index e73aa632..28f14f2e 100644 --- a/SingleModelOpenMPProto/Makefile +++ b/SingleModelOpenMPProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driver.F90 model.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/SingleModelOpenMPProto/driver.F90 b/SingleModelOpenMPProto/driver.F90 index 0b58f118..c4863840 100644 --- a/SingleModelOpenMPProto/driver.F90 +++ b/SingleModelOpenMPProto/driver.F90 @@ -98,19 +98,16 @@ subroutine SetModelServices(driver, rc) line=__LINE__, & file=__FILE__)) & return ! bail out +#if 1 call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=2, & rc=rc) ! expect 2 PEs per PET in the child VM if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & file=__FILE__)) & return ! bail out - call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/MinStackSize", & - value=16*1024*1024, rc=rc) ! stack size no less than 16MiB - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, & - file=__FILE__)) & - return ! bail out -#if 1 +#endif + +#if 0 call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/OpenMpHandling", & value="SET", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -118,7 +115,7 @@ subroutine SetModelServices(driver, rc) file=__FILE__)) & return ! bail out #endif -#if 1 +#if 0 call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/OpenMpNumThreads", & value=4, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -126,6 +123,20 @@ subroutine SetModelServices(driver, rc) file=__FILE__)) & return ! bail out #endif +#if 0 + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/ForceChildPthreads", & + value=.true., rc=rc) ! force child PETs to execute as Pthreads + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_InfoSet(info, key="/NUOPC/Hint/PePerPet/PthreadMinStackSize", & + value=16*1024*1024, rc=rc) ! stack size no less than 16MiB + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif ! SetServices for MODEL component call NUOPC_DriverAddComp(driver, "MODEL", modelSS, modelSVM, info=info, & diff --git a/SingleModelOpenMPUnawareProto/Makefile b/SingleModelOpenMPUnawareProto/Makefile new file mode 100644 index 00000000..b08cfc55 --- /dev/null +++ b/SingleModelOpenMPUnawareProto/Makefile @@ -0,0 +1,73 @@ +# GNU Makefile template for user ESMF application + +################################################################################ +################################################################################ +## This Makefile must be able to find the "esmf.mk" Makefile fragment in the ## +## 'include' line below. Following the ESMF User's Guide, a complete ESMF ## +## installation should ensure that a single environment variable "ESMFMKFILE" ## +## is made available on the system. This variable should point to the ## +## "esmf.mk" file. ## +## ## +## This example Makefile uses the "ESMFMKFILE" environment variable. ## +## ## +## If you notice that this Makefile cannot find variable ESMFMKFILE then ## +## please contact the person responsible for the ESMF installation on your ## +## system. ## +## As a work-around you can simply hardcode the path to "esmf.mk" in the ## +## include line below. However, doing so will render this Makefile a lot less ## +## flexible and non-portable. ## +################################################################################ + +ifneq ($(origin ESMFMKFILE), environment) +$(error Environment variable ESMFMKFILE was not set.) +endif + +include $(ESMFMKFILE) + +################################################################################ +################################################################################ + +.SUFFIXES: .f90 .F90 .c .C + +%.o : %.f90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREENOCPP) $< + +%.o : %.F90 + $(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR) $< + +%.o : %.c + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + +%.o : %.C + $(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) $(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $< + + +# ----------------------------------------------------------------------------- +mainApp: mainApp.o driver.o model.o + $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS) + +# module dependencies: +mainApp.o: driver.o +driver.o: model.o + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +.PHONY: dust clean distclean info edit +dust: + rm -f PET*.ESMF_LogFile *.nc *.stdout +clean: + rm -f mainApp *.o *.mod +distclean: dust clean + +info: + @echo ================================================================== + @echo ESMFMKFILE=$(ESMFMKFILE) + @echo ================================================================== + @cat $(ESMFMKFILE) + @echo ================================================================== + +edit: + nedit mainApp.F90 driver.F90 model.F90 & + +run: + env OMP_NUM_THREADS=3 $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/SingleModelOpenMPUnawareProto/README b/SingleModelOpenMPUnawareProto/README new file mode 100644 index 00000000..e9fc385a --- /dev/null +++ b/SingleModelOpenMPUnawareProto/README @@ -0,0 +1,45 @@ +README for single Model OpenMP Unaware NUOPC prototype +------------------------------------------------------- + +Very simple NUOPC app with a Driver controlling a single Model component. The +model component uses OpenMP if compiled with support, but does so outside of +NUOPC control. This is called NUOPC-unaware threading or resource control. + +Description: + + A single model component driven by a single driver component. + + The driver assumes that each PET launched is a master thread for its own + OpenMP thead team. NUOPC/ESMF is unaware of the OpenMP threading inside of + the model component. The number of OpenMP threads is set via the standard + OMP_NUM_THREADS environment variable, which is set equal to 3 inside the + Makefile run target. + + The ESM driver component uses the default run sequence to drive the MODEL + component. + + No connector components are present. + + The MODEL component uses a simple two-phase initialization, + consisting of advertise and realize. + + +Build: + - Set environment variable ESMFMKFILE to point to the esmf.mk of your ESMF + installation. + - gmake + +Execution: + - Optionally set environment variable ESMF_RUNTIME_COMPLIANCECHECK to ON. + - Set OMP_NUM_THREADS if directly calling into mpirun as per below. + - mpirun -np X ./esmApp (where X is the total number of PETs, typically 4) + +Output: + - PET*.Log files containing compliance checker output if turned on. + - The prototype outputs time stepping information to stdout. + +Code structure: + - Makefile - Makefile that is based on the standard esmf.mk mechanism. + - model.F90 - The MODEL component, specializing generic NUOPC_Model. + - driver.F90 - Driver specializing NUOPC_Driver, driving the single MODEL. + - mainApp.F90 - Main application. diff --git a/SingleModelOpenMPUnawareProto/driver.F90 b/SingleModelOpenMPUnawareProto/driver.F90 new file mode 100644 index 00000000..9f58bf12 --- /dev/null +++ b/SingleModelOpenMPUnawareProto/driver.F90 @@ -0,0 +1,190 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module driver + + !----------------------------------------------------------------------------- + ! Code that specializes generic NUOPC_Driver + !----------------------------------------------------------------------------- + + use MPI + use ESMF + use NUOPC + use NUOPC_Driver, & + driverSS => SetServices + + use MODEL, only: & + modelSS => SetServices + + implicit none + + private + + ! private module data --> ONLY PARAMETERS + integer, parameter :: stepCount = 5 + real(ESMF_KIND_R8), parameter :: stepTime = 30.D0 ! step time [s] + ! should be parent step + + public SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Driver + call NUOPC_CompDerive(driver, driverSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize driver + call NUOPC_CompSpecialize(driver, specLabel=label_SetModelServices, & + specRoutine=SetModelServices, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set driver verbosity + call NUOPC_CompAttributeSet(driver, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine SetModelServices(driver, rc) + type(ESMF_GridComp) :: driver + integer, intent(out) :: rc + + ! local variables + type(ESMF_GridComp) :: child + type(ESMF_CplComp) :: connector + type(ESMF_Time) :: startTime + type(ESMF_Time) :: stopTime + type(ESMF_TimeInterval) :: timeStep + type(ESMF_Clock) :: internalClock + + ! - diagnostics - + type(ESMF_VM) :: vm + logical :: isFlag + character(80) :: msgString + integer :: mpiComm, size, ierr + + rc = ESMF_SUCCESS + + ! SetServices for MODEL component + call NUOPC_DriverAddComp(driver, "MODEL", modelSS, comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompAttributeSet(child, name="Verbosity", value="high", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! - diagnostics - + isFlag = ESMF_GridCompIsPetLocal(child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + write(msgString,*) "GridCompIsPetLocal: ", isFlag + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_GridCompGet(child, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + isFlag = ESMF_VMIsCreated(vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + write(msgString,*) "VmIsCreated: ", isFlag + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_VMGet(vm, mpiCommunicator=mpiComm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + if (mpiComm==MPI_COMM_NULL) then + write(msgString,*) "MPI_COMM_NULL" + else + call MPI_Comm_size(mpiComm, size, ierr) + write(msgString,*) "valid MPI_COMM with size=",size + endif + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! set the driver clock + call ESMF_TimeSet(startTime, s = 0, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_TimeSet(stopTime, s_r8 = stepTime * stepCount, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_TimeIntervalSet(timeStep, s_r8 = stepTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + internalClock = ESMF_ClockCreate(name="Driver Clock", & + timeStep=timeStep, startTime=startTime, stopTime=stopTime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_GridCompSet(driver, clock=internalClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/SingleModelOpenMPUnawareProto/mainApp.F90 b/SingleModelOpenMPUnawareProto/mainApp.F90 new file mode 100644 index 00000000..eb5ee6ff --- /dev/null +++ b/SingleModelOpenMPUnawareProto/mainApp.F90 @@ -0,0 +1,120 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +#define EXPLICIT_MPI_INIT + +program mainApp + + !----------------------------------------------------------------------------- + ! Generic ESMF Main + !----------------------------------------------------------------------------- + + use ESMF +#ifdef EXPLICIT_MPI_INIT + use MPI +#endif + + use driver, only: & + driver_SS => SetServices + + implicit none + + integer :: rc, userRc + type(ESMF_GridComp) :: drvComp + + ! Initialize MPI/ESMF +#ifdef EXPLICIT_MPI_INIT + ! This prototype implements ESMF-aware resource management for threading. + ! Therefore must call ESMF_InitializePreMPI() before MPI_Init*() if later is + ! called explicitly from user level! + call ESMF_InitializePreMPI(rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + call MPI_Init_thread(MPI_THREAD_MULTIPLE, userRc, rc) +#endif + call ESMF_Initialize(defaultCalkind=ESMF_CALKIND_GREGORIAN, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_LogWrite("mainApp STARTING", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !----------------------------------------------------------------------------- + + ! -> CREATE THE DRIVER + drvComp = ESMF_GridCompCreate(name="driver", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! -> SET DRIVER SERVICES + call ESMF_GridCompSetServices(drvComp, driver_SS, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! INITIALIZE THE DRIVER + call ESMF_GridCompInitialize(drvComp, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! RUN THE DRIVER + call ESMF_GridCompRun(drvComp, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! FINALIZE THE DRIVER + call ESMF_GridCompFinalize(drvComp, userRc=userRc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + if (ESMF_LogFoundError(rcToCheck=userRc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + !----------------------------------------------------------------------------- + + call ESMF_LogWrite("mainApp FINISHED", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + ! Finalize ESMF + call ESMF_Finalize() + +end program diff --git a/SingleModelOpenMPUnawareProto/model.F90 b/SingleModelOpenMPUnawareProto/model.F90 new file mode 100644 index 00000000..cc49f64b --- /dev/null +++ b/SingleModelOpenMPUnawareProto/model.F90 @@ -0,0 +1,306 @@ +!============================================================================== +! Earth System Modeling Framework +! Copyright 2002-2021, University Corporation for Atmospheric Research, +! Massachusetts Institute of Technology, Geophysical Fluid Dynamics +! Laboratory, University of Michigan, National Centers for Environmental +! Prediction, Los Alamos National Laboratory, Argonne National Laboratory, +! NASA Goddard Space Flight Center. +! Licensed under the University of Illinois-NCSA License. +!============================================================================== + +module MODEL + + !----------------------------------------------------------------------------- + ! MODEL Component. + !----------------------------------------------------------------------------- + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! specialize model + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_DataInitialize, & + specRoutine=DataInitialize, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Enabeling the following macro, i.e. renaming it to WITHIMPORTFIELDS, + ! will result in a model component that advertise import Field dependencies. + ! In the single model case, where there isn't another model to satisfy these + ! dependencies, it is expected to be caught by the compatability checking. +#define WITHIMPORTFIELDS___disable +#ifdef WITHIMPORTFIELDS + ! importable field: sea_surface_temperature + call NUOPC_Advertise(importState, & + StandardName="sea_surface_temperature", name="sst", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + + ! exportable field: air_pressure_at_sea_level + call NUOPC_Advertise(exportState, & + StandardName="air_pressure_at_sea_level", name="pmsl", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_net_downward_shortwave_flux + call NUOPC_Advertise(exportState, & + StandardName="surface_net_downward_shortwave_flux", name="rsns", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + type(ESMF_Field) :: field + type(ESMF_Grid) :: gridIn + type(ESMF_Grid) :: gridOut + + rc = ESMF_SUCCESS + + ! query for importState and exportState + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create a Grid object for Fields + gridIn = ESMF_GridCreateNoPeriDimUfrm(maxIndex=(/10, 100/), & + minCornerCoord=(/10._ESMF_KIND_R8, 20._ESMF_KIND_R8/), & + maxCornerCoord=(/100._ESMF_KIND_R8, 200._ESMF_KIND_R8/), & + coordSys=ESMF_COORDSYS_CART, staggerLocList=(/ESMF_STAGGERLOC_CENTER/), & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + gridOut = gridIn ! for now out same as in + +#ifdef WITHIMPORTFIELDS + ! importable field: sea_surface_temperature + field = ESMF_FieldCreate(name="sst", grid=gridIn, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(importState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#endif + + ! exportable field: air_pressure_at_sea_level + field = ESMF_FieldCreate(name="pmsl", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! exportable field: surface_net_downward_shortwave_flux + field = ESMF_FieldCreate(name="rsns", grid=gridOut, & + typekind=ESMF_TYPEKIND_R8, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call NUOPC_Realize(exportState, field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_VMLogMemInfo(prefix="After Realize:", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine DataInitialize(model, rc) + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + integer, save :: inHere=1 + + rc = ESMF_SUCCESS + + if (inHere > 1) then + ! indicate that data initialization is complete (breaking out of init-loop) + call NUOPC_CompAttributeSet(model, & + name="InitializeDataComplete", value="true", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + endif + + inHere = inHere + 1 + + end subroutine + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) +!$ use omp_lib + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + type(ESMF_VM) :: vm + integer :: currentSsiPe + character(len=160) :: msgString + + rc = ESMF_SUCCESS + + ! query for clock, importState and exportState + call ESMF_GridCompGet(model, clock=clock, importState=importState, & + exportState=exportState, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! Now can use OpenMP for fine grained parallelism... + ! Here just write info about the PET-local OpenMP threads to Log. +!$omp parallel private(msgString, currentSsiPe) +!$omp critical +!$ call ESMF_VMGet(vm, currentSsiPe=currentSsiPe) +!$ write(msgString,'(A,I4,A,I4,A,I4,A,I4,A,I4)') & +!$ "thread_num=", omp_get_thread_num(), & +!$ " currentSsiPe=", currentSsiPe, & +!$ " num_threads=", omp_get_num_threads(), & +!$ " max_threads=", omp_get_max_threads(), & +!$ " num_procs=", omp_get_num_procs() +!$ call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) +!$omp end critical +!$omp end parallel + + ! HERE THE MODEL ADVANCES: currTime -> currTime + timeStep + + ! Because of the way that the internal Clock was set by default, + ! its timeStep is equal to the parent timeStep. As a consequence the + ! currTime + timeStep is equal to the stopTime of the internal Clock + ! for this call of the Advance() routine. + + call ESMF_ClockPrint(clock, options="currTime", & + preString="---->Advancing Model from: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + call ESMF_ClockPrint(clock, options="stopTime", & + preString="---------------------> to: ", unit=msgString, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + end subroutine + + !----------------------------------------------------------------------------- + +end module diff --git a/SingleModelProto/Makefile b/SingleModelProto/Makefile index e73aa632..28f14f2e 100644 --- a/SingleModelProto/Makefile +++ b/SingleModelProto/Makefile @@ -70,4 +70,4 @@ edit: nedit mainApp.F90 driver.F90 model.F90 & run: - mpirun -np 4 ./mainApp + $(ESMF_INTERNAL_MPIRUN) -np 4 ./mainApp diff --git a/testProtos.sh b/testProtos.sh index dab835f8..87bd7d77 100755 --- a/testProtos.sh +++ b/testProtos.sh @@ -10,8 +10,9 @@ # Licensed under the University of Illinois-NCSA License. #============================================================================== -MPIRUN="mpirun -np" -#MPIRUN="srun -n" +# Obtain ESMF_INTERNAL_MPIRUN from esmf.mk +command=`grep ESMF_INTERNAL_MPIRUN $ESMFMKFILE` +eval $command #TOOLRUN="valgrind --leak-check=full" @@ -21,12 +22,13 @@ function TestProto { ((count++)) testList[count]=$1 echo --------------------------------------------------------------------------- +date echo STARTING: $1 cd $1 -gmake distclean -gmake +make distclean +make set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -37,6 +39,7 @@ testResult[count]="FAIL" fi echo FINISHED: $1 cd .. +date echo --------------------------------------------------------------------------- echo } @@ -51,13 +54,14 @@ fi testList[count]=$1 read -ra ARGS <<< "$3" echo --------------------------------------------------------------------------- +date echo STARTING: $1 cd $1 -gmake distclean -gmake +make distclean +make for arg in "${ARGS[@]}"; do set -x -$MPIRUN 4 $TOOLRUN ./$2 $arg > $2.$arg.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 $arg > $2.$arg.stdout 2>&1 local result=$? set +x if [ $result -ne 0 ] @@ -73,21 +77,23 @@ testResult[count]="FAIL" fi echo FINISHED: $1 cd .. +date echo --------------------------------------------------------------------------- echo } function TestSelectProto { echo --------------------------------------------------------------------------- +date echo STARTING: $1 cd $1 -gmake distclean -gmake ATM=A OCN=A,B +make distclean +make ATM=A OCN=A,B echo "OCN_SELECT: A" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -97,13 +103,13 @@ else testResult[count]="FAIL" fi echo -gmake clean -gmake ATM=B OCN=A,B +make clean +make ATM=B OCN=A,B echo "OCN_SELECT: A" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -113,13 +119,13 @@ else testResult[count]="FAIL" fi echo -gmake clean -gmake ATM=A OCN=B +make clean +make ATM=A OCN=B echo "OCN_SELECT: B" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -130,23 +136,25 @@ testResult[count]="FAIL" fi echo FINISHED: $1 cd .. +date echo --------------------------------------------------------------------------- echo } function TestSelectExternalProto { echo --------------------------------------------------------------------------- +date echo STARTING: $1 cd $1 -gmake distclean +make distclean ./cleanSubs.csh ./buildSubs.csh -gmake ATM=A OCN=A,B,C +make ATM=A OCN=A,B,C echo "OCN_SELECT: A" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -156,13 +164,13 @@ else testResult[count]="FAIL" fi echo -gmake clean -gmake ATM=B OCN=A,B,C +make clean +make ATM=B OCN=A,B,C echo "OCN_SELECT: B" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -172,13 +180,13 @@ else testResult[count]="FAIL" fi echo -gmake clean -gmake ATM=C OCN=A,B,C +make clean +make ATM=C OCN=A,B,C echo "OCN_SELECT: C" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -188,13 +196,13 @@ else testResult[count]="FAIL" fi echo -gmake clean -gmake ATM=D OCN=A,B,C +make clean +make ATM=D OCN=A,B,C echo "OCN_SELECT: A" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -205,13 +213,13 @@ testResult[count]="FAIL" fi echo FINISHED: $1 echo -gmake clean -gmake ATM=E OCN=A,B,C +make clean +make ATM=E OCN=A,B,C echo "OCN_SELECT: B" > esm.config ((count++)) testList[count]=$1 set -x -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -221,12 +229,13 @@ else testResult[count]="FAIL" fi echo -gmake clean -gmake ATM=F OCN=A,B,C +make clean +make ATM=F OCN=A,B,C echo "OCN_SELECT: C" > esm.config -#$MPIRUN 4 ./$2 --- cannot run this because atmF is not fully implemented +#$ESMF_INTERNAL_MPIRUN -np 4 ./$2 --- cannot run this because atmF is not fully implemented echo FINISHED: $1 cd .. +date echo --------------------------------------------------------------------------- echo } @@ -235,12 +244,13 @@ function TestExplorer { ((count++)) testList[count]=$1 echo --------------------------------------------------------------------------- +date echo STARTING: $1 cd $1 -gmake distclean +make distclean set -x ./nuopcExplorerScript ../AtmOcnSelectExternalProto/ATM-A/atmA.mk -$MPIRUN 4 $TOOLRUN ./$2 > $2.stdout 2>&1 +$ESMF_INTERNAL_MPIRUN -np 4 $TOOLRUN ./$2 > $2.stdout 2>&1 local result=$? set +x if [ $result -eq 0 ] @@ -251,6 +261,7 @@ testResult[count]="FAIL" fi echo FINISHED: $1 cd .. +date echo --------------------------------------------------------------------------- echo } @@ -290,6 +301,7 @@ TestProto DriverInDriverDataDepProto mainApp TestProto DriverInDriverProto mainApp TestProto DynPhyProto esmApp TestProto ExternalDriverAPIProto externalApp +TestProto ExternalDriverAPIWeakCplDAProto externalApp TestProto GenericMediatorProto app TestProto HierarchyProto esmApp TestProto NamespaceProto mainApp @@ -298,7 +310,10 @@ TestProto NestingSingleProto mainApp TestProto NestingTelescopeMultipleProto mainApp TestProto SingleModelProto mainApp TestProto SingleModelOpenMPProto mainApp +export OMP_NUM_THREADS=3 +TestProto SingleModelOpenMPUnawareProto mainApp +date echo "== TEST SUMMARY START ==" i=1 while [[ $i -le $count ]] @@ -307,7 +322,8 @@ echo ${testResult[i]}: ${testList[i]} ((i++)) done echo "== TEST SUMMARY STOP ==" +date echo echo --------------------------------------------------------------------------- -grep ERROR */PET*.ESMF_LogFile +grep " ERROR " */PET*.ESMF_LogFile