Skip to content

Commit

Permalink
Add netCDF PIO capability for restarts and run-time history for dev/u…
Browse files Browse the repository at this point in the history 
…fs-weather-model (NOAA-EMC#1303)
  • Loading branch information
@DeniseWorthen
DeniseWorthen authored Nov 14, 2024
1 parent c7004b6 commit abe1e79
Show file tree
Hide file tree
Showing 18 changed files with 2,555 additions and 1,405 deletions.
9 changes: 4 additions & 5 deletions model/src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

# Open switch file
file(STRINGS ${CMAKE_BINARY_DIR}/switch switch_strings)
separate_arguments(switches UNIX_COMMAND ${switch_strings})
Expand Down Expand Up @@ -85,7 +84,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
set(compile_flags -g -fno-second-underscore -ffree-line-length-none)
set(compile_flags_release -O3)
set(compile_flags_debug -Wall -fcheck=all -ffpe-trap=invalid,zero,overflow -frecursive -fbacktrace)

if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
target_compile_options(ww3_lib PUBLIC -fallow-argument-mismatch)
endif()
Expand All @@ -94,7 +93,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "PGI")
set(compile_flags -g -i4 -r4 -Kieee)
set(compile_flags_release -O3)
set(compile_flags_debug -O0 -Mbounds -Mchkfpstk -Mchkstk -Mdalign -Mdclchk -Mdepchk -Miomutex -Ktrap=fp -Mrecursive -traceback)

if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
target_compile_options(ww3_lib PUBLIC -fallow-argument-mismatch -fallow-invalid-boz)
endif()
Expand Down Expand Up @@ -152,7 +151,7 @@ if(UFS_CAP)
elseif(UFS_CAP STREQUAL "NUOPC_MESH")
set(cap_src ${nuopc_mesh_cap_src})
endif()

target_sources(ww3_lib PRIVATE ${cap_src})
target_link_libraries(ww3_lib PUBLIC esmf)
# Don't build executables when building WW3 ESMF library
Expand Down Expand Up @@ -231,7 +230,7 @@ install(

install(FILES ${CMAKE_BINARY_DIR}/switch DESTINATION ${CMAKE_INSTALL_PREFIX})
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})


export(EXPORT WW3Exports
NAMESPACE WW3::
Expand Down
5 changes: 3 additions & 2 deletions model/src/cmake/src_list.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,7 @@ set(ftn_src
wmupdtmd.F90
wmwavemd.F90
w3tidemd.F90
wav_grdout.F90
w3iogoncdmd.F90
wav_history_mod.F90
wav_shr_flags.F90
)

Expand All @@ -67,6 +66,8 @@ set(nuopc_mesh_cap_src
wav_comp_nuopc.F90
wav_import_export.F90
wav_wrapper_mod.F90
wav_pio_mod.F90
wav_restart_mod.F90
)

set(esmf_multi_cap_src
Expand Down
6 changes: 3 additions & 3 deletions model/src/w3gridmd.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -836,7 +836,7 @@ MODULE W3GRIDMD
!
#ifdef W3_ST4
INTEGER :: SWELLFPAR, SDSISO, SDSBRFDF
REAL :: SDSBCHOICE
REAL :: SDSBCHOICE
REAL :: ZWND, ALPHA0, Z0MAX, BETAMAX, SINTHP,&
ZALP, Z0RAT, TAUWSHELTER, SWELLF, &
SWELLF2,SWELLF3,SWELLF4, SWELLF5, &
Expand Down Expand Up @@ -3280,7 +3280,7 @@ SUBROUTINE W3GRID()
JGS_TERMINATE_DIFFERENCE, &
JGS_TERMINATE_NORM, &
JGS_LIMITER, &
JGS_LIMITER_FUNC, &
JGS_LIMITER_FUNC, &
JGS_USE_JACOBI, &
JGS_BLOCK_GAUSS_SEIDEL, &
JGS_MAXITER, &
Expand Down Expand Up @@ -3617,7 +3617,7 @@ SUBROUTINE W3GRID()
END SELECT

IF (FSTOTALIMP .or. FSTOTALEXP) THEN
LPDLIB = .TRUE.
LPDLIB = .TRUE.
ENDIF
!
IF (SUM(UNSTSCHEMES).GT.1) WRITE(NDSO,1035)
Expand Down
165 changes: 103 additions & 62 deletions model/src/w3initmd.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,10 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
#ifdef W3_UOST
USE W3UOSTMD, ONLY: UOST_SETGRID
#endif
use w3timemd, only : set_user_timestring
use w3odatmd, only : runtype, restart_from_binary, use_restartnc, user_restfname
use w3odatmd, only : logfile_is_assigned
use wav_restart_mod, only : read_restart
!/
#ifdef W3_MPI
INCLUDE "mpif.h"
Expand Down Expand Up @@ -512,7 +516,10 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
#ifdef W3_PDLIB
INTEGER :: IScal(1), IPROC
#endif
logical :: exists
integer :: memunit
character(len=16) :: user_timestring !YYYY-MM-DD-SSSSS
character(len=1024) :: fname
!/
!/ ------------------------------------------------------------------- /
!
Expand Down Expand Up @@ -639,53 +646,52 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
IF (FSTOTALIMP .and. .NOT. LPDLIB) THEN
WRITE(NDSE,*) 'IMPTOTAL is selected'
WRITE(NDSE,*) 'But PDLIB is not'
CALL FLUSH(NDSE)
STOP
CALL FLUSH(NDSE)
STOP
ELSE IF (FSTOTALEXP .and. .NOT. LPDLIB) THEN
WRITE(NDSE,*) 'EXPTOTAL is selected'
WRITE(NDSE,*) 'But PDLIB is not'
CALL FLUSH(NDSE)
STOP
CALL FLUSH(NDSE)
STOP
END IF
#ifdef W3_PDLIB
IF (B_JGS_BLOCK_GAUSS_SEIDEL .AND. .NOT. B_JGS_USE_JACOBI) THEN
WRITE(NDSE,*) 'B_JGS_BLOCK_GAUSS_SEIDEL is used but the Jacobi solver is not choosen'
WRITE(NDSE,*) 'Please set JGS_USE_JACOBI .eqv. .true.'
CALL FLUSH(NDSE)
STOP
CALL FLUSH(NDSE)
STOP
ENDIF
#endif

!
! 1.c Open files without unpacking MDS ,,,
!
IE = LEN_TRIM(FEXT)
LFILE = 'log.' // FEXT(:IE)
IFL = LEN_TRIM(LFILE)
if (.not. logfile_is_assigned) then
IE = LEN_TRIM(FEXT)
LFILE = 'log.' // FEXT(:IE)
IFL = LEN_TRIM(LFILE)
#ifdef W3_SHRD
TFILE = 'test.' // FEXT(:IE)
TFILE = 'test.' // FEXT(:IE)
#endif
#ifdef W3_DIST
IW = 1 + INT ( LOG10 ( REAL(NAPROC) + 0.5 ) )
IW = MAX ( 3 , MIN ( 9 , IW ) )
WRITE (FORMAT,'(A5,I1.1,A1,I1.1,A4)') &
'(A4,I', IW, '.', IW, ',2A)'
WRITE (TFILE,FORMAT) 'test', &
OUTPTS(IMOD)%IAPROC, '.', FEXT(:IE)
#endif
IFT = LEN_TRIM(TFILE)
J = LEN_TRIM(FNMPRE)
!
#ifndef W3_CESMCOUPLED
IF ( OUTPTS(IMOD)%IAPROC .EQ. OUTPTS(IMOD)%NAPLOG ) &
OPEN (MDS(1),FILE=FNMPRE(:J)//LFILE(:IFL),ERR=888,IOSTAT=IERR)
#endif
!
IF ( MDS(3).NE.MDS(1) .AND. MDS(3).NE.MDS(4) .AND. TSTOUT ) THEN
INQUIRE (MDS(3),OPENED=OPENED)
IF ( .NOT. OPENED ) OPEN (MDS(3),FILE=FNMPRE(:J)//TFILE(:IFT), ERR=889, &
IOSTAT=IERR)
END IF
IW = 1 + INT ( LOG10 ( REAL(NAPROC) + 0.5 ) )
IW = MAX ( 3 , MIN ( 9 , IW ) )
WRITE (FORMAT,'(A5,I1.1,A1,I1.1,A4)') &
'(A4,I', IW, '.', IW, ',2A)'
WRITE (TFILE,FORMAT) 'test', &
OUTPTS(IMOD)%IAPROC, '.', FEXT(:IE)
#endif
IFT = LEN_TRIM(TFILE)
J = LEN_TRIM(FNMPRE)
!
IF ( OUTPTS(IMOD)%IAPROC .EQ. OUTPTS(IMOD)%NAPLOG ) &
OPEN (MDS(1),FILE=FNMPRE(:J)//LFILE(:IFL),ERR=888,IOSTAT=IERR)
!
IF ( MDS(3).NE.MDS(1) .AND. MDS(3).NE.MDS(4) .AND. TSTOUT ) THEN
INQUIRE (MDS(3),OPENED=OPENED)
IF ( .NOT. OPENED ) OPEN (MDS(3),FILE=FNMPRE(:J)//TFILE(:IFT), ERR=889, &
IOSTAT=IERR)
END IF
end if ! if (.not. logfile_is_assigned)
!
! 1.d Dataset unit numbers
!
Expand Down Expand Up @@ -725,6 +731,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
! 2.a Read model definition file
!
CALL W3IOGR ( 'READ', NDS(5), IMOD, FEXT )

IF (GTYPE .eq. UNGTYPE) THEN
CALL SPATIAL_GRID
CALL NVECTRI
Expand Down Expand Up @@ -952,40 +959,64 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
! 3.a Read restart file
!
VA(:,:) = 0.
if (use_restartnc) then
if (runtype == 'continue' )then
call set_user_timestring(time,user_timestring)
if (restart_from_binary) then
fname = trim(user_restfname)//trim(user_timestring)
else
fname = trim(user_restfname)//trim(user_timestring)//'.nc'
endif
inquire(file=trim(fname), exist=exists)
if (exists) then
if (restart_from_binary) then
call w3iors('READ', nds(6), sig(nk), imod, filename=trim(fname))
else
call read_restart(trim(fname), va=va, mapsta=mapsta, mapst2=mapst2)
end if
else
call extcde (60, msg="required restart file " // trim(fname) // " does not exist")
end if
else
call read_restart('none')
! mapst2 is module variable defined in read of mod_def; maptst is from 2.b above
flcold = .true.
end if
else
#ifdef W3_DEBUGCOH
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "Before W3IORS call", 1)
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "Before W3IORS call", 1)
#endif
#ifdef W3_TIMINGS
CALL PRINT_MY_TIME("Before W3IORS")
CALL PRINT_MY_TIME("Before W3IORS")
#endif
CALL W3IORS ( 'READ', NDS(6), SIG(NK), IMOD)
CALL W3IORS ( 'READ', NDS(6), SIG(NK), IMOD)
#ifdef W3_TIMINGS
CALL PRINT_MY_TIME("After W3IORS")
CALL PRINT_MY_TIME("After W3IORS")
#endif
call print_memcheck(memunit, 'memcheck_____:'//' WW3_INIT SECTION 3a')
call print_memcheck(memunit, 'memcheck_____:'//' WW3_INIT SECTION 3a')

#ifdef W3_DEBUGCOH
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "After W3IORS call", 1)
#endif
FLCOLD = RSTYPE.LE.1 .OR. RSTYPE.EQ.4
IF ( IAPROC .EQ. NAPLOG ) THEN
IF (RSTYPE.EQ.0) THEN
WRITE (NDSO,930) 'cold start (idealized).'
ELSE IF ( RSTYPE .EQ. 1 ) THEN
WRITE (NDSO,930) 'cold start (wind).'
ELSE IF ( RSTYPE .EQ. 4 ) THEN
WRITE (NDSO,930) 'cold start (calm).'
ELSE
WRITE (NDSO,930) 'full restart.'
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "After W3IORS call", 1)
#endif
FLCOLD = RSTYPE.LE.1 .OR. RSTYPE.EQ.4
IF ( IAPROC .EQ. NAPLOG ) THEN
IF (RSTYPE.EQ.0) THEN
WRITE (NDSO,930) 'cold start (idealized).'
ELSE IF ( RSTYPE .EQ. 1 ) THEN
WRITE (NDSO,930) 'cold start (wind).'
ELSE IF ( RSTYPE .EQ. 4 ) THEN
WRITE (NDSO,930) 'cold start (calm).'
ELSE
WRITE (NDSO,930) 'full restart.'
END IF
END IF
END IF
#ifdef W3_DEBUGCOH
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "W3INIT, step 4.2", 1)
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "W3INIT, step 4.2", 1)
#endif
#ifdef W3_TIMINGS
CALL PRINT_MY_TIME("After restart inits")
CALL PRINT_MY_TIME("After restart inits")
#endif

end if ! if (use_restartnc)
!
! 3.b Compare MAPSTA from grid and restart
!
Expand Down Expand Up @@ -1263,7 +1294,6 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
!
MAPTST = MOD(MAPST2/2,2)
MAPST2 = MAPST2 - 2*MAPTST

!
!Li For multi-resolution SMC grid, these 1-NX and 1-NY nested loops
!Li may miss the refined cells as they are not 1-1 corresponding to
Expand Down Expand Up @@ -1337,12 +1367,10 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
CALL SET_IOBDP_PDLIB
ENDIF
#endif

!
#ifdef W3_DEBUGCOH
CALL ALL_VA_INTEGRAL_PRINT(IMOD, "W3INIT, step 8.2", 1)
#endif

!
MAPST2 = MAPST2 + 2*MAPTST
!
Expand Down Expand Up @@ -1396,7 +1424,6 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
!
END DO
END DO

!
! 6. Initialize arrays ---------------------------------------------- /
! Some initialized in W3IORS
Expand All @@ -1413,7 +1440,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
!
! 7. Write info to log file ----------------------------------------- /
!
IF ( IAPROC .EQ. NAPLOG ) THEN
IF ( IAPROC .EQ. NAPLOG) THEN
!
WRITE (NDSO,970) GNAME
IF ( FLLEV ) WRITE (NDSO,971) 'Prescribed'
Expand Down Expand Up @@ -1500,7 +1527,9 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD,
WRITE (NDSO,990) DTME21
END IF
!
WRITE (NDSO,984)
if (.not. logfile_is_assigned) then
WRITE (NDSO,984)
end if
!
END IF
!
Expand Down Expand Up @@ -2171,6 +2200,7 @@ SUBROUTINE W3MPIO ( IMOD )
#endif
USE W3GDATMD, ONLY: GTYPE, UNGTYPE
USE CONSTANTS, ONLY: LPDLIB
use w3odatmd, only : restart_from_binary, use_restartnc, use_historync
!/
#ifdef W3_MPI
INCLUDE "mpif.h"
Expand Down Expand Up @@ -2202,6 +2232,7 @@ SUBROUTINE W3MPIO ( IMOD )
#ifdef W3_MPIT
CHARACTER(LEN=5) :: STRING
#endif
logical :: do_rstsetup
!/
!/ ------------------------------------------------------------------- /
!/
Expand All @@ -2225,7 +2256,7 @@ SUBROUTINE W3MPIO ( IMOD )
IROOT = NAPFLD - 1
!
!
IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB)) THEN
IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB) .and. (.not. use_historync)) THEN
!
! NRQMAX is the maximum number of output fields that require MPI communication,
! aimed to gather field values stored in each processor into one processor in
Expand Down Expand Up @@ -4760,7 +4791,7 @@ SUBROUTINE W3MPIO ( IMOD )
CALL EXTCDE (11)
END IF
!
END IF ! IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB)) THEN
END IF ! IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB) .and. (.not. use_historync)) THEN
!
! 2. Set-up for W3IORS ---------------------------------------------- /
! 2.a General preparations
Expand All @@ -4769,7 +4800,17 @@ SUBROUTINE W3MPIO ( IMOD )
IH = 0
IROOT = NAPRST - 1
!
IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB)) THEN
if (use_restartnc) then
if (restart_from_binary) then
do_rstsetup = .true.
else
do_rstsetup = .false.
end if
else
do_rstsetup = .true.
end if
!
IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB) .and. do_rstsetup) THEN
IF (OARST) THEN
ALLOCATE ( OUTPTS(IMOD)%OUT4%IRQRS(34*NAPROC) )
ELSE
Expand Down Expand Up @@ -5647,7 +5688,7 @@ SUBROUTINE W3MPIO ( IMOD )
!
END IF
!
END IF ! IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB)) THEN
END IF ! IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB) .and. do_rstsetup) THEN
#endif
!
! 3. Set-up for W3IOBC ( SENDs ) ------------------------------------ /
Expand Down
Loading

0 comments on commit abe1e79

Please sign in to comment.