Skip to content

Commit

Permalink
Port to Intel LLVM on Hera, Orion and Hercules (#996)
Browse files Browse the repository at this point in the history
Also update the Github workflows to Intel LLVM.

Retain option to use Intel Classic on those machines.

Fixes #879.
  • Loading branch information
GeorgeGayno-NOAA authored Nov 18, 2024
1 parent 5b2bfa3 commit 122010e
Show file tree
Hide file tree
Showing 75 changed files with 343 additions and 104 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/macos_clang.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
run: |
git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
source spack/share/spack/setup-env.sh
sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[apple-clang@14\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[apple-clang@14\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
spack env create ufs_utils-env spack_ci.yaml
spack env activate ufs_utils-env
spack external find
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ubuntu_gcc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
run: |
git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
source spack/share/spack/setup-env.sh
sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[gcc@10:10\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[gcc@10:10\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
spack env create ufs_utils-env spack_ci.yaml
spack env activate ufs_utils-env
sudo apt install cmake
Expand Down
18 changes: 9 additions & 9 deletions .github/workflows/ubuntu_intel.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ defaults:
run:
shell: bash -leo pipefail {0}

# Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran
# Set I_MPI_CC/F90 so IntelLLVM is used.
env:
cache_key: intel
CC: icc
FC: ifort
CXX: icpc
I_MPI_CC: icc
I_MPI_F90: ifort
CC: mpiicc
FC: mpiifort
CXX: mpiicpc
I_MPI_CC: icx
I_MPI_F90: ifx

# The jobs are split into:
# 1. a dependency build step (setup), and
Expand Down Expand Up @@ -59,7 +59,7 @@ jobs:
run: |
git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
source spack/share/spack/setup-env.sh
sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[intel\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[oneapi\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
spack env create ufs_utils-env spack_ci.yaml
spack env activate ufs_utils-env
spack compiler find
Expand Down Expand Up @@ -99,8 +99,8 @@ jobs:
run: |
source spack/share/spack/setup-env.sh
spack env activate ufs_utils-env
export CC=mpiicc
export FC=mpiifort
export CC=mpiicx
export FC=mpiifx
cd ufs_utils
mkdir -p build && cd build
cmake -DCMAKE_INSTALL_PREFIX=../install ..
Expand Down
17 changes: 13 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ option(CHGRES "Enable building chgres_cube.fd" ON)
option(CHGRES_ALL "Build chgres with all input data options." OFF)
option(OROG_MASK_TOOLS "Enable building orog_mask_tools.fd" ON)
# OROG_MASK_TOOLS must be ON for OROG_NETCDF_TOOLS to build.
option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" OFF)
option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" ON)
option(SFC_CLIMO_GEN "Enable building sfc_climo_gen.fd" ON)
option(VCOORD_GEN "Enable building vcoord_gen.fd" ON)
option(FVCOMTOOLS "Enable building fvcom_tools.fd" ON)
Expand Down Expand Up @@ -67,10 +67,15 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
endif()

# Set compiler flags.
if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback")
set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model precise")
set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check all -ftrapuv")
# On some machines (Hera), using -check all results in an error
# at the linking step. The uninit check is the culprit.
if(CMAKE_Fortran_COMPILER_ID MATCHES "^(IntelLLVM)$")
set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check nouninit")
endif()
if(APPLE)
# The linker on macOS does not include `common symbols` by default.
# Passing the -c flag includes them and fixes an error with undefined symbols.
Expand All @@ -86,10 +91,14 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ggdb -Wall -fcheck=all")
endif()

if(CMAKE_C_COMPILER_ID MATCHES "^(Intel)$")
if(CMAKE_C_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -traceback")
set(CMAKE_C_FLAGS_RELEASE "-O2")
set(CMAKE_C_FLAGS_DEBUG "-O0")
if(CMAKE_C_COMPILER_ID MATCHES "^(Intel)$")
set(CMAKE_C_FLAGS_RELEASE "-O2")
else()
set(CMAKE_C_FLAGS_RELEASE "-O1")
endif()
endif()

# Find packages.
Expand Down
9 changes: 8 additions & 1 deletion build_all.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ source "${DIR_ROOT}/sorc/machine-setup.sh"

# User Options
target=${target:-"NULL"}
compiler=${compiler:-"intel"}
compiler=${compiler:-"intelllvm"} # If IntelLLVM is not available on the machine, will
# default to Intel classic.

if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
unset -f module
Expand All @@ -29,6 +30,12 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
else
set +x
module use "${DIR_ROOT}/modulefiles"
if [[ "$compiler" == "intelllvm" ]]; then
if [[ ! -f ${DIR_ROOT}/modulefiles/build.$target.$compiler.lua ]];then
echo "IntelLLVM not available. Will use Intel Classic."
compiler=intel
fi
fi
module load "build.$target.$compiler" > /dev/null
module list
set -x
Expand Down
5 changes: 2 additions & 3 deletions ci/spack.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
spack:
packages:
all:
compiler: [intel, gcc@10:10, apple-clang@14]
compiler: [oneapi, gcc@10:10, apple-clang@14]
specs:
- netcdf-c@4.9.2
- netcdf-fortran@4.6.1
Expand All @@ -12,8 +12,7 @@ spack:
- nemsio@2.5.4
- sp@2.5.0
- w3emc@2.10.0
- sfcio@1.4.1
- sigio@2.3.2
- sigio@2.3.3
- nccmp@1.9.0.1
- parallelio@2.5.10+fortran~pnetcdf
- esmf@8.6.0
Expand Down
2 changes: 1 addition & 1 deletion driver_scripts/driver_grid.hera.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@

set -x

compiler=${compiler:-"intel"}
compiler=${compiler:-"intelllvm"}
source ../sorc/machine-setup.sh > /dev/null 2>&1
module use ../modulefiles
module load build.$target.$compiler
Expand Down
2 changes: 1 addition & 1 deletion driver_scripts/driver_grid.hercules.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ set -x

source ../sorc/machine-setup.sh > /dev/null 2>&1
module use ../modulefiles
module load build.$target.intel
module load build.$target.intelllvm
module list

#-----------------------------------------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion driver_scripts/driver_grid.orion.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ set -x

source ../sorc/machine-setup.sh > /dev/null 2>&1
module use ../modulefiles
module load build.$target.intel
module load build.$target.intelllvm
module list

#-----------------------------------------------------------------------
Expand Down
74 changes: 74 additions & 0 deletions modulefiles/build.hera.intelllvm.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
help([[
Load environment to compile UFS_UTILS on Hera using Intel
]])

hpss_ver=os.getenv("hpss_ver") or ""
load(pathJoin("hpss", hpss_ver))

prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

load("gnu")
load("intel/2024.2.1")

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

bacio_ver=os.getenv("bacio_ver") or "2.4.1"
load(pathJoin("bacio", bacio_ver))

g2_ver=os.getenv("g2_ver") or "3.4.5"
load(pathJoin("g2", g2_ver))

ip_ver=os.getenv("ip_ver") or "4.3.0"
load(pathJoin("ip", ip_ver))

nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
load(pathJoin("nemsio", nemsio_ver))

sp_ver=os.getenv("sp_ver") or "2.5.0"
load(pathJoin("sp", sp_ver))

w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
load(pathJoin("w3emc", w3emc_ver))

-- Uncomment when CHGRES_ALL is ON
--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
--load(pathJoin("sfcio", sfcio_ver))

sigio_ver=os.getenv("sigio_ver") or "2.3.2"
load(pathJoin("sigio", sigio_ver))

zlib_ver=os.getenv("zlib_ver") or "1.2.13"
load(pathJoin("zlib", zlib_ver))

png_ver=os.getenv("png_ver") or "1.6.37"
load(pathJoin("libpng", png_ver))

netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
load(pathJoin("netcdf-c", netcdf_c_ver))

netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
load(pathJoin("netcdf-fortran", netcdf_fortran_ver))

nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
load(pathJoin("nccmp", nccmp_ver))

esmf_ver=os.getenv("esmf_ver") or "8.6.0"
load(pathJoin("esmf", esmf_ver))

nco_ver=os.getenv("nco_ver") or "5.0.6"
load(pathJoin("nco", nco_ver))

setenv("I_MPI_CC", "icx")
setenv("I_MPI_FC", "ifx")

setenv("CC", "mpicc")
setenv("FC", "mpifc")

whatis("Description: UFS_UTILS build environment")
2 changes: 1 addition & 1 deletion modulefiles/build.hercules.intel.lua
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
help([[
Load environment to compile UFS_UTILS on Hercules using Intel
Load environment to compile UFS_UTILS on Hercules using Intel Classic
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
Expand Down
68 changes: 68 additions & 0 deletions modulefiles/build.hercules.intelllvm.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
help([[
Load environment to compile UFS_UTILS on Hercules using IntelLLVM
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

bacio_ver=os.getenv("bacio_ver") or "2.4.1"
load(pathJoin("bacio", bacio_ver))

g2_ver=os.getenv("g2_ver") or "3.4.5"
load(pathJoin("g2", g2_ver))

ip_ver=os.getenv("ip_ver") or "4.3.0"
load(pathJoin("ip", ip_ver))

nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
load(pathJoin("nemsio", nemsio_ver))

sp_ver=os.getenv("sp_ver") or "2.5.0"
load(pathJoin("sp", sp_ver))

w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
load(pathJoin("w3emc", w3emc_ver))

-- Uncomment when CHGRES_ALL is ON
--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
--load(pathJoin("sfcio", sfcio_ver))

sigio_ver=os.getenv("sigio_ver") or "2.3.2"
load(pathJoin("sigio", sigio_ver))

zlib_ver=os.getenv("zlib_ver") or "1.2.13"
load(pathJoin("zlib", zlib_ver))

png_ver=os.getenv("png_ver") or "1.6.37"
load(pathJoin("libpng", png_ver))

netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
load(pathJoin("netcdf-c", netcdf_c_ver))

netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
load(pathJoin("netcdf-fortran", netcdf_fortran_ver))

nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
load(pathJoin("nccmp", nccmp_ver))

esmf_ver=os.getenv("esmf_ver") or "8.6.0"
load(pathJoin("esmf", esmf_ver))

nco_ver=os.getenv("nco_ver") or "5.0.6"
load(pathJoin("nco", nco_ver))

setenv("I_MPI_CC", "icx")
setenv("I_MPI_F90", "ifx")

setenv("CC", "mpiicc")
setenv("FC", "mpiifort")

whatis("Description: UFS_UTILS build environment")
2 changes: 1 addition & 1 deletion modulefiles/build.orion.intel.lua
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
help([[
Load environment to compile UFS_UTILS on Orion using Intel
Load environment to compile UFS_UTILS on Orion using Intel classic
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core")
Expand Down
68 changes: 68 additions & 0 deletions modulefiles/build.orion.intelllvm.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
help([[
Load environment to compile UFS_UTILS on Orion using IntelLLVM
]])

prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core")

stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
load(pathJoin("stack-intel", stack_intel_ver))

stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))

cmake_ver=os.getenv("cmake_ver") or "3.23.1"
load(pathJoin("cmake", cmake_ver))

bacio_ver=os.getenv("bacio_ver") or "2.4.1"
load(pathJoin("bacio", bacio_ver))

g2_ver=os.getenv("g2_ver") or "3.4.5"
load(pathJoin("g2", g2_ver))

ip_ver=os.getenv("ip_ver") or "4.3.0"
load(pathJoin("ip", ip_ver))

nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
load(pathJoin("nemsio", nemsio_ver))

sp_ver=os.getenv("sp_ver") or "2.5.0"
load(pathJoin("sp", sp_ver))

w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
load(pathJoin("w3emc", w3emc_ver))

-- Uncomment when CHGRES_ALL is ON
--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
--load(pathJoin("sfcio", sfcio_ver))

sigio_ver=os.getenv("sigio_ver") or "2.3.2"
load(pathJoin("sigio", sigio_ver))

zlib_ver=os.getenv("zlib_ver") or "1.2.13"
load(pathJoin("zlib", zlib_ver))

png_ver=os.getenv("png_ver") or "1.6.37"
load(pathJoin("libpng", png_ver))

netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
load(pathJoin("netcdf-c", netcdf_c_ver))

netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
load(pathJoin("netcdf-fortran", netcdf_fortran_ver))

nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
load(pathJoin("nccmp", nccmp_ver))

esmf_ver=os.getenv("esmf_ver") or "8.6.0"
load(pathJoin("esmf", esmf_ver))

nco_ver=os.getenv("nco_ver") or "5.0.6"
load(pathJoin("nco", nco_ver))

setenv("I_MPI_CC", "icx")
setenv("I_MPI_F90", "ifx")

setenv("CC", "mpiicc")
setenv("FC", "mpiifort")

whatis("Description: UFS_UTILS build environment")
Loading

0 comments on commit 122010e

Please sign in to comment.