Merge pull request #13 from lanl/brryan/kokkos_441

Update to Parthenon with Kokkos 4.4.1
lanl · Nov 22, 2024 · e2568f2 · e2568f2
2 parents a93dc69 + c9e41ef
commit e2568f2
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 49 deletions.
diff --git a/CMakePresets.json b/CMakePresets.json
@@ -32,8 +32,8 @@
         "CMAKE_BUILD_TYPE": "Debug",
         "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
-        "Kokkos_ARCH_VOLTA70": "On"
+        "ARTEMIS_ENABLE_CUDA": "ON",
+        "Kokkos_ARCH_VOLTA70": "ON"
       }
     },
     {
@@ -42,8 +42,8 @@
         "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}",
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
-        "Kokkos_ARCH_VOLTA70": "On"
+        "ARTEMIS_ENABLE_CUDA": "ON",
+        "Kokkos_ARCH_VOLTA70": "ON"
       }
     },
     {
@@ -97,10 +97,10 @@
         "CMAKE_BUILD_TYPE": "Debug",
         "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
+        "ARTEMIS_ENABLE_CUDA": "ON",
         "HDF5_INCLUDE_DIR": "$env{HDF5_ROOT}/include",
-        "PARTHENON_DISABLE": "HDF5_COMPRESSION=ON",
-        "Kokkos_ARCH_HOPPER90": "On",
+        "PARTHENON_DISABLE_HDF5_COMPRESSION": "ON",
+        "Kokkos_ARCH_HOPPER90": "ON",
         "NUM_GPU_DEVICES_PER_NODE": "4"
       }
     },
@@ -110,10 +110,10 @@
         "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}",
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
         "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper",
-        "ARTEMIS_ENABLE_CUDA": "On",
+        "ARTEMIS_ENABLE_CUDA": "ON",
         "HDF5_INCLUDE_DIR": "$env{HDF5_ROOT}/include",
-        "PARTHENON_DISABLE": "HDF5_COMPRESSION=ON",
-        "Kokkos_ARCH_HOPPER90": "On",
+        "PARTHENON_DISABLE_HDF5_COMPRESSION": "ON",
+        "Kokkos_ARCH_HOPPER90": "ON",
         "NUM_GPU_DEVICES_PER_NODE": "4"
       }
     }

diff --git a/env/bash b/env/bash
@@ -27,10 +27,12 @@ if [[ $HOSTNAME == ch-fe*  ||  $SLURM_CLUSTER_NAME == "chicoma" ]]; then
     else
         PARTITION="chicoma-cpu"
     fi
-elif [[ "$HOSTNAME" =~ ^ve-rfe[1-3]$ || ( $SLURM_CLUSTER_NAME == "venado" && $SLURM_JOB_PARTITION == "gpu" ) ]]; then
-    PARTITION="venado-gh"
-elif [[ "$HOSTNAME" =~ ^ve-rfe[4-7]$ || ( $SLURM_CLUSTER_NAME == "venado" && $SLURM_JOB_PARTITION == "cpu" ) ]]; then
-    PARTITION="venado-gg"
+elif [[ "$HOSTNAME" =~ ^ve-rfe[1-7]$ || "$HOSTNAME" =~ ^ve-fe[1-7]$ ]]; then
+    if [[ $SLURM_GPUS_ON_NODE > 0 || ("$HOSTNAME" =~ ^ve-rfe[1-3]$ || "$HOSTNAME" =~ ^ve-fe[1-3]$) ]]; then
+        PARTITION="venado-gh"
+    elif [[ $SLURM_GPUS_ON_NODE == 0 || ("$HOSTNAME" =~ ^ve-rfe[4-7]$ || "$HOSTNAME" =~ ^ve-fe[4-7]$) ]]; then
+        PARTITION="venado-gg"
+    fi
 else # Catch-all for Darwin
     if [ -z "$SLURM_JOB_PARTITION" ]; then
         if [[ $HOSTNAME == darwin-fe* ]]; then
@@ -119,36 +121,43 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then
     export ARTEMIS_SUITE=darwin-gpu
     echo "...setup SUCCEEDED"
 elif [[ $PARTITION == "venado-gh" ]]; then
-  module load PrgEnv-gnu
-  module load cray-mpich cray-hdf5-parallel cudatoolkit
-  export MPICH_OFI_NIC_POLICY=GPU   # GPU NUMA ROUND-ROBIN
-  export MPICH_GPU_SUPPORT_ENABLED=1 # Allows GPU Aware MPI
-  export CRAY_ACCEL_TARGET=nvidia90
-  export MPICH_MALLOC_FALLBACK=1
-  export MPICH_SMP_SINGLE_COPY_MODE=NONE
-  export MPICH_MAX_THREAD_SAFETY=multiple
-  export FI_CXI_RX_MATCH_MODE=hybrid
-  export PMI_MMAP_SYNC_WAIT_TIME=600
-  export ARTEMIS_SUITE=venado-gpu
-  module list
-  echo "...setup SUCCEEDED"
+    module load PrgEnv-gnu
+    module load cray-mpich
+    #module load cray-hdf5-parallel # Compiler wrappers fail to find hdf5.h 2024/11/8
+    export HDF5_ROOT=/opt/cray/pe/hdf5-parallel/1.14.3.1/gnu/12.3
+    module load cudatoolkit
+    module load cmake
+    export MPICH_OFI_NIC_POLICY=GPU   # GPU NUMA ROUND-ROBIN
+    export MPICH_GPU_SUPPORT_ENABLED=1 # Allows GPU Aware MPI
+    export CRAY_ACCEL_TARGET=nvidia90
+    export MPICH_MALLOC_FALLBACK=1
+    export MPICH_SMP_SINGLE_COPY_MODE=NONE
+    export MPICH_MAX_THREAD_SAFETY=multiple
+    export FI_CXI_RX_MATCH_MODE=hybrid
+    export PMI_MMAP_SYNC_WAIT_TIME=600
+    export NVCC_WRAPPER_DEFAULT_COMPILER=g++
+    export ARTEMIS_SUITE=venado-gpu
+    module list
+    echo "...setup SUCCEEDED"
 elif [[ $PARTITION == "venado-gg" ]]; then
-  module load PrgEnv-gnu
-  module load cray-mpich cray-hdf5-parallel
-  export MPICH_MALLOC_FALLBACK=1
-  export MPICH_SMP_SINGLE_COPY_MODE=NONE
-  export MPICH_MAX_THREAD_SAFETY=multiple
-  export FI_CXI_RX_MATCH_MODE=hybrid
-  export PMI_MMAP_SYNC_WAIT_TIME=600
-  export ARTEMIS_SUITE=venado-cpu
-  module list
-  echo "...setup SUCCEEDED"
+    module load PrgEnv-gnu
+    module load cray-mpich cray-hdf5-parallel
+    export MPICH_MALLOC_FALLBACK=1
+    export MPICH_SMP_SINGLE_COPY_MODE=NONE
+    export MPICH_MAX_THREAD_SAFETY=multiple
+    export FI_CXI_RX_MATCH_MODE=hybrid
+    export PMI_MMAP_SYNC_WAIT_TIME=600
+    export ARTEMIS_SUITE=venado-cpu
+    module list
+    echo "...setup SUCCEEDED"
 fi
 
-echo ""
-echo "To configure and build the code, you can use the function (-h flag for options)"
-echo "  build_artemis"
-echo ""
+if [[ $PARTITION != "unknown" ]]; then
+    echo ""
+    echo "To configure and build the code, you can use the function (-h flag for options)"
+    echo "  build_artemis"
+    echo ""
+fi
 
 function build_artemis {
 

diff --git a/external/parthenon b/external/parthenon
diff --git a/src/geometry/spherical.hpp b/src/geometry/spherical.hpp
@@ -530,12 +530,6 @@ class Coords<Coordinates::spherical1D>
     // volume average (r,theta)
     const Real rv = x1v();
 
-    // face averaged r on the X2 face
-    const Real rf =
-        2.0 / 3.0 *
-        (bnds.x1[0] * bnds.x1[0] + bnds.x1[0] * bnds.x1[1] + bnds.x1[1] * bnds.x1[1]) /
-        (bnds.x1[0] + bnds.x1[1]);
-
     const Real r2cyl = SQR(rv);
 
     std::array<Real, 2> bx1{r2cyl - SQR(bnds.x1[0]), SQR(bnds.x1[1]) - r2cyl};

diff --git a/src/gravity/point_mass.cpp b/src/gravity/point_mass.cpp
@@ -35,7 +35,7 @@ TaskStatus PointMassGravity(MeshData<Real> *md, const Real time, const Real dt)
   const bool do_dust = artemis_pkg->template Param<bool>("do_dust");
 
   auto &gravity_pkg = pm->packages.Get("gravity");
-  const Real gm = gravity_pkg->template Param<Real>("gm");
+  const Real gm_ = gravity_pkg->template Param<Real>("gm");
   const Real sink_rate = dt * (gravity_pkg->template Param<Real>("sink_rate"));
   const Real sink_rad = gravity_pkg->template Param<Real>("sink");
 
@@ -68,6 +68,9 @@ TaskStatus PointMassGravity(MeshData<Real> *md, const Real time, const Real dt)
         const auto &dx = coords.GetCellCenter();
         const auto &hx = coords.GetScaleFactors();
 
+        // Capture outside constexpr if
+        const Real &gm = gm_;
+
         Real gx1 = 0.0, gx2 = 0.0, gx3 = 0.0;
         Real dr = Null<Real>();
         [[maybe_unused]] Real pos[3] = {pos_[0], pos_[1], pos_[2]};