diff --git a/CMakePresets.json b/CMakePresets.json index 45800b6..4fe00dc 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -32,8 +32,8 @@ "CMAKE_BUILD_TYPE": "Debug", "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper", - "ARTEMIS_ENABLE_CUDA": "On", - "Kokkos_ARCH_VOLTA70": "On" + "ARTEMIS_ENABLE_CUDA": "ON", + "Kokkos_ARCH_VOLTA70": "ON" } }, { @@ -42,8 +42,8 @@ "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", "CMAKE_BUILD_TYPE": "RelWithDebInfo", "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper", - "ARTEMIS_ENABLE_CUDA": "On", - "Kokkos_ARCH_VOLTA70": "On" + "ARTEMIS_ENABLE_CUDA": "ON", + "Kokkos_ARCH_VOLTA70": "ON" } }, { @@ -97,10 +97,10 @@ "CMAKE_BUILD_TYPE": "Debug", "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON", "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper", - "ARTEMIS_ENABLE_CUDA": "On", + "ARTEMIS_ENABLE_CUDA": "ON", "HDF5_INCLUDE_DIR": "$env{HDF5_ROOT}/include", - "PARTHENON_DISABLE": "HDF5_COMPRESSION=ON", - "Kokkos_ARCH_HOPPER90": "On", + "PARTHENON_DISABLE_HDF5_COMPRESSION": "ON", + "Kokkos_ARCH_HOPPER90": "ON", "NUM_GPU_DEVICES_PER_NODE": "4" } }, @@ -110,10 +110,10 @@ "CMAKE_MAKE_PROGRAM": "$env{MAKE_PROGRAM}", "CMAKE_BUILD_TYPE": "RelWithDebInfo", "CMAKE_CXX_COMPILER": "$env{ARTEMIS_HOME}/external/parthenon/external/Kokkos/bin/nvcc_wrapper", - "ARTEMIS_ENABLE_CUDA": "On", + "ARTEMIS_ENABLE_CUDA": "ON", "HDF5_INCLUDE_DIR": "$env{HDF5_ROOT}/include", - "PARTHENON_DISABLE": "HDF5_COMPRESSION=ON", - "Kokkos_ARCH_HOPPER90": "On", + "PARTHENON_DISABLE_HDF5_COMPRESSION": "ON", + "Kokkos_ARCH_HOPPER90": "ON", "NUM_GPU_DEVICES_PER_NODE": "4" } } diff --git a/env/bash b/env/bash index b0fd4d3..d4543e9 100644 --- a/env/bash +++ b/env/bash @@ -27,10 +27,12 @@ if [[ $HOSTNAME == ch-fe* || $SLURM_CLUSTER_NAME == "chicoma" ]]; then else PARTITION="chicoma-cpu" fi -elif [[ "$HOSTNAME" =~ ^ve-rfe[1-3]$ || ( $SLURM_CLUSTER_NAME == "venado" && $SLURM_JOB_PARTITION == "gpu" ) ]]; then - PARTITION="venado-gh" -elif [[ "$HOSTNAME" =~ ^ve-rfe[4-7]$ || ( $SLURM_CLUSTER_NAME == "venado" && $SLURM_JOB_PARTITION == "cpu" ) ]]; then - PARTITION="venado-gg" +elif [[ "$HOSTNAME" =~ ^ve-rfe[1-7]$ || "$HOSTNAME" =~ ^ve-fe[1-7]$ ]]; then + if [[ $SLURM_GPUS_ON_NODE > 0 || ("$HOSTNAME" =~ ^ve-rfe[1-3]$ || "$HOSTNAME" =~ ^ve-fe[1-3]$) ]]; then + PARTITION="venado-gh" + elif [[ $SLURM_GPUS_ON_NODE == 0 || ("$HOSTNAME" =~ ^ve-rfe[4-7]$ || "$HOSTNAME" =~ ^ve-fe[4-7]$) ]]; then + PARTITION="venado-gg" + fi else # Catch-all for Darwin if [ -z "$SLURM_JOB_PARTITION" ]; then if [[ $HOSTNAME == darwin-fe* ]]; then @@ -119,36 +121,43 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then export ARTEMIS_SUITE=darwin-gpu echo "...setup SUCCEEDED" elif [[ $PARTITION == "venado-gh" ]]; then - module load PrgEnv-gnu - module load cray-mpich cray-hdf5-parallel cudatoolkit - export MPICH_OFI_NIC_POLICY=GPU # GPU NUMA ROUND-ROBIN - export MPICH_GPU_SUPPORT_ENABLED=1 # Allows GPU Aware MPI - export CRAY_ACCEL_TARGET=nvidia90 - export MPICH_MALLOC_FALLBACK=1 - export MPICH_SMP_SINGLE_COPY_MODE=NONE - export MPICH_MAX_THREAD_SAFETY=multiple - export FI_CXI_RX_MATCH_MODE=hybrid - export PMI_MMAP_SYNC_WAIT_TIME=600 - export ARTEMIS_SUITE=venado-gpu - module list - echo "...setup SUCCEEDED" + module load PrgEnv-gnu + module load cray-mpich + #module load cray-hdf5-parallel # Compiler wrappers fail to find hdf5.h 2024/11/8 + export HDF5_ROOT=/opt/cray/pe/hdf5-parallel/1.14.3.1/gnu/12.3 + module load cudatoolkit + module load cmake + export MPICH_OFI_NIC_POLICY=GPU # GPU NUMA ROUND-ROBIN + export MPICH_GPU_SUPPORT_ENABLED=1 # Allows GPU Aware MPI + export CRAY_ACCEL_TARGET=nvidia90 + export MPICH_MALLOC_FALLBACK=1 + export MPICH_SMP_SINGLE_COPY_MODE=NONE + export MPICH_MAX_THREAD_SAFETY=multiple + export FI_CXI_RX_MATCH_MODE=hybrid + export PMI_MMAP_SYNC_WAIT_TIME=600 + export NVCC_WRAPPER_DEFAULT_COMPILER=g++ + export ARTEMIS_SUITE=venado-gpu + module list + echo "...setup SUCCEEDED" elif [[ $PARTITION == "venado-gg" ]]; then - module load PrgEnv-gnu - module load cray-mpich cray-hdf5-parallel - export MPICH_MALLOC_FALLBACK=1 - export MPICH_SMP_SINGLE_COPY_MODE=NONE - export MPICH_MAX_THREAD_SAFETY=multiple - export FI_CXI_RX_MATCH_MODE=hybrid - export PMI_MMAP_SYNC_WAIT_TIME=600 - export ARTEMIS_SUITE=venado-cpu - module list - echo "...setup SUCCEEDED" + module load PrgEnv-gnu + module load cray-mpich cray-hdf5-parallel + export MPICH_MALLOC_FALLBACK=1 + export MPICH_SMP_SINGLE_COPY_MODE=NONE + export MPICH_MAX_THREAD_SAFETY=multiple + export FI_CXI_RX_MATCH_MODE=hybrid + export PMI_MMAP_SYNC_WAIT_TIME=600 + export ARTEMIS_SUITE=venado-cpu + module list + echo "...setup SUCCEEDED" fi -echo "" -echo "To configure and build the code, you can use the function (-h flag for options)" -echo " build_artemis" -echo "" +if [[ $PARTITION != "unknown" ]]; then + echo "" + echo "To configure and build the code, you can use the function (-h flag for options)" + echo " build_artemis" + echo "" +fi function build_artemis { diff --git a/external/parthenon b/external/parthenon index 79d5d30..b559452 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 79d5d301068cf34603c5194a796cdcbd3a134dae +Subproject commit b5594528e31cbcb3bd5d2aaa0f944d36d10b2c8f diff --git a/src/geometry/spherical.hpp b/src/geometry/spherical.hpp index c2466a9..b6fac32 100644 --- a/src/geometry/spherical.hpp +++ b/src/geometry/spherical.hpp @@ -530,12 +530,6 @@ class Coords // volume average (r,theta) const Real rv = x1v(); - // face averaged r on the X2 face - const Real rf = - 2.0 / 3.0 * - (bnds.x1[0] * bnds.x1[0] + bnds.x1[0] * bnds.x1[1] + bnds.x1[1] * bnds.x1[1]) / - (bnds.x1[0] + bnds.x1[1]); - const Real r2cyl = SQR(rv); std::array bx1{r2cyl - SQR(bnds.x1[0]), SQR(bnds.x1[1]) - r2cyl}; diff --git a/src/gravity/point_mass.cpp b/src/gravity/point_mass.cpp index a4dd24c..2cbcb63 100644 --- a/src/gravity/point_mass.cpp +++ b/src/gravity/point_mass.cpp @@ -35,7 +35,7 @@ TaskStatus PointMassGravity(MeshData *md, const Real time, const Real dt) const bool do_dust = artemis_pkg->template Param("do_dust"); auto &gravity_pkg = pm->packages.Get("gravity"); - const Real gm = gravity_pkg->template Param("gm"); + const Real gm_ = gravity_pkg->template Param("gm"); const Real sink_rate = dt * (gravity_pkg->template Param("sink_rate")); const Real sink_rad = gravity_pkg->template Param("sink"); @@ -68,6 +68,9 @@ TaskStatus PointMassGravity(MeshData *md, const Real time, const Real dt) const auto &dx = coords.GetCellCenter(); const auto &hx = coords.GetScaleFactors(); + // Capture outside constexpr if + const Real &gm = gm_; + Real gx1 = 0.0, gx2 = 0.0, gx3 = 0.0; Real dr = Null(); [[maybe_unused]] Real pos[3] = {pos_[0], pos_[1], pos_[2]};