dholladay00/separate get sg eos

.gitlab-ci.yml

@@ -9,7 +9,7 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
   SINGULARITY_EOS_GCC_VERSION: "9.3.0"
   SINGULARITY_EOS_OPENMPI_VERSION: "4.1.3"
-  SINGULARITY_EOS_CUDA_VERSION: "11.6.0"
+  SINGULARITY_EOS_CUDA_VERSION: "11.4.2"
   SINGULARITY_EOS_SPACK_SPEC: "singularity-eos@main+mpi+python+tests%gcc@${SINGULARITY_EOS_GCC_VERSION} ^openmpi@${SINGULARITY_EOS_OPENMPI_VERSION}"
   COLOR_CYAN: '\033[1;36m'
   COLOR_PLAIN: '\033[0m'
@@ -121,17 +121,20 @@ before_script:
     - spack add ${SINGULARITY_EOS_SPACK_SPEC}
     - spack install -j $(nproc) --show-log-on-error --no-checksum --yes-to-all -u cmake
     - echo -e "\e[0Ksection_end:`date +%s`:spack_env\r\e[0K"
+    - export SINGULARITY_BUILD_TYPE="RelWithDebInfo"
     - |
       if [[ -n ${SINGULARITY_USE_CUDA} ]];
       then
         export SINGULARITY_NVCC_WRAPPER="$(which nvcc_wrapper)";
         export SINGULARITY_CMAKE_CXX_COMPILER_FLAG="-DCMAKE_CXX_COMPILER=${SINGULARITY_NVCC_WRAPPER}";
         echo "passing '${SINGULARITY_CMAKE_CXX_COMPILER_FLAG}' to cmake"
+        export SINGULARITY_BUILD_TYPE="Release";
       fi
     - mkdir -p build
     - cd build
     - |
       cmake -DCMAKE_INSTALL_PREFIX=${CI_PROJECT_DIR}/install ${SINGULARITY_CMAKE_CXX_COMPILER_FLAG} \
+            -DCMAKE_BUILD_TYPE=${SINGULARITY_BUILD_TYPE} \
             -DSINGULARITY_USE_EOSPAC=ON \
             -DSINGULARITY_USE_HDF5=ON \
             -DSINGULARITY_BUILD_SESAME2SPINER=ON \

cmake/submodule_configs.cmake

@@ -23,7 +23,6 @@ macro(singularity_cmake_config pkg)
     if(SINGULARITY_USE_CUDA)
       set(Kokkos_ENABLE_CUDA ON CACHE BOOL "" FORCE)
       set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE)
-      set(Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ON CACHE BOOL "" FORCE)
     endif()
   elseif(${pkg} STREQUAL "KokkosKernels")
     # Disable TPLs

singularity-eos/CMakeLists.txt

@@ -13,7 +13,6 @@
 #------------------------------------------------------------------------------#
 
 set(EOS_HEADERS
-    closure/mixed_cell_models.hpp
     base/fast-math/logs.hpp
     base/robust_utils.hpp
     base/root-finding-1d/root_finding.hpp
@@ -47,12 +46,25 @@ set(EOS_SRCS
 )
 
 if (SINGULARITY_BUILD_CLOSURE)
-  list(APPEND EOS_SRCS eos/singularity_eos.cpp)
-  list(APPEND EOS_HEADERS eos/singularity_eos.hpp closure/mixed_cell_models.hpp)
+  list(APPEND EOS_SRCS 
+       eos/get_sg_eos.cpp)
+  if (SINGULARITY_USE_KOKKOS)
+    list(APPEND EOS_SRCS 
+         eos/get_sg_eos_p_t.cpp
+         eos/get_sg_eos_rho_t.cpp
+         eos/get_sg_eos_rho_p.cpp
+         eos/get_sg_eos_rho_e.cpp)
+  endif()
+  list(APPEND EOS_HEADERS
+       closure/mixed_cell_models.hpp
+       eos/get_sg_eos.hpp
+       eos/get_sg_eos_lambdas.hpp)
 endif()
 
 if (SINGULARITY_USE_FORTRAN)
   list(APPEND EOS_SRCS eos/singularity_eos.f90)
+  list(APPEND EOS_SRCS eos/singularity_eos.cpp)
+  list(APPEND EOS_HEADERS eos/singularity_eos.hpp)
   # would rather handle this more robustly, being sloppy for now
   list(APPEND EOS_MODS singularity_eos.mod singularity_eos_types.mod)
 endif()

singularity-eos/closure/mixed_cell_models.hpp

@@ -206,7 +206,7 @@ class PTESolverBase {
     Cache = CacheAccessor(AssignIncrement(scratch, nmat * MAX_NUM_LAMBDAS));
   }
 
-  PORTABLE_FORCEINLINE_FUNCTION
+  PORTABLE_INLINE_FUNCTION
   void InitRhoBarandRho() {
     // rhobar is a fixed quantity: the average density of
     // material m averaged over the full PTE volume
@@ -217,7 +217,7 @@ class PTESolverBase {
     }
   }
 
-  PORTABLE_FORCEINLINE_FUNCTION
+  PORTABLE_INLINE_FUNCTION
   void SetVfracFromT(const Real T) {
     Real vsum = 0.0;
     // set volume fractions
@@ -233,7 +233,7 @@ class PTESolverBase {
     }
   }
 
-  PORTABLE_FORCEINLINE_FUNCTION
+  PORTABLE_INLINE_FUNCTION
   Real GetTguess() {
     // guess some non-zero temperature to start
     // If a guess was passed in, it's saved in Tnorm
@@ -272,9 +272,9 @@ class PTESolverBase {
     return Tguess;
   }
 
-  PORTABLE_FORCEINLINE_FUNCTION
-  static Real GetPressureFromPreferred(const EOS &eos, const Real rho, const Real T,
-                                       Real sie, Real *lambda, const bool do_e_lookup) {
+  PORTABLE_INLINE_FUNCTION
+  Real GetPressureFromPreferred(const EOS &eos, const Real rho, const Real T, Real sie,
+                                Real *lambda, const bool do_e_lookup) const {
     Real P{};
     if (eos.PreferredInput() ==
         (thermalqs::density | thermalqs::specific_internal_energy)) {

singularity-eos/eos/eos.hpp

@@ -55,7 +55,7 @@ namespace singularity {
 template <typename... Ts>
 using tl = singularity::detail::type_list<Ts...>;
 
-template <template <typename> typename... Ts>
+template <template <typename> class... Ts>
 using al = singularity::detail::adapt_list<Ts...>;
 
 // transform variadic list: applies modifiers to eos's

singularity-eos/eos/eos_base.hpp

@@ -80,7 +80,10 @@ char *StrCat(char *destination, const char *source) {
   using EosBase<EOSDERIVED>::FillEos;                                                    \
   using EosBase<EOSDERIVED>::EntropyFromDensityTemperature;                              \
   using EosBase<EOSDERIVED>::EntropyFromDensityInternalEnergy;                           \
-  using EosBase<EOSDERIVED>::EntropyIsNotEnabled;
+  using EosBase<EOSDERIVED>::EntropyIsNotEnabled;                                        \
+  using EosBase<EOSDERIVED>::IsModified;                                                 \
+  using EosBase<EOSDERIVED>::UnmodifyOnce;                                               \
+  using EosBase<EOSDERIVED>::GetUnmodifiedObject;
 
 /*
 This is a CRTP that allows for static inheritance so that default behavior for
@@ -426,12 +429,13 @@ class EosBase {
   }
 
   // Tooling for modifiers
-  PORTABLE_FORCEINLINE_FUNCTION
-  bool IsModified() const { return false; }
-  PORTABLE_FORCEINLINE_FUNCTION
-  auto UnmodifyOnce() { return *static_cast<CRTP *>(this); }
-  PORTABLE_FORCEINLINE_FUNCTION
-  auto GetUnmodifiedObject() { return *static_cast<CRTP *>(this); }
+  inline constexpr bool IsModified() const { return false; }
+
+  inline constexpr decltype(auto) UnmodifyOnce() { return *static_cast<CRTP *>(this); }
+
+  inline constexpr decltype(auto) GetUnmodifiedObject() {
+    return *static_cast<CRTP *>(this);
+  }
 };
 } // namespace eos_base
 } // namespace singularity

singularity-eos/eos/eos_eospac.hpp

@@ -41,41 +41,62 @@ using namespace eos_base;
 
 // Only really works in serial
 // Not really supported on device
+#if defined(__CUDACC__)
+#define SG_PIF_NOWARN #pragma nv_exec_check_disable
+#else
+#define SG_PIF_NOWARN
+#endif
+
 class EOSPAC : public EosBase<EOSPAC> {
  public:
   inline EOSPAC() = default;
   inline EOSPAC(int matid, bool invert_at_setup = false);
   inline EOSPAC GetOnDevice() { return *this; }
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real TemperatureFromDensityInternalEnergy(
       const Real rho, const Real sie, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real InternalEnergyFromDensityTemperature(
       const Real rho, const Real temperature, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real PressureFromDensityTemperature(
       const Real rho, const Real temperature, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real PressureFromDensityInternalEnergy(
       const Real rho, const Real sie, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real EntropyFromDensityTemperature(
       const Real rho, const Real temperature, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real EntropyFromDensityInternalEnergy(
       const Real rho, const Real sie, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real SpecificHeatFromDensityTemperature(
       const Real rho, const Real temperature, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real SpecificHeatFromDensityInternalEnergy(
       const Real rho, const Real sie, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real BulkModulusFromDensityTemperature(
       const Real rho, const Real temperature, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real BulkModulusFromDensityInternalEnergy(
       const Real rho, const Real sie, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real GruneisenParamFromDensityTemperature(
       const Real rho, const Real temperature, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION Real GruneisenParamFromDensityInternalEnergy(
       const Real rho, const Real sie, Real *lambda = nullptr) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION void FillEos(Real &rho, Real &temp, Real &energy, Real &press,
                                         Real &cv, Real &bmod, const unsigned long output,
                                         Real *lambda = nullptr) const;
-  PORTABLE_INLINE_FUNCTION
-  void DensityEnergyFromPressureTemperature(const Real press, const Real temp,
-                                            Real *lambda, Real &rho, Real &sie) const;
+  SG_PIF_NOWARN
+  PORTABLE_INLINE_FUNCTION void
+  DensityEnergyFromPressureTemperature(const Real press, const Real temp, Real *lambda,
+                                       Real &rho, Real &sie) const;
+  SG_PIF_NOWARN
   PORTABLE_INLINE_FUNCTION void ValuesAtReferenceState(Real &rho, Real &temp, Real &sie,
                                                        Real &press, Real &cv, Real &bmod,
                                                        Real &dpde, Real &dvdt,
@@ -871,9 +892,10 @@ class EOSPAC : public EosBase<EOSPAC> {
         });
   }
 
-  PORTABLE_INLINE_FUNCTION
-  void PTofRE(Real &rho, Real &sie, Real *lambda, Real &press, Real &temp, Real &dpdr,
-              Real &dpde, Real &dtdr, Real &dtde) const {
+  SG_PIF_NOWARN
+  PORTABLE_INLINE_FUNCTION void PTofRE(Real &rho, Real &sie, Real *lambda, Real &press,
+                                       Real &temp, Real &dpdr, Real &dpde, Real &dtdr,
+                                       Real &dtde) const {
 
     press = PressureFromDensityInternalEnergy(rho, sie, lambda);
     temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
@@ -912,7 +934,9 @@ class EOSPAC : public EosBase<EOSPAC> {
   inline void Finalize() {}
   static std::string EosType() { return std::string("EOSPAC"); }
   static std::string EosPyType() { return EosType(); }
-  inline void PrintParams() const { printf("EOSPAC parameters:\nmatid = %s\n", matid_); }
+  PORTABLE_INLINE_FUNCTION void PrintParams() const {
+    printf("EOSPAC parameters:\nmatid = %i\n", matid_);
+  }
   PORTABLE_FORCEINLINE_FUNCTION Real MinimumDensity() const { return rho_min_; }
   PORTABLE_FORCEINLINE_FUNCTION Real MinimumTemperature() const { return temp_min_; }
 
@@ -1006,6 +1030,7 @@ inline EOSPAC::EOSPAC(const int matid, bool invert_at_setup) : matid_(matid) {
       robust::ratio(dpde_ref_ * cv_ref_, rho_ref_ * rho_ref_ * pressureFromSesame(DPDR));
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::TemperatureFromDensityInternalEnergy(
     const Real rho, const Real sie, Real *lambda) const {
   using namespace EospacWrapper;
@@ -1016,6 +1041,7 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::TemperatureFromDensityInternalEnergy(
   return Real(temperatureFromSesame(T[0]));
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::PressureFromDensityTemperature(const Real rho,
                                                                      const Real temp,
                                                                      Real *lambda) const {
@@ -1027,12 +1053,14 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::PressureFromDensityTemperature(const Real
   return Real(pressureFromSesame(P[0]));
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::EntropyFromDensityTemperature(
     const Real rho, const Real temperature, Real *lambda) const {
   EntropyIsNotEnabled("EOSPAC");
   return 1.0;
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION void EOSPAC::FillEos(Real &rho, Real &temp, Real &sie,
                                               Real &press, Real &cv, Real &bmod,
                                               const unsigned long output,
@@ -1042,7 +1070,7 @@ PORTABLE_INLINE_FUNCTION void EOSPAC::FillEos(Real &rho, Real &temp, Real &sie,
   EOS_REAL E[1] = {sie}, P[1] = {pressureToSesame(press)};
   EOS_REAL dx[1], dy[1];
   EOS_INTEGER nxypairs = 1;
-  Real CV, BMOD_T, BMOD, SIE, PRESS, DPDE, DPDT, DPDR, DEDT, DEDR;
+  Real /*CV,*/ BMOD_T, BMOD, SIE, PRESS, DPDE, DPDT, DPDR, DEDT, DEDR;
   const unsigned long input = ~output;
   if (output == thermalqs::none) {
     UNDEFINED_ERROR;
@@ -1116,6 +1144,7 @@ PORTABLE_INLINE_FUNCTION void EOSPAC::FillEos(Real &rho, Real &temp, Real &sie,
   }
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::InternalEnergyFromDensityTemperature(
     const Real rho, const Real temp, Real *lambda) const {
   using namespace EospacWrapper;
@@ -1124,6 +1153,7 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::InternalEnergyFromDensityTemperature(
   FillEos(RHO, TEMP, sie, press, cv, bmod, output, lambda);
   return sie;
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::BulkModulusFromDensityTemperature(
     const Real rho, const Real temp, Real *lambda) const {
   using namespace EospacWrapper;
@@ -1132,6 +1162,7 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::BulkModulusFromDensityTemperature(
   FillEos(RHO, TEMP, sie, press, cv, bmod, output, lambda);
   return bmod;
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::SpecificHeatFromDensityTemperature(
     const Real rho, const Real temp, Real *lambda) const {
   using namespace EospacWrapper;
@@ -1140,31 +1171,36 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::SpecificHeatFromDensityTemperature(
   FillEos(RHO, TEMP, sie, press, cv, bmod, output, lambda);
   return cv;
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::PressureFromDensityInternalEnergy(
     const Real rho, const Real sie, Real *lambda) const {
   using namespace EospacWrapper;
   Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return PressureFromDensityTemperature(rho, temp, lambda);
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::EntropyFromDensityInternalEnergy(
     const Real rho, const Real sie, Real *lambda) const {
   using namespace EospacWrapper;
   const Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return EntropyFromDensityTemperature(rho, temp, lambda);
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::SpecificHeatFromDensityInternalEnergy(
     const Real rho, const Real sie, Real *lambda) const {
   using namespace EospacWrapper;
   Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return SpecificHeatFromDensityTemperature(rho, temp, lambda);
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::BulkModulusFromDensityInternalEnergy(
     const Real rho, const Real sie, Real *lambda) const {
   using namespace EospacWrapper;
   Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return BulkModulusFromDensityTemperature(rho, temp, lambda);
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::GruneisenParamFromDensityTemperature(
     const Real rho, const Real temperature, Real *lambda) const {
   using namespace EospacWrapper;
@@ -1181,22 +1217,23 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::GruneisenParamFromDensityTemperature(
   DPDE = DPDT / DEDT;
   return robust::ratio(pressureFromSesame(sieToSesame(DPDE)), rho);
 }
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION Real EOSPAC::GruneisenParamFromDensityInternalEnergy(
     const Real rho, const Real sie, Real *lambda) const {
   Real temperature = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return GruneisenParamFromDensityTemperature(rho, temperature, lambda);
 }
 
-PORTABLE_INLINE_FUNCTION
-void EOSPAC::DensityEnergyFromPressureTemperature(const Real press, const Real temp,
-                                                  Real *lambda, Real &rho,
-                                                  Real &sie) const {
+SG_PIF_NOWARN
+PORTABLE_INLINE_FUNCTION void
+EOSPAC::DensityEnergyFromPressureTemperature(const Real press, const Real temp,
+                                             Real *lambda, Real &rho, Real &sie) const {
   using namespace EospacWrapper;
   EOS_REAL P[1] = {pressureToSesame(press)};
   EOS_REAL T[1] = {temperatureToSesame(temp)};
   EOS_REAL dx[1], dy[1], R[1], E[1];
   EOS_INTEGER nxypairs = 1;
-  EOS_INTEGER errorCode;
+  // EOS_INTEGER errorCode;
   EOS_INTEGER table;
 
   table = RofPT_table_;
@@ -1208,6 +1245,7 @@ void EOSPAC::DensityEnergyFromPressureTemperature(const Real press, const Real t
   sie = sieFromSesame(E[0]);
 }
 
+SG_PIF_NOWARN
 PORTABLE_INLINE_FUNCTION void
 EOSPAC::ValuesAtReferenceState(Real &rho, Real &temp, Real &sie, Real &press, Real &cv,
                                Real &bmod, Real &dpde, Real &dvdt, Real *lambda) const {