Merge pull request #6360 from kidder/new_lts_events_and_triggers

Add ability to RunEventsAndTriggers at steps within a slab for local time-stepping

src/Elliptic/Actions/RunEventsAndTriggers.hpp

@@ -8,6 +8,7 @@
 #include "Parallel/AlgorithmExecution.hpp"
 #include "Parallel/GlobalCache.hpp"
 #include "ParallelAlgorithms/EventsAndTriggers/Tags.hpp"
+#include "ParallelAlgorithms/EventsAndTriggers/WhenToCheck.hpp"
 #include "Utilities/Gsl.hpp"
 #include "Utilities/TMPL.hpp"
 #include "Utilities/TaggedTuple.hpp"
@@ -28,7 +29,8 @@ namespace elliptic::Actions {
 /// - Modifies: nothing
 template <typename ObservationId>
 struct RunEventsAndTriggers {
-  using const_global_cache_tags = tmpl::list<Tags::EventsAndTriggers>;
+  using const_global_cache_tags =
+      tmpl::list<Tags::EventsAndTriggers<Triggers::WhenToCheck::AtIterations>>;
 
   template <typename DbTags, typename... InboxTags, typename Metavariables,
             typename ArrayIndex, typename ActionList,
@@ -38,10 +40,11 @@ struct RunEventsAndTriggers {
       Parallel::GlobalCache<Metavariables>& cache,
       const ArrayIndex& array_index, const ActionList /*meta*/,
       const ParallelComponent* const component) {
-    Parallel::get<Tags::EventsAndTriggers>(cache).run_events(
-        make_not_null(&box), cache, array_index, component,
-        {db::tag_name<ObservationId>(),
-         static_cast<double>(db::get<ObservationId>(box))});
+    Parallel::get<Tags::EventsAndTriggers<Triggers::WhenToCheck::AtIterations>>(
+        cache)
+        .run_events(make_not_null(&box), cache, array_index, component,
+                    {db::tag_name<ObservationId>(),
+                     static_cast<double>(db::get<ObservationId>(box))});
 
     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
   }

src/Evolution/Actions/RunEventsAndTriggers.hpp

@@ -10,6 +10,7 @@
 #include "Parallel/AlgorithmExecution.hpp"
 #include "Parallel/GlobalCache.hpp"
 #include "ParallelAlgorithms/EventsAndTriggers/Tags.hpp"
+#include "ParallelAlgorithms/EventsAndTriggers/WhenToCheck.hpp"
 #include "Time/SelfStart.hpp"
 #include "Time/Tags/Time.hpp"
 #include "Time/Triggers/OnSubsteps.hpp"
@@ -24,6 +25,37 @@ struct TimeStepId;
 /// \endcond
 
 namespace evolution::Actions {
+namespace detail {
+template <typename DbTags, typename Metavariables, typename ArrayIndex,
+          typename ParallelComponent, typename EventsAndTriggers_t>
+void run_events_and_triggers(db::DataBox<DbTags>& box,
+                             Parallel::GlobalCache<Metavariables>& cache,
+                             const ArrayIndex& array_index,
+                             const ParallelComponent* const component,
+                             const EventsAndTriggers_t& events_and_triggers,
+                             const TimeStepId& time_step_id) {
+  if (time_step_id.substep() == 0) {
+    events_and_triggers.run_events(
+        make_not_null(&box), cache, array_index, component,
+        {db::tag_name<::Tags::Time>(), db::get<::Tags::Time>(box)});
+  } else {
+    const double substep_offset = 1.0e6;
+    const double observation_value =
+        time_step_id.step_time().value() +
+        substep_offset * static_cast<double>(time_step_id.substep());
+    events_and_triggers.run_events(
+        make_not_null(&box), cache, array_index, component,
+        {db::tag_name<::Tags::Time>(), observation_value},
+        [&box](const Trigger& trigger) {
+          const auto* substep_trigger =
+              dynamic_cast<const ::Triggers::OnSubsteps*>(&trigger);
+          return substep_trigger != nullptr and
+                 substep_trigger->is_triggered(box);
+        });
+  }
+}
+}  // namespace detail
+
 /// \ingroup ActionsGroup
 /// \ingroup EventsAndTriggersGroup
 /// \brief Run the events and triggers
@@ -40,8 +72,14 @@ namespace evolution::Actions {
 /// - Adds: nothing
 /// - Removes: nothing
 /// - Modifies: nothing
+template <bool LocalTimeStepping>
 struct RunEventsAndTriggers {
-  using const_global_cache_tags = tmpl::list<::Tags::EventsAndTriggers>;
+  static constexpr bool local_time_stepping = LocalTimeStepping;
+  using const_global_cache_tags = tmpl::conditional_t<
+      local_time_stepping,
+      tmpl::list<::Tags::EventsAndTriggers<Triggers::WhenToCheck::AtSlabs>,
+                 ::Tags::EventsAndTriggers<Triggers::WhenToCheck::AtSteps>>,
+      tmpl::list<::Tags::EventsAndTriggers<Triggers::WhenToCheck::AtSlabs>>>;
 
   template <typename DbTags, typename... InboxTags, typename Metavariables,
             typename ArrayIndex, typename ActionList,
@@ -52,31 +90,26 @@ struct RunEventsAndTriggers {
       const ArrayIndex& array_index, const ActionList /*meta*/,
       const ParallelComponent* const component) {
     const auto time_step_id = db::get<::Tags::TimeStepId>(box);
-    if (SelfStart::is_self_starting(time_step_id) or
-        not time_step_id.step_time().is_at_slab_boundary()) {
+    if (SelfStart::is_self_starting(time_step_id)) {
       return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     }
 
-    if (time_step_id.substep() == 0) {
-      Parallel::get<::Tags::EventsAndTriggers>(cache).run_events(
-          make_not_null(&box), cache, array_index, component,
-          {db::tag_name<::Tags::Time>(), db::get<::Tags::Time>(box)});
-    } else {
-      const double substep_offset = 1.0e6;
-      const double observation_value = time_step_id.step_time().value() +
-                                       substep_offset * time_step_id.substep();
+    if constexpr (local_time_stepping) {
+      const auto& events_and_triggers_at_steps = Parallel::get<
+          ::Tags::EventsAndTriggers<Triggers::WhenToCheck::AtSteps>>(cache);
+      detail::run_events_and_triggers(box, cache, array_index, component,
+                                      events_and_triggers_at_steps,
+                                      time_step_id);
+    }
 
-      Parallel::get<::Tags::EventsAndTriggers>(cache).run_events(
-          make_not_null(&box), cache, array_index, component,
-          {db::tag_name<::Tags::Time>(), observation_value},
-          [&box](const Trigger& trigger) {
-            const auto* substep_trigger =
-                dynamic_cast<const ::Triggers::OnSubsteps*>(&trigger);
-            return substep_trigger != nullptr and
-                   substep_trigger->is_triggered(box);
-          });
+    if (not time_step_id.step_time().is_at_slab_boundary()) {
+      return {Parallel::AlgorithmExecution::Continue, std::nullopt};
     }
 
+    const auto& events_and_triggers_at_slabs = Parallel::get<
+        ::Tags::EventsAndTriggers<Triggers::WhenToCheck::AtSlabs>>(cache);
+    detail::run_events_and_triggers(box, cache, array_index, component,
+                                    events_and_triggers_at_slabs, time_step_id);
     return {Parallel::AlgorithmExecution::Continue, std::nullopt};
   }
 };

src/Evolution/Executables/Burgers/EvolveBurgers.hpp

@@ -333,10 +333,10 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/CurvedScalarWave/EvolveCurvedScalarWave.hpp

@@ -304,10 +304,10 @@ struct EvolutionMetavars {
                                             Parallel::Actions::TerminatePhase>>,
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/CurvedScalarWave/EvolveWorldtubeCurvedScalarWave.hpp

@@ -329,10 +329,10 @@ struct EvolutionMetavars {
                                             Parallel::Actions::TerminatePhase>>,
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/ForceFree/EvolveForceFree.hpp

@@ -259,10 +259,11 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, dg_step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, dg_step_actions,
+                  Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/GeneralizedHarmonic/EvolveGhBinaryBlackHole.hpp

@@ -616,7 +616,7 @@ struct EvolutionMetavars {
               Parallel::Phase::Evolve,
               tmpl::list<
                   ::domain::Actions::CheckFunctionsOfTimeAreReady<volume_dim>,
-                  evolution::Actions::RunEventsAndTriggers,
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
                   Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
                   PhaseControl::Actions::ExecutePhaseChange>>>>>;
 

src/Evolution/Executables/GeneralizedHarmonic/EvolveGhNoBlackHole.hpp

@@ -74,7 +74,8 @@ struct EvolutionMetavars
                                             Parallel::Actions::TerminatePhase>>,
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<::evolution::Actions::RunEventsAndTriggers,
+              tmpl::list<::evolution::Actions::RunEventsAndTriggers<
+                             local_time_stepping>,
                          Actions::ChangeSlabSize, step_actions,
                          Actions::AdvanceTime,
                          PhaseControl::Actions::ExecutePhaseChange>>>>>;

src/Evolution/Executables/GeneralizedHarmonic/EvolveGhSingleBlackHole.hpp

@@ -80,6 +80,7 @@
 
 template <bool UseLts>
 struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3, UseLts> {
+  static constexpr bool local_time_stepping = UseLts;
   static constexpr size_t volume_dim = 3;
   using gh_base = GeneralizedHarmonicTemplateBase<volume_dim, UseLts>;
   using typename gh_base::initialize_initial_data_dependent_quantities_actions;
@@ -260,7 +261,7 @@ struct EvolutionMetavars : public GeneralizedHarmonicTemplateBase<3, UseLts> {
               Parallel::Phase::Evolve,
               tmpl::list<
                   ::domain::Actions::CheckFunctionsOfTimeAreReady<volume_dim>,
-                  evolution::Actions::RunEventsAndTriggers,
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
                   Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
                   PhaseControl::Actions::ExecutePhaseChange>>>>>;
 

src/Evolution/Executables/GrMhd/GhValenciaDivClean/GhValenciaDivCleanBase.hpp

@@ -866,7 +866,7 @@ struct GhValenciaDivCleanTemplateBase<
                   VariableFixing::Actions::FixVariables<
                       VariableFixing::LimitLorentzFactor>,
                   Actions::UpdateConservatives,
-                  evolution::Actions::RunEventsAndTriggers,
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
                   Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
                   PhaseControl::Actions::ExecutePhaseChange>>,
           Parallel::PhaseActions<

src/Evolution/Executables/GrMhd/ValenciaDivClean/EvolveValenciaDivClean.hpp

@@ -580,28 +580,29 @@ struct EvolutionMetavars<tmpl::list<InterpolationTargetTags...>,
 
   using dg_element_array_component = DgElementArray<
       EvolutionMetavars,
-      tmpl::list<Parallel::PhaseActions<Parallel::Phase::Initialization,
-                                        initialization_actions>,
-
-                 Parallel::PhaseActions<
-                     Parallel::Phase::InitializeTimeStepperHistory,
-                     SelfStart::self_start_procedure<step_actions, system>>,
-
-                 Parallel::PhaseActions<
-                     Parallel::Phase::Register,
-                     tmpl::push_back<dg_registration_list,
-                                     Parallel::Actions::TerminatePhase>>,
-
-                 Parallel::PhaseActions<
-                     Parallel::Phase::Evolve,
-                     tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                                Actions::ChangeSlabSize, step_actions,
-                                Actions::AdvanceTime,
-                                PhaseControl::Actions::ExecutePhaseChange>>,
-                 Parallel::PhaseActions<
-                     Parallel::Phase::PostFailureCleanup,
-                     tmpl::list<Actions::RunEventsOnFailure<Tags::Time>,
-                                Parallel::Actions::TerminatePhase>>>>;
+      tmpl::list<
+          Parallel::PhaseActions<Parallel::Phase::Initialization,
+                                 initialization_actions>,
+
+          Parallel::PhaseActions<
+              Parallel::Phase::InitializeTimeStepperHistory,
+              SelfStart::self_start_procedure<step_actions, system>>,
+
+          Parallel::PhaseActions<
+              Parallel::Phase::Register,
+              tmpl::push_back<dg_registration_list,
+                              Parallel::Actions::TerminatePhase>>,
+
+          Parallel::PhaseActions<
+              Parallel::Phase::Evolve,
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>,
+          Parallel::PhaseActions<
+              Parallel::Phase::PostFailureCleanup,
+              tmpl::list<Actions::RunEventsOnFailure<Tags::Time>,
+                         Parallel::Actions::TerminatePhase>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/NewtonianEuler/EvolveNewtonianEuler.hpp

@@ -389,10 +389,10 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/RadiationTransport/M1Grey/EvolveM1Grey.hpp

@@ -259,10 +259,10 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/RelativisticEuler/Valencia/EvolveValencia.hpp

@@ -135,8 +135,7 @@ struct EvolutionMetavars {
   using analytic_variables_tags =
       typename system::primitive_variables_tag::tags_list;
 
-  using equation_of_state_tag =
-      hydro::Tags::EquationOfState<equation_of_state_type>;
+  using equation_of_state_tag = hydro::Tags::EquationOfState<true, 2>;
 
   using limiter = Tags::Limiter<Limiters::Minmod<
       Dim, tmpl::list<RelativisticEuler::Valencia::Tags::TildeD,
@@ -269,13 +268,13 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<VariableFixing::Actions::FixVariables<
-                             VariableFixing::FixToAtmosphere<volume_dim>>,
-                         Actions::UpdateConservatives,
-                         evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  VariableFixing::Actions::FixVariables<
+                      VariableFixing::FixToAtmosphere<volume_dim>>,
+                  Actions::UpdateConservatives,
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/ScalarAdvection/EvolveScalarAdvection.hpp

@@ -359,10 +359,10 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct registration
       : tt::ConformsTo<Parallel::protocols::RegistrationMetavariables> {

src/Evolution/Executables/ScalarTensor/EvolveScalarTensorSingleBlackHole.hpp

@@ -215,7 +215,7 @@ struct EvolutionMetavars : public ScalarTensorTemplateBase<EvolutionMetavars> {
               Parallel::Phase::Evolve,
               tmpl::list<
                   ::domain::Actions::CheckFunctionsOfTimeAreReady<volume_dim>,
-                  evolution::Actions::RunEventsAndTriggers,
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
                   Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
                   PhaseControl::Actions::ExecutePhaseChange>>>>>;
 

src/Evolution/Executables/ScalarWave/EvolveScalarWave.hpp

@@ -295,10 +295,10 @@ struct EvolutionMetavars {
 
           Parallel::PhaseActions<
               Parallel::Phase::Evolve,
-              tmpl::list<evolution::Actions::RunEventsAndTriggers,
-                         Actions::ChangeSlabSize, step_actions,
-                         Actions::AdvanceTime,
-                         PhaseControl::Actions::ExecutePhaseChange>>>>;
+              tmpl::list<
+                  evolution::Actions::RunEventsAndTriggers<local_time_stepping>,
+                  Actions::ChangeSlabSize, step_actions, Actions::AdvanceTime,
+                  PhaseControl::Actions::ExecutePhaseChange>>>>;
 
   struct amr : tt::ConformsTo<::amr::protocols::AmrMetavariables> {
     using element_array = dg_element_array;