From 6adfa1d553c6533ea988b2ed9bb70b264057ea8b Mon Sep 17 00:00:00 2001 From: mari2895 Date: Tue, 6 Feb 2024 17:34:50 -0500 Subject: [PATCH 1/4] updated phoebus based on recent tasking rewrite changes in parthenon PR #987 --- src/fixup/fixup.cpp | 3 +- src/fixup/fixup_c2p.cpp | 3 +- src/fixup/fixup_radc2p.cpp | 3 +- src/fixup/fixup_src.cpp | 3 +- src/phoebus_driver.cpp | 82 +++++++++++++------------------- src/radiation/moments.cpp | 8 ++-- src/radiation/moments_source.cpp | 2 +- 7 files changed, 45 insertions(+), 59 deletions(-) diff --git a/src/fixup/fixup.cpp b/src/fixup/fixup.cpp index 73d6dcc2..f6980de1 100644 --- a/src/fixup/fixup.cpp +++ b/src/fixup/fixup.cpp @@ -33,6 +33,7 @@ #include "radiation/closure_m1.hpp" #include "radiation/closure_mocmc.hpp" #include "radiation/radiation.hpp" +#include using Microphysics::RadiationType; using Microphysics::EOS::EOS; @@ -700,7 +701,7 @@ TaskStatus ApplyFloors(T *rc) { "Rad floors not supported with cooling function/Monte Carlo!"); return ApplyFloorsImpl>(rc); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus ApplyFloors>(MeshBlockData *rc); diff --git a/src/fixup/fixup_c2p.cpp b/src/fixup/fixup_c2p.cpp index eb9bf5c2..70ed43e2 100644 --- a/src/fixup/fixup_c2p.cpp +++ b/src/fixup/fixup_c2p.cpp @@ -31,6 +31,7 @@ #include "radiation/closure_m1.hpp" #include "radiation/closure_mocmc.hpp" #include "radiation/radiation.hpp" +#include using radiation::ClosureEquation; using radiation::ClosureSettings; @@ -377,7 +378,7 @@ TaskStatus ConservedToPrimitiveFixup(T *rc) { } else { return ConservedToPrimitiveFixupImpl>(rc); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus diff --git a/src/fixup/fixup_radc2p.cpp b/src/fixup/fixup_radc2p.cpp index 11d4ed66..200f6505 100644 --- a/src/fixup/fixup_radc2p.cpp +++ b/src/fixup/fixup_radc2p.cpp @@ -30,6 +30,7 @@ #include "radiation/closure_m1.hpp" #include "radiation/closure_mocmc.hpp" #include "radiation/radiation.hpp" +#include using Microphysics::RadiationType; using radiation::ClosureEquation; @@ -246,7 +247,7 @@ TaskStatus RadConservedToPrimitiveFixup(T *rc) { } else if (method == "mocmc") { return RadConservedToPrimitiveFixupImpl>(rc); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus diff --git a/src/fixup/fixup_src.cpp b/src/fixup/fixup_src.cpp index c96eacde..a3867f96 100644 --- a/src/fixup/fixup_src.cpp +++ b/src/fixup/fixup_src.cpp @@ -31,6 +31,7 @@ #include "radiation/closure_m1.hpp" #include "radiation/closure_mocmc.hpp" #include "radiation/radiation.hpp" +#include using Microphysics::RadiationType; using Microphysics::EOS::EOS; @@ -346,7 +347,7 @@ TaskStatus SourceFixup(T *rc) { } else if (method == "mocmc") { return SourceFixupImpl>(rc); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus SourceFixup>(MeshBlockData *rc); diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 9008c61e..8d5a3594 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -360,38 +360,33 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // TODO(BRR) Setting scale factor of B field need only be done once per simulation // Evaluate current Mdot, Phi - auto sum_mdot_1 = tl.AddTask(none, fixup::SumMdotPhiForNetFieldScaling, md.get(), t, - stage, &(net_field_totals->val)); - - sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, sum_mdot_1); - sync_region_5_dep_id++; + auto sum_mdot_1 = + tl.AddTask(TaskQualifier::local_sync, none, fixup::SumMdotPhiForNetFieldScaling, + md.get(), t, stage, &(net_field_totals->val)); TaskID start_reduce_1 = (i == 0 ? tl.AddTask(sum_mdot_1, fixup::NetFieldStartReduce, md.get(), t, stage, net_field_totals) : none); // Test the reduction until it completes - TaskID finish_reduce_1 = tl.AddTask(start_reduce_1, fixup::NetFieldCheckReduce, - md.get(), t, stage, net_field_totals); - sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, finish_reduce_1); - sync_region_5_dep_id++; + TaskID finish_reduce_1 = + tl.AddTask(TaskQualifier::local_sync, start_reduce_1, fixup::NetFieldCheckReduce, + md.get(), t, stage, net_field_totals); auto mod_net = tl.AddTask(finish_reduce_1, fixup::ModifyNetField, md.get(), t, beta * dt, stage, true, 1.); // Evaluate Mdot, Phi (only Phi changes) after modifying B field - auto sum_mdot_2 = tl.AddTask(mod_net, fixup::SumMdotPhiForNetFieldScaling, md.get(), - t, stage, &(net_field_totals_2->val)); - sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, sum_mdot_2); - sync_region_5_dep_id++; + auto sum_mdot_2 = tl.AddTask(TaskQualifier::local_sync, mod_net, + fixup::SumMdotPhiForNetFieldScaling, md.get(), t, stage, + &(net_field_totals_2->val)); TaskID start_reduce_2 = (i == 0 ? tl.AddTask(sum_mdot_2, fixup::NetFieldStartReduce, md.get(), t, stage, net_field_totals_2) : none); // Test the reduction until it completes - TaskID finish_reduce_2 = tl.AddTask(start_reduce_2, fixup::NetFieldCheckReduce, - md.get(), t, stage, net_field_totals_2); - sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, finish_reduce_2); - sync_region_5_dep_id++; + TaskID finish_reduce_2 = + tl.AddTask(TaskQualifier::local_sync, start_reduce_2, fixup::NetFieldCheckReduce, + md.get(), t, stage, net_field_totals_2); // Remove artificial contribution to net field auto mod_net_2 = tl.AddTask(finish_reduce_2, fixup::ModifyNetField, md.get(), t, @@ -441,16 +436,15 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { TaskID interp_to_monopole = none; if (monopole_gr_active) { auto interp_to_monopole = - tl.AddTask(none, MonopoleGR::InterpolateMatterTo1D, sc0.get()); - sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, interp_to_monopole); + tl.AddTask(TaskQualifier::local_sync, none, + MonopoleGR::InterpolateMatterTo1D, sc0.get()); // TODO(JMM): Is this the right place for this? // TODO(JMM): Should this stuff be in the synchronous region? auto matter_to_host = - (ib == 0 ? tl.AddTask(interp_to_monopole, MonopoleGR::MatterToHost, - monopole.get(), true) + (ib == 0 ? tl.AddTask(TaskQualifier::local_sync, interp_to_monopole, + MonopoleGR::MatterToHost, monopole.get(), true) : none); - sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, matter_to_host); auto start_reduce_matter = (ib == 0 ? tl.AddTask(matter_to_host, &MonoMatRed_t::StartReduce, pmono_mat_red, MPI_SUM) @@ -460,11 +454,10 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { MPI_SUM) : none); auto finish_reduce_matter = - tl.AddTask(start_reduce_matter, &MonoMatRed_t::CheckReduce, pmono_mat_red); - sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, finish_reduce_matter); - auto finish_reduce_vols = - tl.AddTask(start_reduce_vols, &MonoVolRed_t::CheckReduce, pmono_vol_red); - sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, finish_reduce_vols); + tl.AddTask(TaskQualifier::local_sync, start_reduce_matter, + &MonoMatRed_t::CheckReduce, pmono_mat_red); + auto finish_reduce_vols = tl.AddTask(TaskQualifier::local_sync, start_reduce_vols, + &MonoVolRed_t::CheckReduce, pmono_vol_red); auto finish_mono_reds = finish_reduce_matter | finish_reduce_vols; auto divide_vols = (ib == 0 ? tl.AddTask(finish_mono_reds, MonopoleGR::DivideVols, monopole.get()) @@ -614,21 +607,17 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { int reg_dep_id = 0; // auto &tl = async_region_3[0]; - auto update_count = tl.AddTask(none, radiation::MOCMCUpdateParticleCount, pmesh, - &particle_resolution->val); - - async_region_3.AddRegionalDependencies(reg_dep_id, 0, update_count); - reg_dep_id++; + auto update_count = + tl.AddTask(TaskQualifier::local_sync, none, radiation::MOCMCUpdateParticleCount, + pmesh, &particle_resolution->val); auto start_count_reduce = tl.AddTask(update_count, &AllReduce>::StartReduce, particle_resolution, MPI_SUM); auto finish_count_reduce = - tl.AddTask(start_count_reduce, &AllReduce>::CheckReduce, - particle_resolution); - async_region_3.AddRegionalDependencies(reg_dep_id, 0, finish_count_reduce); - reg_dep_id++; + tl.AddTask(TaskQualifier::local_sync, start_count_reduce, + &AllReduce>::CheckReduce, particle_resolution); // Report particle count auto report_count = @@ -715,10 +704,8 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer, MPI_LOR) : none); finish_gain_reducer = - tl.AddTask(start_gain_reducer, &parthenon::AllReduce::CheckReduce, - pdo_gain_reducer); - int reg_dep_id = 0; - sync_region_lb.AddRegionalDependencies(reg_dep_id++, ib, finish_gain_reducer); + tl.AddTask(TaskQualifier::local_sync, start_gain_reducer, + &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); } auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, @@ -935,22 +922,17 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { int reg_dep_id = 0; auto &tl = tuning_region[0]; - auto update_resolution = - tl.AddTask(none, radiation::MonteCarloUpdateParticleResolution, pmesh, - &particle_resolution->val); - - tuning_region.AddRegionalDependencies(reg_dep_id, 0, update_resolution); - reg_dep_id++; + auto update_resolution = tl.AddTask(TaskQualifier::local_sync, none, + radiation::MonteCarloUpdateParticleResolution, + pmesh, &particle_resolution->val); auto start_resolution_reduce = tl.AddTask(update_resolution, &AllReduce>::StartReduce, particle_resolution, MPI_SUM); auto finish_resolution_reduce = - tl.AddTask(start_resolution_reduce, &AllReduce>::CheckReduce, - particle_resolution); - tuning_region.AddRegionalDependencies(reg_dep_id, 0, finish_resolution_reduce); - reg_dep_id++; + tl.AddTask(TaskQualifier::local_sync, start_resolution_reduce, + &AllReduce>::CheckReduce, particle_resolution); // Report tuning auto report_resolution = diff --git a/src/radiation/moments.cpp b/src/radiation/moments.cpp index 293c71ed..aa1633cd 100644 --- a/src/radiation/moments.cpp +++ b/src/radiation/moments.cpp @@ -189,7 +189,7 @@ TaskStatus MomentCon2Prim(T *rc) { } else { PARTHENON_FAIL("Radiation method unknown"); } - return TaskStatus::fail; + return TaskStatus::complete; } // template TaskStatus MomentCon2Prim>(MeshData *); template TaskStatus MomentCon2Prim>(MeshBlockData *); @@ -287,7 +287,7 @@ TaskStatus MomentPrim2Con(T *rc, IndexDomain domain) { } else { PARTHENON_FAIL("Radiation method unknown!"); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus MomentPrim2Con>(MeshBlockData *, @@ -768,7 +768,7 @@ TaskStatus CalculateFluxes(T *rc) { } else { PARTHENON_FAIL("Radiation method unknown!"); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus CalculateFluxes>(MeshBlockData *); @@ -928,7 +928,7 @@ TaskStatus CalculateGeometricSource(T *rc, T *rc_src) { } else { PARTHENON_FAIL("Radiation method unknown!"); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus CalculateGeometricSource>(MeshBlockData *, MeshBlockData *); diff --git a/src/radiation/moments_source.cpp b/src/radiation/moments_source.cpp index 5d127b68..5e641ed6 100644 --- a/src/radiation/moments_source.cpp +++ b/src/radiation/moments_source.cpp @@ -981,7 +981,7 @@ TaskStatus MomentFluidSource(T *rc, Real dt, bool update_fluid) { } else { PARTHENON_FAIL("Radiation method unknown!"); } - return TaskStatus::fail; + return TaskStatus::complete; } template TaskStatus MomentFluidSource>(MeshBlockData *, Real, bool); From 54e96a51e60d926e6e698aebcf6a4c1d9cff8394 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Tue, 6 Feb 2024 17:46:25 -0500 Subject: [PATCH 2/4] pinning new version of parthenon --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 0072037f..f7ea1d01 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 0072037fcadb208059152ce1502d1edc1f3c9ba0 +Subproject commit f7ea1d01fe7282a1e08f3978a73c9acf6654209e From 051e8c92de2887f7d7a15d7741f373f4a7e12050 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 23 Feb 2024 19:28:13 -0500 Subject: [PATCH 3/4] combined taskqualifiers for the reducers --- src/phoebus_driver.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 8d5a3594..ea661ef9 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -369,7 +369,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { : none); // Test the reduction until it completes TaskID finish_reduce_1 = - tl.AddTask(TaskQualifier::local_sync, start_reduce_1, fixup::NetFieldCheckReduce, + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_1, fixup::NetFieldCheckReduce, md.get(), t, stage, net_field_totals); auto mod_net = tl.AddTask(finish_reduce_1, fixup::ModifyNetField, md.get(), t, @@ -385,7 +385,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { : none); // Test the reduction until it completes TaskID finish_reduce_2 = - tl.AddTask(TaskQualifier::local_sync, start_reduce_2, fixup::NetFieldCheckReduce, + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_2, fixup::NetFieldCheckReduce, md.get(), t, stage, net_field_totals_2); // Remove artificial contribution to net field @@ -454,9 +454,9 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { MPI_SUM) : none); auto finish_reduce_matter = - tl.AddTask(TaskQualifier::local_sync, start_reduce_matter, + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_matter, &MonoMatRed_t::CheckReduce, pmono_mat_red); - auto finish_reduce_vols = tl.AddTask(TaskQualifier::local_sync, start_reduce_vols, + auto finish_reduce_vols = tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_vols, &MonoVolRed_t::CheckReduce, pmono_vol_red); auto finish_mono_reds = finish_reduce_matter | finish_reduce_vols; auto divide_vols = @@ -616,7 +616,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { particle_resolution, MPI_SUM); auto finish_count_reduce = - tl.AddTask(TaskQualifier::local_sync, start_count_reduce, + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_count_reduce, &AllReduce>::CheckReduce, particle_resolution); // Report particle count @@ -704,7 +704,7 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer, MPI_LOR) : none); finish_gain_reducer = - tl.AddTask(TaskQualifier::local_sync, start_gain_reducer, + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_gain_reducer, &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); } auto calculate_four_force = @@ -931,7 +931,7 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { particle_resolution, MPI_SUM); auto finish_resolution_reduce = - tl.AddTask(TaskQualifier::local_sync, start_resolution_reduce, + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_resolution_reduce, &AllReduce>::CheckReduce, particle_resolution); // Report tuning From 4dc67969bb1d5d9db73cc4d7cdf43df7c6dd0010 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 23 Feb 2024 19:42:41 -0500 Subject: [PATCH 4/4] formatting --- src/phoebus_driver.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index ea661ef9..82abce29 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -368,9 +368,9 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { md.get(), t, stage, net_field_totals) : none); // Test the reduction until it completes - TaskID finish_reduce_1 = - tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_1, fixup::NetFieldCheckReduce, - md.get(), t, stage, net_field_totals); + TaskID finish_reduce_1 = tl.AddTask( + TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_1, + fixup::NetFieldCheckReduce, md.get(), t, stage, net_field_totals); auto mod_net = tl.AddTask(finish_reduce_1, fixup::ModifyNetField, md.get(), t, beta * dt, stage, true, 1.); @@ -384,9 +384,9 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { md.get(), t, stage, net_field_totals_2) : none); // Test the reduction until it completes - TaskID finish_reduce_2 = - tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_2, fixup::NetFieldCheckReduce, - md.get(), t, stage, net_field_totals_2); + TaskID finish_reduce_2 = tl.AddTask( + TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_2, + fixup::NetFieldCheckReduce, md.get(), t, stage, net_field_totals_2); // Remove artificial contribution to net field auto mod_net_2 = tl.AddTask(finish_reduce_2, fixup::ModifyNetField, md.get(), t, @@ -454,10 +454,11 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { MPI_SUM) : none); auto finish_reduce_matter = - tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_matter, - &MonoMatRed_t::CheckReduce, pmono_mat_red); - auto finish_reduce_vols = tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_vols, - &MonoVolRed_t::CheckReduce, pmono_vol_red); + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, + start_reduce_matter, &MonoMatRed_t::CheckReduce, pmono_mat_red); + auto finish_reduce_vols = + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, + start_reduce_vols, &MonoVolRed_t::CheckReduce, pmono_vol_red); auto finish_mono_reds = finish_reduce_matter | finish_reduce_vols; auto divide_vols = (ib == 0 ? tl.AddTask(finish_mono_reds, MonopoleGR::DivideVols, monopole.get()) @@ -615,9 +616,9 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { tl.AddTask(update_count, &AllReduce>::StartReduce, particle_resolution, MPI_SUM); - auto finish_count_reduce = - tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_count_reduce, - &AllReduce>::CheckReduce, particle_resolution); + auto finish_count_reduce = tl.AddTask( + TaskQualifier::local_sync | TaskQualifier::once_per_region, start_count_reduce, + &AllReduce>::CheckReduce, particle_resolution); // Report particle count auto report_count = @@ -704,8 +705,9 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer, MPI_LOR) : none); finish_gain_reducer = - tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_gain_reducer, - &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, + start_gain_reducer, &parthenon::AllReduce::CheckReduce, + pdo_gain_reducer); } auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, @@ -931,8 +933,9 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { particle_resolution, MPI_SUM); auto finish_resolution_reduce = - tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, start_resolution_reduce, - &AllReduce>::CheckReduce, particle_resolution); + tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region, + start_resolution_reduce, &AllReduce>::CheckReduce, + particle_resolution); // Report tuning auto report_resolution =