Skip to content

Commit

Permalink
Merge pull request #198 from lanl/mg/fix_TaskStatus_fail
Browse files Browse the repository at this point in the history 
updated phoebus based on recent tasking rewrite changes in parthenon (PR #987 in parthenon)
  • Loading branch information
@Yurlungur
Yurlungur authored Mar 1, 2024
2 parents 4e3d2c7 + 4dc6796 commit d806b3e
Show file tree
Hide file tree
Showing 8 changed files with 47 additions and 58 deletions.
2 changes: 1 addition & 1 deletion external/parthenon
Submodule parthenon updated from 007203 to f7ea1d
3 changes: 2 additions & 1 deletion src/fixup/fixup.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "radiation/closure_m1.hpp"
#include "radiation/closure_mocmc.hpp"
#include "radiation/radiation.hpp"
#include <parthenon/package.hpp>

using Microphysics::RadiationType;
using Microphysics::EOS::EOS;
Expand Down Expand Up @@ -700,7 +701,7 @@ TaskStatus ApplyFloors(T *rc) {
"Rad floors not supported with cooling function/Monte Carlo!");
return ApplyFloorsImpl<T, radiation::ClosureEdd<settings>>(rc);
}
return TaskStatus::fail;
return TaskStatus::complete;
}

template TaskStatus ApplyFloors<MeshBlockData<Real>>(MeshBlockData<Real> *rc);
Expand Down
3 changes: 2 additions & 1 deletion src/fixup/fixup_c2p.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "radiation/closure_m1.hpp"
#include "radiation/closure_mocmc.hpp"
#include "radiation/radiation.hpp"
#include <parthenon/package.hpp>

using radiation::ClosureEquation;
using radiation::ClosureSettings;
Expand Down Expand Up @@ -377,7 +378,7 @@ TaskStatus ConservedToPrimitiveFixup(T *rc) {
} else {
return ConservedToPrimitiveFixupImpl<T, radiation::ClosureEdd<settings>>(rc);
}
return TaskStatus::fail;
return TaskStatus::complete;
}

template TaskStatus
Expand Down
3 changes: 2 additions & 1 deletion src/fixup/fixup_radc2p.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "radiation/closure_m1.hpp"
#include "radiation/closure_mocmc.hpp"
#include "radiation/radiation.hpp"
#include <parthenon/package.hpp>

using Microphysics::RadiationType;
using radiation::ClosureEquation;
Expand Down Expand Up @@ -246,7 +247,7 @@ TaskStatus RadConservedToPrimitiveFixup(T *rc) {
} else if (method == "mocmc") {
return RadConservedToPrimitiveFixupImpl<T, radiation::ClosureMOCMC<settings>>(rc);
}
return TaskStatus::fail;
return TaskStatus::complete;
}

template TaskStatus
Expand Down
3 changes: 2 additions & 1 deletion src/fixup/fixup_src.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "radiation/closure_m1.hpp"
#include "radiation/closure_mocmc.hpp"
#include "radiation/radiation.hpp"
#include <parthenon/package.hpp>

using Microphysics::RadiationType;
using Microphysics::EOS::EOS;
Expand Down Expand Up @@ -346,7 +347,7 @@ TaskStatus SourceFixup(T *rc) {
} else if (method == "mocmc") {
return SourceFixupImpl<T, radiation::ClosureMOCMC<settings>>(rc);
}
return TaskStatus::fail;
return TaskStatus::complete;
}

template TaskStatus SourceFixup<MeshBlockData<Real>>(MeshBlockData<Real> *rc);
Expand Down
81 changes: 33 additions & 48 deletions src/phoebus_driver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,38 +363,33 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) {
// TODO(BRR) Setting scale factor of B field need only be done once per simulation

// Evaluate current Mdot, Phi
auto sum_mdot_1 = tl.AddTask(none, fixup::SumMdotPhiForNetFieldScaling, md.get(), t,
stage, &(net_field_totals->val));

sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, sum_mdot_1);
sync_region_5_dep_id++;
auto sum_mdot_1 =
tl.AddTask(TaskQualifier::local_sync, none, fixup::SumMdotPhiForNetFieldScaling,
md.get(), t, stage, &(net_field_totals->val));

TaskID start_reduce_1 = (i == 0 ? tl.AddTask(sum_mdot_1, fixup::NetFieldStartReduce,
md.get(), t, stage, net_field_totals)
: none);
// Test the reduction until it completes
TaskID finish_reduce_1 = tl.AddTask(start_reduce_1, fixup::NetFieldCheckReduce,
md.get(), t, stage, net_field_totals);
sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, finish_reduce_1);
sync_region_5_dep_id++;
TaskID finish_reduce_1 = tl.AddTask(
TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_1,
fixup::NetFieldCheckReduce, md.get(), t, stage, net_field_totals);

auto mod_net = tl.AddTask(finish_reduce_1, fixup::ModifyNetField, md.get(), t,
beta * dt, stage, true, 1.);

// Evaluate Mdot, Phi (only Phi changes) after modifying B field
auto sum_mdot_2 = tl.AddTask(mod_net, fixup::SumMdotPhiForNetFieldScaling, md.get(),
t, stage, &(net_field_totals_2->val));
sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, sum_mdot_2);
sync_region_5_dep_id++;
auto sum_mdot_2 = tl.AddTask(TaskQualifier::local_sync, mod_net,
fixup::SumMdotPhiForNetFieldScaling, md.get(), t, stage,
&(net_field_totals_2->val));

TaskID start_reduce_2 = (i == 0 ? tl.AddTask(sum_mdot_2, fixup::NetFieldStartReduce,
md.get(), t, stage, net_field_totals_2)
: none);
// Test the reduction until it completes
TaskID finish_reduce_2 = tl.AddTask(start_reduce_2, fixup::NetFieldCheckReduce,
md.get(), t, stage, net_field_totals_2);
sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, finish_reduce_2);
sync_region_5_dep_id++;
TaskID finish_reduce_2 = tl.AddTask(
TaskQualifier::local_sync | TaskQualifier::once_per_region, start_reduce_2,
fixup::NetFieldCheckReduce, md.get(), t, stage, net_field_totals_2);

// Remove artificial contribution to net field
auto mod_net_2 = tl.AddTask(finish_reduce_2, fixup::ModifyNetField, md.get(), t,
Expand Down Expand Up @@ -444,16 +439,15 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) {
TaskID interp_to_monopole = none;
if (monopole_gr_active) {
auto interp_to_monopole =
tl.AddTask(none, MonopoleGR::InterpolateMatterTo1D<MDT>, sc0.get());
sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, interp_to_monopole);
tl.AddTask(TaskQualifier::local_sync, none,
MonopoleGR::InterpolateMatterTo1D<MDT>, sc0.get());

// TODO(JMM): Is this the right place for this?
// TODO(JMM): Should this stuff be in the synchronous region?
auto matter_to_host =
(ib == 0 ? tl.AddTask(interp_to_monopole, MonopoleGR::MatterToHost,
monopole.get(), true)
(ib == 0 ? tl.AddTask(TaskQualifier::local_sync, interp_to_monopole,
MonopoleGR::MatterToHost, monopole.get(), true)
: none);
sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, matter_to_host);
auto start_reduce_matter =
(ib == 0 ? tl.AddTask(matter_to_host, &MonoMatRed_t::StartReduce, pmono_mat_red,
MPI_SUM)
Expand All @@ -463,11 +457,11 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) {
MPI_SUM)
: none);
auto finish_reduce_matter =
tl.AddTask(start_reduce_matter, &MonoMatRed_t::CheckReduce, pmono_mat_red);
sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, finish_reduce_matter);
tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region,
start_reduce_matter, &MonoMatRed_t::CheckReduce, pmono_mat_red);
auto finish_reduce_vols =
tl.AddTask(start_reduce_vols, &MonoVolRed_t::CheckReduce, pmono_vol_red);
sync_region_2.AddRegionalDependencies(reg_dep_id++, ib, finish_reduce_vols);
tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region,
start_reduce_vols, &MonoVolRed_t::CheckReduce, pmono_vol_red);
auto finish_mono_reds = finish_reduce_matter | finish_reduce_vols;
auto divide_vols =
(ib == 0 ? tl.AddTask(finish_mono_reds, MonopoleGR::DivideVols, monopole.get())
Expand Down Expand Up @@ -649,21 +643,17 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) {
int reg_dep_id = 0;
// auto &tl = async_region_3[0];

auto update_count = tl.AddTask(none, radiation::MOCMCUpdateParticleCount, pmesh,
&particle_resolution->val);

async_region_3.AddRegionalDependencies(reg_dep_id, 0, update_count);
reg_dep_id++;
auto update_count =
tl.AddTask(TaskQualifier::local_sync, none, radiation::MOCMCUpdateParticleCount,
pmesh, &particle_resolution->val);

auto start_count_reduce =
tl.AddTask(update_count, &AllReduce<std::vector<Real>>::StartReduce,
particle_resolution, MPI_SUM);

auto finish_count_reduce =
tl.AddTask(start_count_reduce, &AllReduce<std::vector<Real>>::CheckReduce,
particle_resolution);
async_region_3.AddRegionalDependencies(reg_dep_id, 0, finish_count_reduce);
reg_dep_id++;
auto finish_count_reduce = tl.AddTask(
TaskQualifier::local_sync | TaskQualifier::once_per_region, start_count_reduce,
&AllReduce<std::vector<Real>>::CheckReduce, particle_resolution);

// Report particle count
auto report_count =
Expand Down Expand Up @@ -750,10 +740,9 @@ TaskListStatus PhoebusDriver::RadiationPostStep() {
pdo_gain_reducer, MPI_LOR)
: none);
finish_gain_reducer =
tl.AddTask(start_gain_reducer, &parthenon::AllReduce<bool>::CheckReduce,
tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region,
start_gain_reducer, &parthenon::AllReduce<bool>::CheckReduce,
pdo_gain_reducer);
int reg_dep_id = 0;
sync_region_lb.AddRegionalDependencies(reg_dep_id++, ib, finish_gain_reducer);
}
auto calculate_four_force =
tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce,
Expand Down Expand Up @@ -971,22 +960,18 @@ TaskListStatus PhoebusDriver::MonteCarloStep() {
int reg_dep_id = 0;
auto &tl = tuning_region[0];

auto update_resolution =
tl.AddTask(none, radiation::MonteCarloUpdateParticleResolution, pmesh,
&particle_resolution->val);

tuning_region.AddRegionalDependencies(reg_dep_id, 0, update_resolution);
reg_dep_id++;
auto update_resolution = tl.AddTask(TaskQualifier::local_sync, none,
radiation::MonteCarloUpdateParticleResolution,
pmesh, &particle_resolution->val);

auto start_resolution_reduce =
tl.AddTask(update_resolution, &AllReduce<std::vector<Real>>::StartReduce,
particle_resolution, MPI_SUM);

auto finish_resolution_reduce =
tl.AddTask(start_resolution_reduce, &AllReduce<std::vector<Real>>::CheckReduce,
tl.AddTask(TaskQualifier::local_sync | TaskQualifier::once_per_region,
start_resolution_reduce, &AllReduce<std::vector<Real>>::CheckReduce,
particle_resolution);
tuning_region.AddRegionalDependencies(reg_dep_id, 0, finish_resolution_reduce);
reg_dep_id++;

// Report tuning
auto report_resolution =
Expand Down
8 changes: 4 additions & 4 deletions src/radiation/moments.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ TaskStatus MomentCon2Prim(T *rc) {
} else {
PARTHENON_FAIL("Radiation method unknown");
}
return TaskStatus::fail;
return TaskStatus::complete;
}
// template TaskStatus MomentCon2Prim<MeshData<Real>>(MeshData<Real> *);
template TaskStatus MomentCon2Prim<MeshBlockData<Real>>(MeshBlockData<Real> *);
Expand Down Expand Up @@ -287,7 +287,7 @@ TaskStatus MomentPrim2Con(T *rc, IndexDomain domain) {
} else {
PARTHENON_FAIL("Radiation method unknown!");
}
return TaskStatus::fail;
return TaskStatus::complete;
}

template TaskStatus MomentPrim2Con<MeshBlockData<Real>>(MeshBlockData<Real> *,
Expand Down Expand Up @@ -768,7 +768,7 @@ TaskStatus CalculateFluxes(T *rc) {
} else {
PARTHENON_FAIL("Radiation method unknown!");
}
return TaskStatus::fail;
return TaskStatus::complete;
}
template TaskStatus CalculateFluxes<MeshBlockData<Real>>(MeshBlockData<Real> *);

Expand Down Expand Up @@ -928,7 +928,7 @@ TaskStatus CalculateGeometricSource(T *rc, T *rc_src) {
} else {
PARTHENON_FAIL("Radiation method unknown!");
}
return TaskStatus::fail;
return TaskStatus::complete;
}
template TaskStatus CalculateGeometricSource<MeshBlockData<Real>>(MeshBlockData<Real> *,
MeshBlockData<Real> *);
Expand Down
2 changes: 1 addition & 1 deletion src/radiation/moments_source.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -981,7 +981,7 @@ TaskStatus MomentFluidSource(T *rc, Real dt, bool update_fluid) {
} else {
PARTHENON_FAIL("Radiation method unknown!");
}
return TaskStatus::fail;
return TaskStatus::complete;
}
template TaskStatus MomentFluidSource<MeshBlockData<Real>>(MeshBlockData<Real> *, Real,
bool);
Expand Down

0 comments on commit d806b3e

Please sign in to comment.