From 7f7d8381bcd9f72a82ee3e537fe17b4107643ee0 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 4 Mar 2024 21:37:04 +0000 Subject: [PATCH 1/6] feat: update to parthenon develop and pull in swarm AddEmptyParticles refactor --- external/parthenon | 2 +- src/pgen/advection.cpp | 5 ++--- src/pgen/torus.cpp | 5 ++--- src/radiation/mocmc.cpp | 5 ++--- src/radiation/monte_carlo.cpp | 11 +++++------ 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/external/parthenon b/external/parthenon index f7ea1d01..0aa104d7 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit f7ea1d01fe7282a1e08f3978a73c9acf6654209e +Subproject commit 0aa104d78abbf9112f2ea34fcdb658271f1bffce diff --git a/src/pgen/advection.cpp b/src/pgen/advection.cpp index 4570eb00..1ec4b9c9 100644 --- a/src/pgen/advection.cpp +++ b/src/pgen/advection.cpp @@ -123,8 +123,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { const auto num_tracers_total = tracer_pkg->Param("num_tracers"); const int number_block = num_tracers_total; - ParArrayND new_indices; - swarm->AddEmptyParticles(number_block, new_indices); + auto new_particle_context = swarm->AddEmptyParticles(number_block); auto &x = swarm->Get("x").Get(); auto &y = swarm->Get("y").Get(); @@ -134,7 +133,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { auto swarm_d = swarm->GetDeviceContext(); const int gid = pmb->gid; - const int max_active_index = swarm->GetMaxActiveIndex(); + const int max_active_index = new_particle_context.GetNewParticlesMaxIndex(); pmb->par_for( "ProblemGenerator::Torus::DistributeTracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { diff --git a/src/pgen/torus.cpp b/src/pgen/torus.cpp index 0ad6b910..7ef332e7 100644 --- a/src/pgen/torus.cpp +++ b/src/pgen/torus.cpp @@ -665,8 +665,7 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { const int num_tracers = std::round(num_tracers_total * number_block / number_mesh); - ParArrayND new_indices; - swarm->AddEmptyParticles(num_tracers, new_indices); + auto new_particles_context = swarm->AddEmptyParticles(num_tracers); auto &x = swarm->Get("x").Get(); auto &y = swarm->Get("y").Get(); @@ -678,7 +677,7 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { auto swarm_d = swarm->GetDeviceContext(); const int gid = pmb->gid; - const int max_active_index = swarm->GetMaxActiveIndex(); + const int max_active_index = new_particles_context.GetNewParticlesMaxIndex(); pmb->par_for( "ProblemGenerator::Torus::DistributeTracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index 4d479c7b..ad94d227 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -128,8 +128,7 @@ void MOCMCInitSamples(T *rc) { }, Kokkos::Sum(nsamp_tot)); - ParArrayND new_indices; - auto new_mask = swarm->AddEmptyParticles(nsamp_tot, new_indices); + auto new_mask = swarm->AddEmptyParticles(nsamp_tot); // Calculate array of starting index for each zone to compute particles ParArrayND starting_index("Starting index", nx_k, nx_j, nx_i); @@ -185,7 +184,7 @@ void MOCMCInitSamples(T *rc) { Geometry::Tetrads tetrads(ucon, trial, cov_g); for (int nsamp = 0; nsamp < static_cast(v(b, dn, k, j, i)); nsamp++) { - const int n = new_indices(start_idx + nsamp); + const int n = new_mask.GetNewParticleIndex(start_idx + nsamp); // Create particles at zone centers x(n) = minx_i + (i - ib.s + rng_gen.drand()) * dx_i; diff --git a/src/radiation/monte_carlo.cpp b/src/radiation/monte_carlo.cpp index 30684f90..05bc0250 100644 --- a/src/radiation/monte_carlo.cpp +++ b/src/radiation/monte_carlo.cpp @@ -282,8 +282,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const auto num_emitted = rad->Param("num_emitted"); rad->UpdateParam("num_emitted", num_emitted + Nstot); - ParArrayND new_indices; - const auto new_particles_mask = swarm->AddEmptyParticles(Nstot, new_indices); + const auto new_particles_mask = swarm->AddEmptyParticles(Nstot); auto &t = swarm->Get("t").Get(); auto &x = swarm->Get("x").Get(); @@ -338,8 +337,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, // Loop over particles to create in this zone for (int n = 0; n < dNs; n++) { - const int m = - new_indices(starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); + const int m = new_particles_mask.GetNewParticleIndex( + starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); // Set particle species swarm_species(m) = static_cast(s); @@ -403,8 +402,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, int dNs = v(iNs + sidx, k, j, i); // Loop over particles to create in this zone for (int n = 0; n < static_cast(dNs); n++) { - const int m = - new_indices(starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); + const int m = new_particles_mask.GetNewParticleIndex( + starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); swarm_d.MarkParticleForRemoval(m); } }); From 6c29659a2af2190950b76b04bfea5b5724571aae Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 4 Mar 2024 22:54:13 +0000 Subject: [PATCH 2/6] pin parthenon to swarms refactor --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 0aa104d7..67050775 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 0aa104d78abbf9112f2ea34fcdb658271f1bffce +Subproject commit 670507758409a93728b7405944115e043fb5c318 From 93812ffd511b7ae3b5bd7c717d029771fe19b2d8 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 4 Mar 2024 18:49:21 -0700 Subject: [PATCH 3/6] now move to parthenon develop --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 67050775..3225612e 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 670507758409a93728b7405944115e043fb5c318 +Subproject commit 3225612e267456e5731f80c594d45e6457274c3f From 3779d15d0e7fc2c2eb5266c120d7dab4f57beb95 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 5 Mar 2024 09:10:16 -0700 Subject: [PATCH 4/6] feat: naming consistency --- src/radiation/mocmc.cpp | 4 ++-- src/radiation/monte_carlo.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index ad94d227..509a2415 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -128,7 +128,7 @@ void MOCMCInitSamples(T *rc) { }, Kokkos::Sum(nsamp_tot)); - auto new_mask = swarm->AddEmptyParticles(nsamp_tot); + auto new_particles_context = swarm->AddEmptyParticles(nsamp_tot); // Calculate array of starting index for each zone to compute particles ParArrayND starting_index("Starting index", nx_k, nx_j, nx_i); @@ -184,7 +184,7 @@ void MOCMCInitSamples(T *rc) { Geometry::Tetrads tetrads(ucon, trial, cov_g); for (int nsamp = 0; nsamp < static_cast(v(b, dn, k, j, i)); nsamp++) { - const int n = new_mask.GetNewParticleIndex(start_idx + nsamp); + const int n = new_particles_context.GetNewParticleIndex(start_idx + nsamp); // Create particles at zone centers x(n) = minx_i + (i - ib.s + rng_gen.drand()) * dx_i; diff --git a/src/radiation/monte_carlo.cpp b/src/radiation/monte_carlo.cpp index 05bc0250..07ca8d91 100644 --- a/src/radiation/monte_carlo.cpp +++ b/src/radiation/monte_carlo.cpp @@ -282,7 +282,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const auto num_emitted = rad->Param("num_emitted"); rad->UpdateParam("num_emitted", num_emitted + Nstot); - const auto new_particles_mask = swarm->AddEmptyParticles(Nstot); + const auto new_particles_context = swarm->AddEmptyParticles(Nstot); auto &t = swarm->Get("t").Get(); auto &x = swarm->Get("x").Get(); @@ -337,7 +337,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, // Loop over particles to create in this zone for (int n = 0; n < dNs; n++) { - const int m = new_particles_mask.GetNewParticleIndex( + const int m = new_particles_context.GetNewParticleIndex( starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); // Set particle species From b004c4e1c31d697047b53ea8d95d5679a92e92e8 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 5 Mar 2024 09:11:32 -0700 Subject: [PATCH 5/6] feat: naming --- src/pgen/advection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pgen/advection.cpp b/src/pgen/advection.cpp index 1ec4b9c9..bc1eae06 100644 --- a/src/pgen/advection.cpp +++ b/src/pgen/advection.cpp @@ -123,7 +123,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { const auto num_tracers_total = tracer_pkg->Param("num_tracers"); const int number_block = num_tracers_total; - auto new_particle_context = swarm->AddEmptyParticles(number_block); + auto new_particles_context = swarm->AddEmptyParticles(number_block); auto &x = swarm->Get("x").Get(); auto &y = swarm->Get("y").Get(); @@ -133,7 +133,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { auto swarm_d = swarm->GetDeviceContext(); const int gid = pmb->gid; - const int max_active_index = new_particle_context.GetNewParticlesMaxIndex(); + const int max_active_index = new_particles_context.GetNewParticlesMaxIndex(); pmb->par_for( "ProblemGenerator::Torus::DistributeTracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { From e460ec4360e91544b6e51f884fb54dae3f71f5cc Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 5 Mar 2024 09:12:57 -0700 Subject: [PATCH 6/6] oopsies --- src/radiation/monte_carlo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/radiation/monte_carlo.cpp b/src/radiation/monte_carlo.cpp index 07ca8d91..85c66f8b 100644 --- a/src/radiation/monte_carlo.cpp +++ b/src/radiation/monte_carlo.cpp @@ -402,7 +402,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, int dNs = v(iNs + sidx, k, j, i); // Loop over particles to create in this zone for (int n = 0; n < static_cast(dNs); n++) { - const int m = new_particles_mask.GetNewParticleIndex( + const int m = new_particles_context.GetNewParticleIndex( starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); swarm_d.MarkParticleForRemoval(m); }