From f968cf17895dbb27af152089a4ba040d65cc7646 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Tue, 14 May 2024 10:39:24 -0600 Subject: [PATCH 1/5] First cut --- src/bvals/comms/boundary_communication.cpp | 69 ++++++++++++++++++++++ src/bvals/comms/bvals_in_one.hpp | 14 +++++ 2 files changed, 83 insertions(+) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index aa7a5c9675b4..50e160b2233d 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -384,6 +384,21 @@ ProlongateBounds(std::shared_ptr> &); template TaskStatus ProlongateBounds(std::shared_ptr> &); +TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl, + std::shared_ptr> &md) { + auto send = tl.AddTask(dependency, SendSwarmBoundaryBuffers, md); + auto recv = tl.AddTask(dependency, ReceiveSwarmBoundaryBuffers, md); + return recv; +} + +TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl, + std::shared_ptr> &md, + std::vector swarm_names) { + auto send = tl.AddTask(dependency, SendSwarmBoundaryBuffers, md, swarm_names); + auto recv = tl.AddTask(dependency, ReceiveSwarmBoundaryBuffers, md, swarm_names); + return recv; +} + // Adds all relevant boundary communication to a single task list template TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, @@ -441,4 +456,58 @@ TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, auto receive = tl.AddTask(dependency, ReceiveBoundBufs, md); return tl.AddTask(receive, SetBounds, md); } + +TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr> &md) { + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &sc = md->GetBlockData(n)->GetSwarmData(); + sc->ResetCommunication(); + // TODO(BRR) remove this BoundaryCommSubset argument? + sc->Send(BoundaryCommSubset::all); + } + + return TaskStatus::complete; +} + +TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr> &md, + std::vector swarm_names) { + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &sc = md->GetBlockData(n)->GetSwarmData(); + sc->ResetCommunication(swarm_names); + sc->Send(BoundaryCommSubset::all, swarm_names); + } + + return TaskStatus::complete; +} + +TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr> &md) { + TaskStatus status = TaskStatus::complete; + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &sc = md->GetBlockData(n)->GetSwarmData(); + auto local_status = sc->Receive(BoundaryCommSubset::all); + if (local_status == TaskStatus::incomplete) { + status = TaskStatus::incomplete; + } + } + + return status; +} + +TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr> &md, + std::vector swarm_names) { + TaskStatus status = TaskStatus::complete; + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &sc = md->GetBlockData(n)->GetSwarmData(); + auto local_status = sc->Receive(BoundaryCommSubset::all, swarm_names); + if (local_status == TaskStatus::incomplete) { + status = TaskStatus::incomplete; + } + } + + return status; +} + } // namespace parthenon diff --git a/src/bvals/comms/bvals_in_one.hpp b/src/bvals/comms/bvals_in_one.hpp index 0180bd6a8305..f831db284ed1 100644 --- a/src/bvals/comms/bvals_in_one.hpp +++ b/src/bvals/comms/bvals_in_one.hpp @@ -79,6 +79,13 @@ static TaskStatus SetFluxCorrections(std::shared_ptr> &md) { return SetBounds(md); } +TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr> &md); +TaskStatus SendSwarmBoundaryBuffers(std::shared_ptr> &md, + std::vector swarm_names); +TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr> &md); +TaskStatus ReceiveSwarmBoundaryBuffers(std::shared_ptr> &md, + std::vector swarm_names); + // Adds all relevant boundary communication to a single task list template TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, @@ -88,6 +95,13 @@ TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel); +// Adds all relevant swarm boundary communication to a single task list +TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl, + std::shared_ptr> &md); +TaskID AddSwarmBoundaryExchangeTasks(TaskID dependency, TaskList &tl, + std::shared_ptr> &md, + std::vector swarm_names); + // These tasks should not be called in down stream code TaskStatus BuildBoundaryBuffers(std::shared_ptr> &md); TaskStatus BuildGMGBoundaryBuffers(std::shared_ptr> &md); From 84d54955d62dc8af345536802e2f0280e4e57efd Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Tue, 14 May 2024 10:59:09 -0600 Subject: [PATCH 2/5] Attempt at fix for multiswarm BC issue --- src/bvals/bvals.cpp | 5 +++-- src/interface/swarm_comms.cpp | 4 ++-- src/interface/swarm_container.cpp | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/bvals/bvals.cpp b/src/bvals/bvals.cpp index 3581b6c0732f..bef53380dc05 100644 --- a/src/bvals/bvals.cpp +++ b/src/bvals/bvals.cpp @@ -43,8 +43,9 @@ namespace parthenon { -BoundarySwarm::BoundarySwarm(std::weak_ptr pmb, const std::string &label) - : bswarm_index(), pmy_block(pmb), pmy_mesh_(pmb.lock()->pmy_mesh) { +BoundarySwarm::BoundarySwarm(std::weak_ptr pmb, const std::string &label, + const int swarm_idx) + : bswarm_index(swarm_idx), pmy_block(pmb), pmy_mesh_(pmb.lock()->pmy_mesh) { #ifdef MPI_PARALLEL swarm_comm = pmy_mesh_->GetMPIComm(label); #endif diff --git a/src/interface/swarm_comms.cpp b/src/interface/swarm_comms.cpp index 67bff0a69565..89fff9c7277e 100644 --- a/src/interface/swarm_comms.cpp +++ b/src/interface/swarm_comms.cpp @@ -660,13 +660,13 @@ bool Swarm::FinalizeCommunicationIterative() { return true; } -void Swarm::AllocateComms(std::weak_ptr wpmb) { +void Swarm::AllocateComms(std::weak_ptr wpmb, const int swarm_idx) { if (wpmb.expired()) return; std::shared_ptr pmb = wpmb.lock(); // Create the boundary object - vbswarm = std::make_shared(pmb, label_); + vbswarm = std::make_shared(pmb, label_, swarm_idx); // Enroll SwarmVariable object vbswarm->bswarm_index = pmb->pbswarm->bswarms.size(); diff --git a/src/interface/swarm_container.cpp b/src/interface/swarm_container.cpp index ca5874125d14..d9f81cc9805b 100644 --- a/src/interface/swarm_container.cpp +++ b/src/interface/swarm_container.cpp @@ -86,7 +86,8 @@ void SwarmContainer::Add(const std::string &label, const Metadata &metadata) { auto swarm = std::make_shared(label, metadata); swarm->SetBlockPointer(GetBlockPointer()); - swarm->AllocateComms(GetBlockPointer()); + const int swarm_idx = swarmVector_.size() - 1; + swarm->AllocateComms(GetBlockPointer(), swarm_idx); Add(swarm); } From 09a62fa30900cd5bbc226f3ef307b7e98ffcf471 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Thu, 16 May 2024 10:04:47 -0600 Subject: [PATCH 3/5] Trying to split off swarm tags from field tags --- src/bvals/bvals.cpp | 4 ++-- src/bvals/comms/tag_map.cpp | 22 +++++++++++++++++++++- src/bvals/comms/tag_map.hpp | 2 ++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/bvals/bvals.cpp b/src/bvals/bvals.cpp index bef53380dc05..0cac7c5de03a 100644 --- a/src/bvals/bvals.cpp +++ b/src/bvals/bvals.cpp @@ -75,8 +75,8 @@ void BoundarySwarm::SetupPersistentMPI() { NeighborBlock &nb = pmb->neighbors[n]; // Neighbor on different MPI process if (nb.rank != Globals::my_rank) { - send_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetTag(pmb.get(), nb); - recv_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetTag(pmb.get(), nb); + send_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetSwarmTag(pmb.get(), nb); + recv_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetSwarmTag(pmb.get(), nb); if (bd_var_.req_send[nb.bufid] != MPI_REQUEST_NULL) { MPI_Request_free(&bd_var_.req_send[nb.bufid]); } diff --git a/src/bvals/comms/tag_map.cpp b/src/bvals/comms/tag_map.cpp index e03c8a7df70b..268e715c6547 100644 --- a/src/bvals/comms/tag_map.cpp +++ b/src/bvals/comms/tag_map.cpp @@ -63,6 +63,10 @@ void TagMap::AddMeshDataToMap(std::shared_ptr> &md) { auto &pair_map = map_[other_rank]; // Add channel key with an invalid tag pair_map[MakeChannelPair(pmb, nb)] = -1; + if (swarm_map_.count(other_rank) < 1) swarm_map_[other_rank] = rank_pair_map_t(); + auto &swarm_pair_map = swarm_map_[other_rank]; + // Add channel key with an invalid tag + swarm_pair_map[MakeChannelPair(pmb, nb)] = -1; } } } @@ -88,15 +92,22 @@ void TagMap::ResolveMap() { PARTHENON_FAIL("MPI error, cannot query largest supported MPI tag value."); } #endif - for (auto it = map_.begin(); it != map_.end(); ++it) { + auto it = map_.begin(); + auto swarm_it = swarm_map_.begin(); + while (it != map_.end() && swarm_it != swarm_map_.end()) { auto &pair_map = it->second; int idx = 0; std::for_each(pair_map.begin(), pair_map.end(), [&idx](auto &pair) { pair.second = idx++; }); + auto &swarm_pair_map = swarm_it->second; + std::for_each(swarm_pair_map.begin(), swarm_pair_map.end(), + [&idx](auto &swarm_pair) { swarm_pair.second = idx++; }); #ifdef MPI_PARALLEL if (idx > (*reinterpret_cast(max_tag)) && it->first != Globals::my_rank) PARTHENON_FAIL("Number of tags exceeds the maximum allowed by this MPI version."); #endif + ++it; + ++swarm_it; } } @@ -109,4 +120,13 @@ int TagMap::GetTag(const MeshBlock *pmb, const NeighborBlock &nb) { return pair_map[cpair]; } +int TagMap::GetSwarmTag(const MeshBlock *pmb, const NeighborBlock &nb) { + const int other_rank = nb.rank; + auto &swarm_pair_map = swarm_map_[other_rank]; + auto cpair = MakeChannelPair(pmb, nb); + PARTHENON_REQUIRE(swarm_pair_map.count(cpair) == 1, + "Trying to get tag for key that hasn't been entered.\n"); + return swarm_pair_map[cpair]; +} + } // namespace parthenon diff --git a/src/bvals/comms/tag_map.hpp b/src/bvals/comms/tag_map.hpp index d7552d1d69ad..49a6790c5353 100644 --- a/src/bvals/comms/tag_map.hpp +++ b/src/bvals/comms/tag_map.hpp @@ -85,6 +85,7 @@ class TagMap { using tag_map_t = std::unordered_map; tag_map_t map_; + tag_map_t swarm_map_; // Given the two blocks (one described by the MeshBlock and the other described by the // firsts NeighborBlock information) return an ordered pair of BlockGeometricElementIds @@ -108,6 +109,7 @@ class TagMap { // After the map has been resolved, get the tag for a particular MeshBlock NeighborBlock // pair int GetTag(const MeshBlock *pmb, const NeighborBlock &nb); + int GetSwarmTag(const MeshBlock *pmb, const NeighborBlock &nb); }; } // namespace parthenon From 8bcf9d118d2c6b01ed9f8d4ba79500a7807ba44e Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Thu, 16 May 2024 10:08:08 -0600 Subject: [PATCH 4/5] Undo problem From 33d92a47bd181035bd9dcf58ca4cf3cfa6817ab0 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Thu, 16 May 2024 10:09:11 -0600 Subject: [PATCH 5/5] Revert "Trying to split off swarm tags from field tags" This reverts commit 09a62fa30900cd5bbc226f3ef307b7e98ffcf471. --- src/bvals/bvals.cpp | 4 ++-- src/bvals/comms/tag_map.cpp | 22 +--------------------- src/bvals/comms/tag_map.hpp | 2 -- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/src/bvals/bvals.cpp b/src/bvals/bvals.cpp index 0cac7c5de03a..bef53380dc05 100644 --- a/src/bvals/bvals.cpp +++ b/src/bvals/bvals.cpp @@ -75,8 +75,8 @@ void BoundarySwarm::SetupPersistentMPI() { NeighborBlock &nb = pmb->neighbors[n]; // Neighbor on different MPI process if (nb.rank != Globals::my_rank) { - send_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetSwarmTag(pmb.get(), nb); - recv_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetSwarmTag(pmb.get(), nb); + send_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetTag(pmb.get(), nb); + recv_tag[nb.bufid] = pmb->pmy_mesh->tag_map.GetTag(pmb.get(), nb); if (bd_var_.req_send[nb.bufid] != MPI_REQUEST_NULL) { MPI_Request_free(&bd_var_.req_send[nb.bufid]); } diff --git a/src/bvals/comms/tag_map.cpp b/src/bvals/comms/tag_map.cpp index 268e715c6547..e03c8a7df70b 100644 --- a/src/bvals/comms/tag_map.cpp +++ b/src/bvals/comms/tag_map.cpp @@ -63,10 +63,6 @@ void TagMap::AddMeshDataToMap(std::shared_ptr> &md) { auto &pair_map = map_[other_rank]; // Add channel key with an invalid tag pair_map[MakeChannelPair(pmb, nb)] = -1; - if (swarm_map_.count(other_rank) < 1) swarm_map_[other_rank] = rank_pair_map_t(); - auto &swarm_pair_map = swarm_map_[other_rank]; - // Add channel key with an invalid tag - swarm_pair_map[MakeChannelPair(pmb, nb)] = -1; } } } @@ -92,22 +88,15 @@ void TagMap::ResolveMap() { PARTHENON_FAIL("MPI error, cannot query largest supported MPI tag value."); } #endif - auto it = map_.begin(); - auto swarm_it = swarm_map_.begin(); - while (it != map_.end() && swarm_it != swarm_map_.end()) { + for (auto it = map_.begin(); it != map_.end(); ++it) { auto &pair_map = it->second; int idx = 0; std::for_each(pair_map.begin(), pair_map.end(), [&idx](auto &pair) { pair.second = idx++; }); - auto &swarm_pair_map = swarm_it->second; - std::for_each(swarm_pair_map.begin(), swarm_pair_map.end(), - [&idx](auto &swarm_pair) { swarm_pair.second = idx++; }); #ifdef MPI_PARALLEL if (idx > (*reinterpret_cast(max_tag)) && it->first != Globals::my_rank) PARTHENON_FAIL("Number of tags exceeds the maximum allowed by this MPI version."); #endif - ++it; - ++swarm_it; } } @@ -120,13 +109,4 @@ int TagMap::GetTag(const MeshBlock *pmb, const NeighborBlock &nb) { return pair_map[cpair]; } -int TagMap::GetSwarmTag(const MeshBlock *pmb, const NeighborBlock &nb) { - const int other_rank = nb.rank; - auto &swarm_pair_map = swarm_map_[other_rank]; - auto cpair = MakeChannelPair(pmb, nb); - PARTHENON_REQUIRE(swarm_pair_map.count(cpair) == 1, - "Trying to get tag for key that hasn't been entered.\n"); - return swarm_pair_map[cpair]; -} - } // namespace parthenon diff --git a/src/bvals/comms/tag_map.hpp b/src/bvals/comms/tag_map.hpp index 49a6790c5353..d7552d1d69ad 100644 --- a/src/bvals/comms/tag_map.hpp +++ b/src/bvals/comms/tag_map.hpp @@ -85,7 +85,6 @@ class TagMap { using tag_map_t = std::unordered_map; tag_map_t map_; - tag_map_t swarm_map_; // Given the two blocks (one described by the MeshBlock and the other described by the // firsts NeighborBlock information) return an ordered pair of BlockGeometricElementIds @@ -109,7 +108,6 @@ class TagMap { // After the map has been resolved, get the tag for a particular MeshBlock NeighborBlock // pair int GetTag(const MeshBlock *pmb, const NeighborBlock &nb); - int GetSwarmTag(const MeshBlock *pmb, const NeighborBlock &nb); }; } // namespace parthenon