From 221f51a4e3c4f1cf8b9201d30028e80be1bbd26e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Del=20Pino?= <stephane.delpino44@gmail.com>
Date: Wed, 5 Feb 2025 19:16:25 +0100
Subject: [PATCH] Continue debugging for connectivity dispatcher for load
balancing
[ci-skip]
---
src/mesh/ConnectivityDispatcher.cpp | 84 ++++++++++++++++++++++++-----
1 file changed, 72 insertions(+), 12 deletions(-)
diff --git a/src/mesh/ConnectivityDispatcher.cpp b/src/mesh/ConnectivityDispatcher.cpp
index 23ec824c4..1ce0c9154 100644
--- a/src/mesh/ConnectivityDispatcher.cpp
+++ b/src/mesh/ConnectivityDispatcher.cpp
@@ -11,6 +11,8 @@ template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildNewOwner()
{
+ std::cout << "#### " << rang::fgB::yellow << itemName(item_type) << rang::fg::reset << " _buildNewOwner() ####\n";
+
if constexpr (item_type == ItemType::cell) {
CRSGraph connectivity_graph = m_connectivity.ownCellToCellGraph();
Partitioner P;
@@ -33,7 +35,7 @@ ConnectivityDispatcher<Dimension>::_buildNewOwner()
synchronize(cell_new_owner);
std::cout << "cell_new_owner = " << cell_new_owner << '\n';
- std::cout << "cell_number = " << m_connectivity.cellNumber() << '\n';
+ std::cout << "cell_number = " << m_connectivity.template number<item_type>() << '\n';
this->_dispatchedInfo<ItemType::cell>().m_new_owner = cell_new_owner;
} else {
@@ -59,6 +61,10 @@ ConnectivityDispatcher<Dimension>::_buildNewOwner()
});
synchronize(item_new_owner);
+
+ std::cout << itemName(item_type) << "_new_owner = " << item_new_owner << '\n';
+ std::cout << itemName(item_type) << "_number = " << m_connectivity.template number<item_type>() << '\n';
+
this->_dispatchedInfo<item_type>().m_new_owner = item_new_owner;
}
}
@@ -87,27 +93,36 @@ ConnectivityDispatcher<Dimension>::_buildItemListToSend()
const auto& cell_is_owned = m_connectivity.cellIsOwned();
const auto& cell_new_owner = this->_dispatchedInfo<ItemType::cell>().m_new_owner;
+ const auto& cell_number = m_connectivity.cellNumber();
+
+ std::cout << "- cell_new_owner = " << cell_new_owner << '\n';
std::vector<std::vector<CellId>> cell_vector_to_send_by_proc(parallel::size());
Array<bool> send_to_rank(parallel::size());
+
for (CellId cell_id = 0; cell_id < m_connectivity.numberOfCells(); ++cell_id) {
send_to_rank.fill(false);
const auto& cell_to_node = cell_to_node_matrix[cell_id];
- for (size_t i_node = 0; i_node < cell_to_node.size(); ++i_node) {
- const NodeId& node_id = cell_to_node[i_node];
- const auto& node_to_cell = node_to_cell_matrix[node_id];
- for (size_t i_node_cell = 0; i_node_cell < node_to_cell.size(); ++i_node_cell) {
- const CellId& node_cell = node_to_cell[i_node_cell];
- if (cell_is_owned[node_cell]) {
+ std::cout << rang::fgB::cyan << " looking for cell " << cell_id << "(" << cell_number[cell_id] << ")"
+ << rang::fg::reset << "\n";
+
+ if (cell_is_owned[cell_id]) {
+ for (size_t i_node = 0; i_node < cell_to_node.size(); ++i_node) {
+ const NodeId& node_id = cell_to_node[i_node];
+ const auto& node_to_cell = node_to_cell_matrix[node_id];
+ for (size_t i_node_cell = 0; i_node_cell < node_to_cell.size(); ++i_node_cell) {
+ const CellId& node_cell = node_to_cell[i_node_cell];
send_to_rank[cell_new_owner[node_cell]] = true;
+ std::cout << rang::fgB::green << " storing cell " << node_cell << "(" << cell_number[node_cell] << ")"
+ << rang::fg::reset << "\n";
}
}
- }
- for (size_t k = 0; k < send_to_rank.size(); ++k) {
- if (send_to_rank[k]) {
- cell_vector_to_send_by_proc[k].push_back(cell_id);
+ for (size_t i_rank = 0; i_rank < send_to_rank.size(); ++i_rank) {
+ if (send_to_rank[i_rank]) {
+ cell_vector_to_send_by_proc[i_rank].push_back(cell_id);
+ }
}
}
}
@@ -119,7 +134,7 @@ ConnectivityDispatcher<Dimension>::_buildItemListToSend()
}
for (size_t i = 0; i < parallel::size(); ++i) {
- std::cout << parallel::rank() << ": send to " << i << ": " << cell_list_to_send_by_proc[i] << '\n';
+ std::cout << itemName(item_type) << " send to " << i << ": " << cell_list_to_send_by_proc[i] << '\n';
}
} else {
const auto& cell_list_to_send_by_proc = this->_dispatchedInfo<ItemType::cell>().m_list_to_send_by_proc;
@@ -149,6 +164,10 @@ ConnectivityDispatcher<Dimension>::_buildItemListToSend()
}
item_list_to_send_by_proc[i_rank] = convert_to_array(item_id_vector);
}
+
+ for (size_t i = 0; i < parallel::size(); ++i) {
+ std::cout << itemName(item_type) << " send to " << i << ": " << item_list_to_send_by_proc[i] << '\n';
+ }
}
}
@@ -164,7 +183,14 @@ ConnectivityDispatcher<Dimension>::_buildNumberOfItemToExchange()
}
this->_dispatchedInfo<item_type>().m_list_to_send_size_by_proc = nb_item_to_send_by_proc;
+ std::cout << rang::fgB::yellow << itemName(item_type) << " nb_item_to_send_by_proc = " << nb_item_to_send_by_proc
+ << rang::fg::reset << '\n';
+
this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc = parallel::allToAll(nb_item_to_send_by_proc);
+
+ std::cout << rang::fgB::blue << itemName(item_type)
+ << " m_list_to_recv_size_by_proc = " << this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc
+ << rang::fg::reset << '\n';
}
template <size_t Dimension>
@@ -718,6 +744,40 @@ ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType
}
}
+ std::cout << "--- initial mesh ---\n";
+ std::cout << "--- cells\n";
+ for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
+ std::cout << "cell " << cell_id << "(" << connectivity.cellNumber()[cell_id] << "):" << rang::style::bold
+ << ((connectivity.cellIsOwned()[cell_id]) ? 'O' : 'G') << rang::style::reset << ": ";
+ auto node_list = connectivity.cellToNodeMatrix()[cell_id];
+ for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
+ const NodeId node_id = node_list[i_node];
+ std::cout << ' ' << node_id << "(" << connectivity.nodeNumber()[node_id] << "):" << rang::style::bold
+ << ((connectivity.nodeIsOwned()[node_id]) ? 'O' : 'G') << rang::style::reset;
+ }
+ std::cout << '\n';
+ }
+ if constexpr (Dimension > 1) {
+ std::cout << "--- faces\n";
+ for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
+ std::cout << "cell " << face_id << "(" << connectivity.faceNumber()[face_id] << "):" << rang::style::bold
+ << ((connectivity.faceIsOwned()[face_id]) ? 'O' : 'G') << rang::style::reset << ": ";
+ auto node_list = connectivity.faceToNodeMatrix()[face_id];
+ for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
+ const NodeId node_id = node_list[i_node];
+ std::cout << ' ' << node_id << "(" << connectivity.nodeNumber()[node_id] << "):" << rang::style::bold
+ << ((connectivity.nodeIsOwned()[node_id]) ? 'O' : 'G') << rang::style::reset;
+ }
+ std::cout << '\n';
+ }
+ }
+ std::cout << "--- nodes\n";
+ for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
+ std::cout << "node " << node_id << "(" << connectivity.nodeNumber()[node_id] << "):" << rang::style::bold
+ << ((connectivity.nodeIsOwned()[node_id]) ? 'O' : 'G') << rang::style::reset << ": ";
+ std::cout << '\n';
+ }
+
this->_buildNewOwner<ItemType::cell>();
if constexpr (Dimension > 1) {
this->_buildNewOwner<ItemType::face>();
--
GitLab