From b79fe22fff66964c19ae073cedb698b7156b36bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Del=20Pino?= <stephane.delpino44@gmail.com>
Date: Tue, 18 Feb 2025 19:20:01 +0100
Subject: [PATCH] Fix ref item list exchange for load balancing
---
src/mesh/ConnectivityDispatcher.cpp | 248 ++++++++++++++--------------
1 file changed, 123 insertions(+), 125 deletions(-)
diff --git a/src/mesh/ConnectivityDispatcher.cpp b/src/mesh/ConnectivityDispatcher.cpp
index 45e5ebd8a..d78d400e2 100644
--- a/src/mesh/ConnectivityDispatcher.cpp
+++ b/src/mesh/ConnectivityDispatcher.cpp
@@ -590,151 +590,149 @@ ConnectivityDispatcher<Dimension>::_buildItemReferenceList()
}();
if (number_of_item_list_sender > 0) {
- if (number_of_item_list_sender < parallel::size()) {
- const size_t sender_rank = [&]() {
- size_t i_rank = 0;
- for (; i_rank < parallel::size(); ++i_rank) {
- if (number_of_item_ref_list_per_proc[i_rank] > 0) {
- break;
- }
- }
- return i_rank;
- }();
-
- Assert(number_of_item_list_sender < parallel::size());
-
- // sending is boundary property
- Array<RefItemListBase::Type> ref_item_list_type{number_of_item_ref_list_per_proc[sender_rank]};
- if (parallel::rank() == sender_rank) {
- for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
- ++i_item_ref_list) {
- auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
- ref_item_list_type[i_item_ref_list] = item_ref_list.type();
+ const size_t sender_rank = [&]() {
+ size_t i_rank = 0;
+ for (; i_rank < parallel::size(); ++i_rank) {
+ if (number_of_item_ref_list_per_proc[i_rank] > 0) {
+ break;
}
}
- parallel::broadcast(ref_item_list_type, sender_rank);
-
- // sending references tags
- Array<RefId::TagNumberType> ref_tag_list{number_of_item_ref_list_per_proc[sender_rank]};
- if (parallel::rank() == sender_rank) {
- for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
- ++i_item_ref_list) {
- auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
- ref_tag_list[i_item_ref_list] = item_ref_list.refId().tagNumber();
- }
+ return i_rank;
+ }();
+
+ Assert(number_of_item_list_sender < parallel::size());
+
+ // sending is boundary property
+ Array<RefItemListBase::Type> ref_item_list_type{number_of_item_ref_list_per_proc[sender_rank]};
+ if (parallel::rank() == sender_rank) {
+ for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
+ ++i_item_ref_list) {
+ auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
+ ref_item_list_type[i_item_ref_list] = item_ref_list.type();
}
- parallel::broadcast(ref_tag_list, sender_rank);
-
- // sending references name size
- Array<size_t> ref_name_size_list{number_of_item_ref_list_per_proc[sender_rank]};
- if (parallel::rank() == sender_rank) {
- for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
- ++i_item_ref_list) {
- auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
- ref_name_size_list[i_item_ref_list] = item_ref_list.refId().tagName().size();
- }
+ }
+ parallel::broadcast(ref_item_list_type, sender_rank);
+
+ // sending references tags
+ Array<RefId::TagNumberType> ref_tag_list{number_of_item_ref_list_per_proc[sender_rank]};
+ if (parallel::rank() == sender_rank) {
+ for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
+ ++i_item_ref_list) {
+ auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
+ ref_tag_list[i_item_ref_list] = item_ref_list.refId().tagNumber();
}
- parallel::broadcast(ref_name_size_list, sender_rank);
-
- // sending references name size
- Array<RefId::TagNameType::value_type> ref_name_cat{sum(ref_name_size_list)};
- if (parallel::rank() == sender_rank) {
- size_t i_char = 0;
- for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
- ++i_item_ref_list) {
- auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
- for (auto c : item_ref_list.refId().tagName()) {
- ref_name_cat[i_char++] = c;
- }
- }
+ }
+ parallel::broadcast(ref_tag_list, sender_rank);
+
+ // sending references name size
+ Array<size_t> ref_name_size_list{number_of_item_ref_list_per_proc[sender_rank]};
+ if (parallel::rank() == sender_rank) {
+ for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
+ ++i_item_ref_list) {
+ auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
+ ref_name_size_list[i_item_ref_list] = item_ref_list.refId().tagName().size();
}
- parallel::broadcast(ref_name_cat, sender_rank);
-
- std::vector<RefId> ref_id_list = [&]() {
- std::vector<RefId> mutable_ref_id_list;
- mutable_ref_id_list.reserve(ref_name_size_list.size());
- size_t begining = 0;
- for (size_t i_ref = 0; i_ref < ref_name_size_list.size(); ++i_ref) {
- const size_t size = ref_name_size_list[i_ref];
- mutable_ref_id_list.emplace_back(ref_tag_list[i_ref], std::string{&(ref_name_cat[begining]), size});
- begining += size;
+ }
+ parallel::broadcast(ref_name_size_list, sender_rank);
+
+ // sending references name size
+ Array<RefId::TagNameType::value_type> ref_name_cat{sum(ref_name_size_list)};
+ if (parallel::rank() == sender_rank) {
+ size_t i_char = 0;
+ for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
+ ++i_item_ref_list) {
+ auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
+ for (auto c : item_ref_list.refId().tagName()) {
+ ref_name_cat[i_char++] = c;
}
- return mutable_ref_id_list;
- }();
-
- using block_type = int32_t;
- constexpr size_t block_size = sizeof(block_type);
- const size_t nb_block = ref_id_list.size() / block_size + (ref_id_list.size() % block_size != 0);
- for (size_t i_block = 0; i_block < nb_block; ++i_block) {
- ItemValue<block_type, item_type> item_references(m_connectivity);
- item_references.fill(0);
-
- if (m_connectivity.template numberOfRefItemList<item_type>() > 0) {
- const size_t max_i_ref = std::min(ref_id_list.size(), block_size * (i_block + 1));
- for (size_t i_ref = block_size * i_block, i = 0; i_ref < max_i_ref; ++i_ref, ++i) {
- block_type ref_bit{1 << i};
- auto item_ref_list = m_connectivity.template refItemList<item_type>(i_ref);
-
- const auto& item_list = item_ref_list.list();
- for (size_t i_item = 0; i_item < item_list.size(); ++i_item) {
- const ItemId& item_id = item_list[i_item];
- item_references[item_id] |= ref_bit;
- }
+ }
+ }
+ parallel::broadcast(ref_name_cat, sender_rank);
+
+ std::vector<RefId> ref_id_list = [&]() {
+ std::vector<RefId> mutable_ref_id_list;
+ mutable_ref_id_list.reserve(ref_name_size_list.size());
+ size_t begining = 0;
+ for (size_t i_ref = 0; i_ref < ref_name_size_list.size(); ++i_ref) {
+ const size_t size = ref_name_size_list[i_ref];
+ mutable_ref_id_list.emplace_back(ref_tag_list[i_ref], std::string{&(ref_name_cat[begining]), size});
+ begining += size;
+ }
+ return mutable_ref_id_list;
+ }();
+
+ using block_type = int32_t;
+ constexpr size_t block_size = sizeof(block_type);
+ const size_t nb_block = ref_id_list.size() / block_size + (ref_id_list.size() % block_size != 0);
+ for (size_t i_block = 0; i_block < nb_block; ++i_block) {
+ ItemValue<block_type, item_type> item_references(m_connectivity);
+ item_references.fill(0);
+
+ if (m_connectivity.template numberOfRefItemList<item_type>() > 0) {
+ const size_t max_i_ref = std::min(ref_id_list.size(), block_size * (i_block + 1));
+ for (size_t i_ref = block_size * i_block, i = 0; i_ref < max_i_ref; ++i_ref, ++i) {
+ block_type ref_bit{1 << i};
+ auto item_ref_list = m_connectivity.template refItemList<item_type>(i_ref);
+
+ const auto& item_list = item_ref_list.list();
+ for (size_t i_item = 0; i_item < item_list.size(); ++i_item) {
+ const ItemId& item_id = item_list[i_item];
+ item_references[item_id] |= ref_bit;
}
}
+ }
- const auto& nb_item_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_size_by_proc;
+ const auto& nb_item_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_size_by_proc;
- const auto& send_item_id_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;
+ const auto& send_item_id_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;
- std::vector<Array<const block_type>> send_item_refs_by_proc(parallel::size());
+ std::vector<Array<const block_type>> send_item_refs_by_proc(parallel::size());
- for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
- Array<block_type> send_item_refs(nb_item_to_send_by_proc[i_rank]);
- const Array<const ItemId> send_item_id = send_item_id_by_proc[i_rank];
- parallel_for(
- send_item_id.size(), PUGS_LAMBDA(size_t l) {
- const ItemId& item_id = send_item_id[l];
- send_item_refs[l] = item_references[item_id];
- });
- send_item_refs_by_proc[i_rank] = send_item_refs;
- }
+ for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
+ Array<block_type> send_item_refs(nb_item_to_send_by_proc[i_rank]);
+ const Array<const ItemId> send_item_id = send_item_id_by_proc[i_rank];
+ parallel_for(
+ send_item_id.size(), PUGS_LAMBDA(size_t l) {
+ const ItemId& item_id = send_item_id[l];
+ send_item_refs[l] = item_references[item_id];
+ });
+ send_item_refs_by_proc[i_rank] = send_item_refs;
+ }
- std::vector<Array<block_type>> recv_item_refs_by_proc(parallel::size());
- for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
- recv_item_refs_by_proc[i_rank] =
- Array<block_type>(this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc[i_rank]);
- }
- parallel::exchange(send_item_refs_by_proc, recv_item_refs_by_proc);
-
- const auto& recv_item_id_correspondance_by_proc =
- this->_dispatchedInfo<item_type>().m_recv_id_correspondance_by_proc;
- std::vector<block_type> item_refs(m_new_descriptor.template itemNumberVector<item_type>().size());
- for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
- for (size_t r = 0; r < recv_item_refs_by_proc[i_rank].size(); ++r) {
- const ItemId& item_id = recv_item_id_correspondance_by_proc[i_rank][r];
- item_refs[item_id] = recv_item_refs_by_proc[i_rank][r];
- }
+ std::vector<Array<block_type>> recv_item_refs_by_proc(parallel::size());
+ for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
+ recv_item_refs_by_proc[i_rank] =
+ Array<block_type>(this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc[i_rank]);
+ }
+ parallel::exchange(send_item_refs_by_proc, recv_item_refs_by_proc);
+
+ const auto& recv_item_id_correspondance_by_proc =
+ this->_dispatchedInfo<item_type>().m_recv_id_correspondance_by_proc;
+ std::vector<block_type> item_refs(m_new_descriptor.template itemNumberVector<item_type>().size());
+ for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
+ for (size_t r = 0; r < recv_item_refs_by_proc[i_rank].size(); ++r) {
+ const ItemId& item_id = recv_item_id_correspondance_by_proc[i_rank][r];
+ item_refs[item_id] = recv_item_refs_by_proc[i_rank][r];
}
+ }
- const size_t max_i_ref = std::min(ref_id_list.size(), block_size * (i_block + 1));
- for (size_t i_ref = block_size * i_block, i = 0; i_ref < max_i_ref; ++i_ref, ++i) {
- block_type ref_bit{1 << i};
+ const size_t max_i_ref = std::min(ref_id_list.size(), block_size * (i_block + 1));
+ for (size_t i_ref = block_size * i_block, i = 0; i_ref < max_i_ref; ++i_ref, ++i) {
+ block_type ref_bit{1 << i};
- std::vector<ItemId> item_id_vector;
+ std::vector<ItemId> item_id_vector;
- for (uint32_t i_item = 0; i_item < item_refs.size(); ++i_item) {
- const ItemId item_id{i_item};
- if (item_refs[item_id] & ref_bit) {
- item_id_vector.push_back(item_id);
- }
+ for (uint32_t i_item = 0; i_item < item_refs.size(); ++i_item) {
+ const ItemId item_id{i_item};
+ if (item_refs[item_id] & ref_bit) {
+ item_id_vector.push_back(item_id);
}
+ }
- Array<const ItemId> item_id_array = convert_to_array(item_id_vector);
+ Array<const ItemId> item_id_array = convert_to_array(item_id_vector);
- RefItemListBase::Type type = ref_item_list_type[i_ref];
- m_new_descriptor.addRefItemList(RefItemList<item_type>(ref_id_list[i_ref], item_id_array, type));
- }
+ RefItemListBase::Type type = ref_item_list_type[i_ref];
+ m_new_descriptor.addRefItemList(RefItemList<item_type>(ref_id_list[i_ref], item_id_array, type));
}
}
}
--
GitLab