diff --git a/src/mesh/ItemValueSynchronizer.hpp b/src/mesh/ItemValueSynchronizer.hpp index 6e73d5a2c4f8e49748486ea837c0689fa517b2e5..610ac380c99bfe311164a951ce4759a82d9785b9 100644 --- a/src/mesh/ItemValueSynchronizer.hpp +++ b/src/mesh/ItemValueSynchronizer.hpp @@ -8,6 +8,51 @@ class ItemValueSynchronizer { + template <ItemType item_type> + using ExchangeItemTypeInfo = std::vector<Array<const ItemIdT<item_type>>>; + + ExchangeItemTypeInfo<ItemType::cell> m_requested_cell_info; + ExchangeItemTypeInfo<ItemType::cell> m_provided_cell_info; + + ExchangeItemTypeInfo<ItemType::face> m_requested_face_info; + ExchangeItemTypeInfo<ItemType::face> m_provided_face_info; + + ExchangeItemTypeInfo<ItemType::edge> m_requested_edge_info; + ExchangeItemTypeInfo<ItemType::edge> m_provided_edge_info; + + ExchangeItemTypeInfo<ItemType::node> m_requested_node_info; + ExchangeItemTypeInfo<ItemType::node> m_provided_node_info; + + template <ItemType item_type> + PASTIS_INLINE + constexpr auto& _getRequestedItemInfo() + { + if constexpr (item_type == ItemType::cell) { + return m_requested_cell_info; + } else if constexpr (item_type == ItemType::face) { + return m_requested_face_info; + } else if constexpr (item_type == ItemType::edge) { + return m_requested_edge_info; + } else if constexpr (item_type == ItemType::node) { + return m_requested_node_info; + } + } + + template <ItemType item_type> + PASTIS_INLINE + constexpr auto& _getProvidedItemInfo() + { + if constexpr (item_type == ItemType::cell) { + return m_provided_cell_info; + } else if constexpr (item_type == ItemType::face) { + return m_provided_face_info; + } else if constexpr (item_type == ItemType::edge) { + return m_provided_edge_info; + } else if constexpr (item_type == ItemType::node) { + return m_provided_node_info; + } + } + template <typename ConnectivityType, typename DataType, ItemType item_type, @@ -23,67 +68,79 @@ class ItemValueSynchronizer const auto& item_owner = connectivity.template owner<item_type>(); - std::vector<Array<const ItemId>> ghost_items_per_proc - = [&] () { - std::vector<std::vector<ItemId>> ghost_items_vector_per_proc(parallel::size()); - for (ItemId item_id=0; item_id<item_value.size(); ++item_id) { - if (const size_t owner = item_owner[item_id]; owner != parallel::rank()) { - ghost_items_vector_per_proc[owner].emplace_back(item_id); - } - } - std::vector<Array<const ItemId>> ghost_items_per_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - const auto& ghost_items_vector = ghost_items_vector_per_proc[i_rank]; - ghost_items_per_proc[i_rank] = convert_to_array(ghost_items_vector); - } - return ghost_items_per_proc; - }(); - - Array<unsigned int> local_number_of_requested_values(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - local_number_of_requested_values[i_rank] = ghost_items_per_proc[i_rank].size(); - } + auto& ghost_items_per_proc = this->_getRequestedItemInfo<item_type>(); + auto& to_send_item_id_list_by_proc = this->_getProvidedItemInfo<item_type>(); - Array<unsigned int> local_number_of_values_to_send - = parallel::allToAll(local_number_of_requested_values); + Assert(ghost_items_per_proc.size() == to_send_item_id_list_by_proc.size()); - std::vector<Array<const int>> requested_item_number_list_by_proc(parallel::size()); - const auto& item_number = connectivity.template number<item_type>(); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - const auto& ghost_items = ghost_items_per_proc[i_rank]; - Array<int> item_number_list(ghost_items.size()); - parallel_for (ghost_items.size(), PASTIS_LAMBDA(size_t i_item) { - item_number_list[i_item] = item_number[ghost_items[i_item]]; - }); - requested_item_number_list_by_proc[i_rank] = item_number_list; - } + if (ghost_items_per_proc.size() == 0) { + pout() << "... building synchronization info\n"; + ghost_items_per_proc + = [&] () { + std::vector<std::vector<ItemId>> ghost_items_vector_per_proc(parallel::size()); + for (ItemId item_id=0; item_id<item_value.size(); ++item_id) { + if (const size_t owner = item_owner[item_id]; owner != parallel::rank()) { + ghost_items_vector_per_proc[owner].emplace_back(item_id); + } + } + std::vector<Array<const ItemId>> ghost_items_per_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + const auto& ghost_items_vector = ghost_items_vector_per_proc[i_rank]; + ghost_items_per_proc[i_rank] = convert_to_array(ghost_items_vector); + } + return ghost_items_per_proc; + }(); - std::vector<Array<int>> to_send_item_number_list_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - to_send_item_number_list_by_proc[i_rank] = Array<int>{local_number_of_values_to_send[i_rank]}; - } + Array<unsigned int> local_number_of_requested_values(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + local_number_of_requested_values[i_rank] = ghost_items_per_proc[i_rank].size(); + } - parallel::exchange(requested_item_number_list_by_proc, to_send_item_number_list_by_proc); + Array<unsigned int> local_number_of_values_to_send + = parallel::allToAll(local_number_of_requested_values); + + std::vector<Array<const int>> requested_item_number_list_by_proc(parallel::size()); + const auto& item_number = connectivity.template number<item_type>(); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + const auto& ghost_items = ghost_items_per_proc[i_rank]; + Array<int> item_number_list(ghost_items.size()); + parallel_for (ghost_items.size(), PASTIS_LAMBDA(size_t i_item) { + item_number_list[i_item] = item_number[ghost_items[i_item]]; + }); + requested_item_number_list_by_proc[i_rank] = item_number_list; + } - std::map<int, ItemId> item_number_to_id_correspondance; - for (ItemId item_id=0; item_id<item_number.size(); ++item_id) { - item_number_to_id_correspondance[item_number[item_id]] = item_id; - } + std::vector<Array<int>> to_send_item_number_list_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + to_send_item_number_list_by_proc[i_rank] = Array<int>{local_number_of_values_to_send[i_rank]}; + } - std::vector<Array<const ItemId>> to_send_item_id_list_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - Array<ItemId> to_send_item_id{local_number_of_values_to_send[i_rank]}; - const Array<int>& to_send_item_number = to_send_item_number_list_by_proc[i_rank]; - for (size_t i=0; i<to_send_item_number.size(); ++i) { - to_send_item_id[i] = item_number_to_id_correspondance.find(to_send_item_number[i])->second; + parallel::exchange(requested_item_number_list_by_proc, to_send_item_number_list_by_proc); + + std::map<int, ItemId> item_number_to_id_correspondance; + for (ItemId item_id=0; item_id<item_number.size(); ++item_id) { + item_number_to_id_correspondance[item_number[item_id]] = item_id; } - to_send_item_id_list_by_proc[i_rank] = to_send_item_id; + + to_send_item_id_list_by_proc + = [&] () { + std::vector<Array<const ItemId>> to_send_item_id_list_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + Array<ItemId> to_send_item_id{local_number_of_values_to_send[i_rank]}; + const Array<int>& to_send_item_number = to_send_item_number_list_by_proc[i_rank]; + for (size_t i=0; i<to_send_item_number.size(); ++i) { + to_send_item_id[i] = item_number_to_id_correspondance.find(to_send_item_number[i])->second; + } + to_send_item_id_list_by_proc[i_rank] = to_send_item_id; + } + return to_send_item_id_list_by_proc; + } (); } std::vector<Array<const DataType>> to_send_data_by_proc(parallel::size()); for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - Array<DataType> to_send_data{local_number_of_values_to_send[i_rank]}; const Array<const ItemId>& to_send_item_id = to_send_item_id_list_by_proc[i_rank]; + Array<DataType> to_send_data{to_send_item_id.size()}; parallel_for(to_send_item_id.size(), PASTIS_LAMBDA(size_t i) { to_send_data[i] = item_value[to_send_item_id[i]]; }); @@ -115,9 +172,6 @@ class ItemValueSynchronizer PASTIS_INLINE void synchronize(ItemValue<DataType, item_type, ConnectivityPtr>& item_value) { - static int cpt=0; - pout() << "Calling synchronize(" << cpt++ << ")...\n"; - Assert(item_value.connectivity_ptr().use_count()>0, "No connectivity is associated to this ItemValue"); const IConnectivity& connectivity = *item_value.connectivity_ptr(); diff --git a/src/mesh/ItemValueUtils.hpp b/src/mesh/ItemValueUtils.hpp index 771b4e2b675aaac2eab00ba8cb5575f3963e6fbb..e0e1f9608a53578f579fbdda1bfaa9d9afd2d290 100644 --- a/src/mesh/ItemValueUtils.hpp +++ b/src/mesh/ItemValueUtils.hpp @@ -300,7 +300,7 @@ void synchronize(ItemValue<DataType, item_type, ConnectivityPtr>& item_value) { static_assert(not std::is_const_v<DataType>, "cannot synchronize ItemValue of const data"); if (parallel::size() > 1) { - ItemValueSynchronizer synchronizer; + static ItemValueSynchronizer synchronizer; synchronizer.synchronize(item_value); } }