From 033af9cb25e44f34510b1cb5380af025f979c0cb Mon Sep 17 00:00:00 2001 From: Stephane Del Pino <stephane.delpino44@gmail.com> Date: Thu, 7 Mar 2019 16:26:59 +0100 Subject: [PATCH] Continue clean-up Face dispatching is now performed in a separate function --- src/mesh/MeshDispatcher.cpp | 486 ++++++++++++++++++------------------ src/mesh/MeshDispatcher.hpp | 13 +- 2 files changed, 253 insertions(+), 246 deletions(-) diff --git a/src/mesh/MeshDispatcher.cpp b/src/mesh/MeshDispatcher.cpp index 0677ccce7..a75d31fcf 100644 --- a/src/mesh/MeshDispatcher.cpp +++ b/src/mesh/MeshDispatcher.cpp @@ -118,224 +118,15 @@ MeshDispatcher<Dimension>::_buildNbCellToSend() } template <int Dimension> -MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) - : m_mesh(mesh), - m_cell_new_owner(_getCellNewOwner()), - m_face_new_owner(_getFaceNewOwner()), - m_node_new_owner(_getNodeNewOwner()), - m_cell_list_to_send_by_proc(_buildCellListToSend()), - m_nb_cell_to_send_by_proc(_buildNbCellToSend()), - m_nb_cell_to_recv_by_proc(parallel::allToAll(m_nb_cell_to_send_by_proc)) +void +MeshDispatcher<Dimension>::_dispatchFaces() { - std::vector<Array<int>> recv_cell_number_by_proc - = this->exchange(mesh.connectivity().cellNumber()); - - const auto& cell_to_node_matrix = mesh.connectivity().cellToNodeMatrix(); - - std::vector<Array<int>> cell_node_number_to_send_by_proc = - [&] () { - const NodeValue<const int>& node_number = mesh.connectivity().nodeNumber(); - std::vector<Array<int>> cell_node_number_to_send_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { - std::vector<int> node_number_by_cell_vector; - for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) { - const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j]; - const auto& cell_node_list = cell_to_node_matrix[cell_id]; - for (size_t r=0; r<cell_node_list.size(); ++r) { - const NodeId& node_id = cell_node_list[r]; - node_number_by_cell_vector.push_back(node_number[node_id]); - } - } - cell_node_number_to_send_by_proc[i_rank] = convert_to_array(node_number_by_cell_vector); - } - return cell_node_number_to_send_by_proc; - } (); - - std::vector<Array<int>> recv_number_of_node_per_cell_by_proc - = [&] () { - CellValue<int> number_of_node_per_cell(mesh.connectivity()); - parallel_for(mesh.numberOfCells(), PASTIS_LAMBDA(const CellId& j){ - number_of_node_per_cell[j] = cell_to_node_matrix[j].size(); - }); - return this->exchange(number_of_node_per_cell); - } (); - - std::vector<Array<int>> recv_cell_node_number_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { - recv_cell_node_number_by_proc[i_rank] - = Array<int>(sum(recv_number_of_node_per_cell_by_proc[i_rank])); - } - parallel::exchange(cell_node_number_to_send_by_proc, recv_cell_node_number_by_proc); - - const std::unordered_map<int, int> node_number_id_map - = [&] () { - std::unordered_map<int, int> node_number_id_map; - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - int cpt=0; - for (size_t i=0; i<recv_cell_node_number_by_proc[i_rank].size(); ++i) { - int node_number = recv_cell_node_number_by_proc[i_rank][i]; - auto [iterator, inserted] = node_number_id_map.insert(std::make_pair(node_number, cpt)); - if (inserted) cpt++; - } - } - return node_number_id_map; - } (); - - m_new_descriptor.node_number_vector.resize(node_number_id_map.size()); - for (const auto& [number, id] : node_number_id_map) { - m_new_descriptor.node_number_vector[id] = number; - } - - for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { - int l=0; - for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) { - std::vector<unsigned int> node_vector; - for (int k=0; k<recv_number_of_node_per_cell_by_proc[i_rank][i]; ++k) { - const auto& searched_node_id = node_number_id_map.find(recv_cell_node_number_by_proc[i_rank][l++]); - Assert(searched_node_id != node_number_id_map.end()); - node_vector.push_back(searched_node_id->second); - } - m_new_descriptor.cell_by_node_vector.emplace_back(node_vector); - } - } - - const std::unordered_map<int, int> cell_number_id_map - = [&] () { - std::unordered_map<int, int> cell_number_id_map; - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - int cpt=0; - for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) { - int cell_number = recv_cell_number_by_proc[i_rank][i]; - auto [iterator, inserted] = cell_number_id_map.insert(std::make_pair(cell_number, cpt)); - if (inserted) cpt++; - } - } - return cell_number_id_map; - } (); - - m_new_descriptor.cell_number_vector.resize(cell_number_id_map.size()); - for (const auto& [number, id] : cell_number_id_map) { - m_new_descriptor.cell_number_vector[id] = number; - } - - { - std::vector<Array<CellType>> recv_cell_type_by_proc - = this->exchange(mesh.connectivity().cellType()); - m_new_descriptor.cell_type_vector.resize(cell_number_id_map.size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) { - int cell_number = recv_cell_number_by_proc[i_rank][i]; - const auto& searched_cell_id = cell_number_id_map.find(cell_number); - Assert(searched_cell_id != cell_number_id_map.end()); - m_new_descriptor.cell_type_vector[searched_cell_id->second] = recv_cell_type_by_proc[i_rank][i]; - } - } - } - - std::vector<Array<const NodeId>> send_node_id_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - Array<bool> tag(mesh.numberOfNodes()); - tag.fill(false); - std::vector<NodeId> node_id_vector; - for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) { - const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j]; - const auto& cell_node_list = cell_to_node_matrix[cell_id]; - for (size_t r=0; r<cell_node_list.size(); ++r) { - const NodeId& node_id = cell_node_list[r]; - if (not tag[node_id]) { - node_id_vector.push_back(node_id); - tag[node_id] = true; - } - } - } - send_node_id_by_proc[i_rank] = convert_to_array(node_id_vector); - } - - Array<unsigned int> nb_node_to_send_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - nb_node_to_send_by_proc[i_rank] = send_node_id_by_proc[i_rank].size(); - } - Array<const unsigned int> nb_node_to_recv_by_proc - = parallel::allToAll(nb_node_to_send_by_proc); - - std::vector<Array<const NodeId>> recv_node_id_correspondance_by_proc(parallel::size()); - { - const NodeValue<const int>& node_number = mesh.connectivity().nodeNumber(); - std::vector<Array<const int>> send_node_number_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - Array<int> send_node_number(send_node_id_by_proc[i_rank].size()); - const Array<const NodeId> send_node_id = send_node_id_by_proc[i_rank]; - parallel_for(send_node_number.size(), PASTIS_LAMBDA(const size_t& j){ - send_node_number[j] = node_number[send_node_id[j]]; - }); - send_node_number_by_proc[i_rank] = send_node_number; - } - - std::vector<Array<int>> recv_node_number_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { - recv_node_number_by_proc[i_rank] = Array<int>(nb_node_to_recv_by_proc[i_rank]); - } - parallel::exchange(send_node_number_by_proc, recv_node_number_by_proc); - - for (size_t i_rank=0; i_rank<nb_node_to_recv_by_proc.size(); ++i_rank) { - Array<NodeId> node_id_correspondace(nb_node_to_recv_by_proc[i_rank]); - for (size_t l=0; l<nb_node_to_recv_by_proc[i_rank]; ++l) { - const int& node_number = recv_node_number_by_proc[i_rank][l]; - const auto& searched_node_id = node_number_id_map.find(node_number); - Assert(searched_node_id != node_number_id_map.end()); - node_id_correspondace[l] = searched_node_id->second; - } - recv_node_id_correspondance_by_proc[i_rank] = node_id_correspondace; - } - } - - { - std::vector<Array<int>> recv_cell_new_owner_by_proc - = this->exchange(m_cell_new_owner); - m_new_descriptor.cell_owner_vector.resize(cell_number_id_map.size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) { - int cell_number = recv_cell_number_by_proc[i_rank][i]; - const auto& searched_cell_id = cell_number_id_map.find(cell_number); - Assert(searched_cell_id != cell_number_id_map.end()); - m_new_descriptor.cell_owner_vector[searched_cell_id->second] = recv_cell_new_owner_by_proc[i_rank][i]; - } - } - } - - { - std::vector<Array<const int>> send_node_owner_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - Array<int> send_node_owner(nb_node_to_send_by_proc[i_rank]); - const Array<const NodeId> send_node_id = send_node_id_by_proc[i_rank]; - parallel_for(send_node_id.size(), PASTIS_LAMBDA(const size_t& r) { - const NodeId& node_id = send_node_id[r]; - send_node_owner[r] = m_node_new_owner[node_id]; - }); - send_node_owner_by_proc[i_rank] = send_node_owner; - } - - std::vector<Array<int>> recv_node_owner_by_proc(parallel::size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - recv_node_owner_by_proc[i_rank] = Array<int>(nb_node_to_recv_by_proc[i_rank]); - } - parallel::exchange(send_node_owner_by_proc, recv_node_owner_by_proc); - - m_new_descriptor.node_owner_vector.resize(m_new_descriptor.node_number_vector.size()); - for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - for (size_t r=0; r<recv_node_owner_by_proc[i_rank].size(); ++r) { - const NodeId& node_id = recv_node_id_correspondance_by_proc[i_rank][r]; - m_new_descriptor.node_owner_vector[node_id] = recv_node_owner_by_proc[i_rank][r]; - } - } - } - - if constexpr(Dimension>1) { // Faces + if constexpr (Dimension>1) { std::vector<Array<int>> recv_number_of_face_per_cell_by_proc = [&] () { - CellValue<int> number_of_face_per_cell(mesh.connectivity()); - const auto& cell_to_face_matrix = mesh.connectivity().cellToFaceMatrix(); - parallel_for(mesh.numberOfCells(), PASTIS_LAMBDA(const CellId& j){ + CellValue<int> number_of_face_per_cell(m_mesh.connectivity()); + const auto& cell_to_face_matrix = m_mesh.connectivity().cellToFaceMatrix(); + parallel_for(m_mesh.numberOfCells(), PASTIS_LAMBDA(const CellId& j){ number_of_face_per_cell[j] = cell_to_face_matrix[j].size(); }); @@ -344,8 +135,8 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) std::vector<Array<int>> recv_cell_face_number_by_proc(parallel::size()); { - const auto& cell_to_face_matrix = mesh.connectivity().cellToFaceMatrix(); - const FaceValue<const int>& face_number = mesh.connectivity().faceNumber(); + const auto& cell_to_face_matrix = m_mesh.connectivity().cellToFaceMatrix(); + const FaceValue<const int>& face_number = m_mesh.connectivity().faceNumber(); std::vector<Array<int>> cell_face_number_to_send_by_proc(parallel::size()); { @@ -393,7 +184,7 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { int l=0; - for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) { + for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) { std::vector<unsigned int> face_vector; for (int k=0; k<recv_number_of_face_per_cell_by_proc[i_rank][i]; ++k) { const auto& searched_face_id = face_number_id_map.find(recv_cell_face_number_by_proc[i_rank][l++]); @@ -407,7 +198,7 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) { std::vector<Array<bool>> cell_face_is_reversed_to_send_by_proc(parallel::size()); { - const auto& cell_face_is_reversed = mesh.connectivity().cellFaceIsReversed(); + const auto& cell_face_is_reversed = m_mesh.connectivity().cellFaceIsReversed(); for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { std::vector<bool> face_is_reversed_by_cell_vector; for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) { @@ -431,7 +222,7 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { int l=0; - for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) { + for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) { std::vector<bool> face_is_reversed_vector; for (int k=0; k<recv_number_of_face_per_cell_by_proc[i_rank][i]; ++k) { face_is_reversed_vector.push_back(recv_cell_face_is_reversed_by_proc[i_rank][l++]); @@ -443,10 +234,10 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) std::vector<Array<const FaceId>> send_face_id_by_proc(parallel::size()); { - const auto& cell_to_face_matrix = mesh.connectivity().cellToFaceMatrix(); + const auto& cell_to_face_matrix = m_mesh.connectivity().cellToFaceMatrix(); for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { - Array<bool> tag(mesh.numberOfFaces()); + Array<bool> tag(m_mesh.numberOfFaces()); tag.fill(false); std::vector<FaceId> face_id_vector; for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) { @@ -466,7 +257,7 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) std::vector<Array<const int>> send_face_number_by_proc(parallel::size()); { - const FaceValue<const int>& face_number = mesh.connectivity().faceNumber(); + const FaceValue<const int>& face_number = m_mesh.connectivity().faceNumber(); for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { Array<int> send_face_number(send_face_id_by_proc[i_rank].size()); @@ -537,9 +328,9 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) std::vector<Array<int>> recv_face_number_of_node_by_proc(parallel::size()); { - const auto& face_to_node_matrix = mesh.connectivity().faceToNodeMatrix(); - FaceValue<int> number_of_node_per_face(mesh.connectivity()); - parallel_for(mesh.numberOfFaces(), PASTIS_LAMBDA(const FaceId& j){ + const auto& face_to_node_matrix = m_mesh.connectivity().faceToNodeMatrix(); + FaceValue<int> number_of_node_per_face(m_mesh.connectivity()); + parallel_for(m_mesh.numberOfFaces(), PASTIS_LAMBDA(const FaceId& j){ number_of_node_per_face[j] = face_to_node_matrix[j].size(); }); @@ -562,8 +353,8 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) std::vector<Array<int>> face_node_number_to_send_by_proc(parallel::size()); { - const auto& face_to_node_matrix = mesh.connectivity().faceToNodeMatrix(); - const NodeValue<const int>& node_number = mesh.connectivity().nodeNumber(); + const auto& face_to_node_matrix = m_mesh.connectivity().faceToNodeMatrix(); + const NodeValue<const int>& node_number = m_mesh.connectivity().nodeNumber(); for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { std::vector<int> node_number_by_face_vector; for (size_t l=0; l<send_face_id_by_proc[i_rank].size(); ++l) { @@ -592,8 +383,8 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) for (size_t i=0; i<recv_face_number_by_proc[i_rank].size(); ++i) { std::vector<unsigned int> node_vector; for (int k=0; k<recv_face_number_of_node_by_proc[i_rank][i]; ++k) { - const auto& searched_node_id = node_number_id_map.find(recv_face_node_number_by_proc[i_rank][l++]); - Assert(searched_node_id != node_number_id_map.end()); + const auto& searched_node_id = m_node_number_id_map.find(recv_face_node_number_by_proc[i_rank][l++]); + Assert(searched_node_id != m_node_number_id_map.end()); node_vector.push_back(searched_node_id->second); } m_new_descriptor.face_to_node_vector.emplace_back(node_vector); @@ -603,7 +394,7 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) // Getting references Array<const size_t> number_of_ref_face_list_per_proc - = parallel::allGather(mesh.connectivity().numberOfRefFaceList()); + = parallel::allGather(m_mesh.connectivity().numberOfRefFaceList()); const size_t number_of_face_list_sender = [&] () { @@ -640,9 +431,9 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) // sending references tags Array<RefId::TagNumberType> ref_tag_list{number_of_ref_face_list_per_proc[sender_rank]}; if (parallel::rank() == sender_rank){ - for (size_t i_ref_face_list=0; i_ref_face_list<mesh.connectivity().numberOfRefFaceList(); + for (size_t i_ref_face_list=0; i_ref_face_list<m_mesh.connectivity().numberOfRefFaceList(); ++i_ref_face_list) { - auto ref_face_list = mesh.connectivity().refFaceList(i_ref_face_list); + auto ref_face_list = m_mesh.connectivity().refFaceList(i_ref_face_list); ref_tag_list[i_ref_face_list] = ref_face_list.refId().tagNumber(); } } @@ -651,9 +442,9 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) // sending references name size Array<size_t> ref_name_size_list{number_of_ref_face_list_per_proc[sender_rank]}; if (parallel::rank() == sender_rank){ - for (size_t i_ref_face_list=0; i_ref_face_list<mesh.connectivity().numberOfRefFaceList(); + for (size_t i_ref_face_list=0; i_ref_face_list<m_mesh.connectivity().numberOfRefFaceList(); ++i_ref_face_list) { - auto ref_face_list = mesh.connectivity().refFaceList(i_ref_face_list); + auto ref_face_list = m_mesh.connectivity().refFaceList(i_ref_face_list); ref_name_size_list[i_ref_face_list] = ref_face_list.refId().tagName().size(); } } @@ -663,9 +454,9 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) Array<RefId::TagNameType::value_type> ref_name_cat{sum(ref_name_size_list)}; if (parallel::rank() == sender_rank){ size_t i_char=0; - for (size_t i_ref_face_list=0; i_ref_face_list<mesh.connectivity().numberOfRefFaceList(); + for (size_t i_ref_face_list=0; i_ref_face_list<m_mesh.connectivity().numberOfRefFaceList(); ++i_ref_face_list) { - auto ref_face_list = mesh.connectivity().refFaceList(i_ref_face_list); + auto ref_face_list = m_mesh.connectivity().refFaceList(i_ref_face_list); for (auto c : ref_face_list.refId().tagName()) { ref_name_cat[i_char++] = c; } @@ -691,14 +482,14 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) constexpr size_t block_size = sizeof(block_type); const size_t nb_block = ref_id_list.size()/block_size + (ref_id_list.size()%block_size != 0); for (size_t i_block=0; i_block<nb_block; ++i_block) { - FaceValue<block_type> face_references(mesh.connectivity()); + FaceValue<block_type> face_references(m_mesh.connectivity()); face_references.fill(0); - if (mesh.connectivity().numberOfRefFaceList() > 0) { + if (m_mesh.connectivity().numberOfRefFaceList() > 0) { const size_t max_i_ref = std::min(ref_id_list.size(), block_size*(i_block+1)); for (size_t i_ref=block_size*i_block, i=0; i_ref<max_i_ref; ++i_ref, ++i) { block_type ref_bit{1<<i}; - auto ref_face_list = mesh.connectivity().refFaceList(i_ref); + auto ref_face_list = m_mesh.connectivity().refFaceList(i_ref); const auto& face_list = ref_face_list.faceList(); for (size_t i_face=0; i_face<face_list.size(); ++i_face) { @@ -755,9 +546,224 @@ MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) } } } + } +} + + +template <int Dimension> +MeshDispatcher<Dimension>::MeshDispatcher(const MeshType& mesh) + : m_mesh(mesh), + m_cell_new_owner(_getCellNewOwner()), + m_face_new_owner(_getFaceNewOwner()), + m_node_new_owner(_getNodeNewOwner()), + m_cell_list_to_send_by_proc(_buildCellListToSend()), + m_nb_cell_to_send_by_proc(_buildNbCellToSend()), + m_nb_cell_to_recv_by_proc(parallel::allToAll(m_nb_cell_to_send_by_proc)) +{ + m_recv_cell_number_by_proc = this->exchange(mesh.connectivity().cellNumber()); + + const auto& cell_to_node_matrix = mesh.connectivity().cellToNodeMatrix(); + + std::vector<Array<int>> cell_node_number_to_send_by_proc = + [&] () { + const NodeValue<const int>& node_number = mesh.connectivity().nodeNumber(); + std::vector<Array<int>> cell_node_number_to_send_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { + std::vector<int> node_number_by_cell_vector; + for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) { + const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j]; + const auto& cell_node_list = cell_to_node_matrix[cell_id]; + for (size_t r=0; r<cell_node_list.size(); ++r) { + const NodeId& node_id = cell_node_list[r]; + node_number_by_cell_vector.push_back(node_number[node_id]); + } + } + cell_node_number_to_send_by_proc[i_rank] = convert_to_array(node_number_by_cell_vector); + } + return cell_node_number_to_send_by_proc; + } (); + + std::vector<Array<int>> recv_number_of_node_per_cell_by_proc + = [&] () { + CellValue<int> number_of_node_per_cell(mesh.connectivity()); + parallel_for(mesh.numberOfCells(), PASTIS_LAMBDA(const CellId& j){ + number_of_node_per_cell[j] = cell_to_node_matrix[j].size(); + }); + return this->exchange(number_of_node_per_cell); + } (); + + std::vector<Array<int>> recv_cell_node_number_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { + recv_cell_node_number_by_proc[i_rank] + = Array<int>(sum(recv_number_of_node_per_cell_by_proc[i_rank])); + } + parallel::exchange(cell_node_number_to_send_by_proc, recv_cell_node_number_by_proc); + + m_node_number_id_map = + [&] () { + std::unordered_map<int, int> node_number_id_map; + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + int cpt=0; + for (size_t i=0; i<recv_cell_node_number_by_proc[i_rank].size(); ++i) { + int node_number = recv_cell_node_number_by_proc[i_rank][i]; + auto [iterator, inserted] = node_number_id_map.insert(std::make_pair(node_number, cpt)); + if (inserted) cpt++; + } + } + return node_number_id_map; + } (); + + m_new_descriptor.node_number_vector.resize(m_node_number_id_map.size()); + for (const auto& [number, id] : m_node_number_id_map) { + m_new_descriptor.node_number_vector[id] = number; + } + + for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { + int l=0; + for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) { + std::vector<unsigned int> node_vector; + for (int k=0; k<recv_number_of_node_per_cell_by_proc[i_rank][i]; ++k) { + const auto& searched_node_id = m_node_number_id_map.find(recv_cell_node_number_by_proc[i_rank][l++]); + Assert(searched_node_id != m_node_number_id_map.end()); + node_vector.push_back(searched_node_id->second); + } + m_new_descriptor.cell_by_node_vector.emplace_back(node_vector); + } + } + + const std::unordered_map<int, int> cell_number_id_map + = [&] () { + std::unordered_map<int, int> cell_number_id_map; + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + int cpt=0; + for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) { + int cell_number = m_recv_cell_number_by_proc[i_rank][i]; + auto [iterator, inserted] = cell_number_id_map.insert(std::make_pair(cell_number, cpt)); + if (inserted) cpt++; + } + } + return cell_number_id_map; + } (); + + m_new_descriptor.cell_number_vector.resize(cell_number_id_map.size()); + for (const auto& [number, id] : cell_number_id_map) { + m_new_descriptor.cell_number_vector[id] = number; + } + { + std::vector<Array<CellType>> recv_cell_type_by_proc + = this->exchange(mesh.connectivity().cellType()); + m_new_descriptor.cell_type_vector.resize(cell_number_id_map.size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) { + int cell_number = m_recv_cell_number_by_proc[i_rank][i]; + const auto& searched_cell_id = cell_number_id_map.find(cell_number); + Assert(searched_cell_id != cell_number_id_map.end()); + m_new_descriptor.cell_type_vector[searched_cell_id->second] = recv_cell_type_by_proc[i_rank][i]; + } + } } + std::vector<Array<const NodeId>> send_node_id_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + Array<bool> tag(mesh.numberOfNodes()); + tag.fill(false); + std::vector<NodeId> node_id_vector; + for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) { + const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j]; + const auto& cell_node_list = cell_to_node_matrix[cell_id]; + for (size_t r=0; r<cell_node_list.size(); ++r) { + const NodeId& node_id = cell_node_list[r]; + if (not tag[node_id]) { + node_id_vector.push_back(node_id); + tag[node_id] = true; + } + } + } + send_node_id_by_proc[i_rank] = convert_to_array(node_id_vector); + } + + Array<unsigned int> nb_node_to_send_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + nb_node_to_send_by_proc[i_rank] = send_node_id_by_proc[i_rank].size(); + } + Array<const unsigned int> nb_node_to_recv_by_proc + = parallel::allToAll(nb_node_to_send_by_proc); + + std::vector<Array<const NodeId>> recv_node_id_correspondance_by_proc(parallel::size()); + { + const NodeValue<const int>& node_number = mesh.connectivity().nodeNumber(); + std::vector<Array<const int>> send_node_number_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + Array<int> send_node_number(send_node_id_by_proc[i_rank].size()); + const Array<const NodeId> send_node_id = send_node_id_by_proc[i_rank]; + parallel_for(send_node_number.size(), PASTIS_LAMBDA(const size_t& j){ + send_node_number[j] = node_number[send_node_id[j]]; + }); + send_node_number_by_proc[i_rank] = send_node_number; + } + + std::vector<Array<int>> recv_node_number_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) { + recv_node_number_by_proc[i_rank] = Array<int>(nb_node_to_recv_by_proc[i_rank]); + } + parallel::exchange(send_node_number_by_proc, recv_node_number_by_proc); + + for (size_t i_rank=0; i_rank<nb_node_to_recv_by_proc.size(); ++i_rank) { + Array<NodeId> node_id_correspondace(nb_node_to_recv_by_proc[i_rank]); + for (size_t l=0; l<nb_node_to_recv_by_proc[i_rank]; ++l) { + const int& node_number = recv_node_number_by_proc[i_rank][l]; + const auto& searched_node_id = m_node_number_id_map.find(node_number); + Assert(searched_node_id != m_node_number_id_map.end()); + node_id_correspondace[l] = searched_node_id->second; + } + recv_node_id_correspondance_by_proc[i_rank] = node_id_correspondace; + } + } + + { + std::vector<Array<int>> recv_cell_new_owner_by_proc + = this->exchange(m_cell_new_owner); + m_new_descriptor.cell_owner_vector.resize(cell_number_id_map.size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) { + int cell_number = m_recv_cell_number_by_proc[i_rank][i]; + const auto& searched_cell_id = cell_number_id_map.find(cell_number); + Assert(searched_cell_id != cell_number_id_map.end()); + m_new_descriptor.cell_owner_vector[searched_cell_id->second] = recv_cell_new_owner_by_proc[i_rank][i]; + } + } + } + + { + std::vector<Array<const int>> send_node_owner_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + Array<int> send_node_owner(nb_node_to_send_by_proc[i_rank]); + const Array<const NodeId> send_node_id = send_node_id_by_proc[i_rank]; + parallel_for(send_node_id.size(), PASTIS_LAMBDA(const size_t& r) { + const NodeId& node_id = send_node_id[r]; + send_node_owner[r] = m_node_new_owner[node_id]; + }); + send_node_owner_by_proc[i_rank] = send_node_owner; + } + + std::vector<Array<int>> recv_node_owner_by_proc(parallel::size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + recv_node_owner_by_proc[i_rank] = Array<int>(nb_node_to_recv_by_proc[i_rank]); + } + parallel::exchange(send_node_owner_by_proc, recv_node_owner_by_proc); + + m_new_descriptor.node_owner_vector.resize(m_new_descriptor.node_number_vector.size()); + for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) { + for (size_t r=0; r<recv_node_owner_by_proc[i_rank].size(); ++r) { + const NodeId& node_id = recv_node_id_correspondance_by_proc[i_rank][r]; + m_new_descriptor.node_owner_vector[node_id] = recv_node_owner_by_proc[i_rank][r]; + } + } + } + + this->_dispatchFaces(); + using ConnectivityType = Connectivity<Dimension>; std::shared_ptr p_connectivity = ConnectivityType::build(m_new_descriptor); diff --git a/src/mesh/MeshDispatcher.hpp b/src/mesh/MeshDispatcher.hpp index f6a209e92..097245672 100644 --- a/src/mesh/MeshDispatcher.hpp +++ b/src/mesh/MeshDispatcher.hpp @@ -5,6 +5,8 @@ #include <ItemValue.hpp> #include <ItemValueUtils.hpp> +#include <unordered_map> + template <int Dimension> class MeshDispatcher { @@ -28,6 +30,9 @@ class MeshDispatcher Array<int> m_nb_cell_to_send_by_proc; Array<int> m_nb_cell_to_recv_by_proc; + std::vector<Array<int>> m_recv_cell_number_by_proc; + std::unordered_map<int, int> m_node_number_id_map; + CellValue<int> _getCellNewOwner(); FaceValue<int> _getFaceNewOwner(); NodeValue<int> _getNodeNewOwner(); @@ -35,6 +40,8 @@ class MeshDispatcher const CellListToSendByProc _buildCellListToSend() const; Array<int> _buildNbCellToSend(); + + void _dispatchFaces(); public: const std::shared_ptr<MeshType> dispatchedMesh() const @@ -84,12 +91,6 @@ class MeshDispatcher return recv_cell_value_by_proc; } - [[deprecated]] - const CellListToSendByProc& cell_list_to_send_by_proc() const - { - return m_cell_list_to_send_by_proc; - } - MeshDispatcher(const MeshType& mesh); MeshDispatcher(const MeshDispatcher&) = delete; -- GitLab