Skip to content
Snippets Groups Projects
Commit 7efc43fa authored by Stéphane Del Pino's avatar Stéphane Del Pino
Browse files

Fix connectivity dispatcher when for already partitioned meshes

parent 50ed2e37
No related branches found
No related tags found
1 merge request!204Remove m_cell_global_index from Connectivity
This commit is part of merge request !204. Comments created here will be created in the context of that merge request.
Show whitespace changes
Inline Side-by-side
src/mesh/ConnectivityDispatcher.cpp
+ 144
− 25
View file @ 7efc43fa
......@@ -5,9 +5,7 @@
#include <utils/Partitioner.hpp>
#include <iostream>
#warning remove
#include <utils/Demangle.hpp>
#include <unordered_set>
template <size_t Dimension>
template <ItemType item_type>
......@@ -84,7 +82,6 @@ ConnectivityDispatcher<Dimension>::_buildItemListToSend()
const auto& cell_is_owned = m_connectivity.cellIsOwned();
const auto& cell_new_owner = this->_dispatchedInfo<ItemType::cell>().m_new_owner;
const auto& cell_number = m_connectivity.cellNumber();
std::vector<std::vector<CellId>> cell_vector_to_send_by_proc(parallel::size());
Array<bool> send_to_rank(parallel::size());
......@@ -120,6 +117,8 @@ ConnectivityDispatcher<Dimension>::_buildItemListToSend()
const auto& cell_list_to_send_by_proc = this->_dispatchedInfo<ItemType::cell>().m_list_to_send_by_proc;
const auto& item_is_owned = m_connectivity.template isOwned<item_type>();
const auto& item_owner = m_connectivity.template owner<item_type>();
const auto& item_number = m_connectivity.template number<item_type>();
using ItemId = ItemIdT<item_type>;
const auto& cell_to_sub_item_matrix = m_connectivity.template getItemToItemMatrix<ItemType::cell, item_type>();
......@@ -127,20 +126,133 @@ ConnectivityDispatcher<Dimension>::_buildItemListToSend()
auto& item_list_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;
item_list_to_send_by_proc.resize(parallel::size());
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
Array<bool> tag(m_connectivity.template numberOf<item_type>());
struct ToNumber
{
int to;
int number;
bool
operator==(const ToNumber& x) const
{
return to == x.to and number == x.number;
}
ToNumber() = default;
ToNumber(int _to, int _number) : to{_to}, number{_number} {}
};
struct hashFunction
{
size_t
operator()(const ToNumber& x) const
{
return x.to ^ x.number;
}
};
std::vector<std::unordered_set<ToNumber, hashFunction>> request_not_owned_to_send_by_proc(parallel::size());
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
tag.fill(false);
std::vector<ItemId> item_id_vector;
for (size_t j = 0; j < cell_list_to_send_by_proc[i_rank].size(); ++j) {
const CellId& cell_id = cell_list_to_send_by_proc[i_rank][j];
for (size_t i_cell = 0; i_cell < cell_list_to_send_by_proc[i_rank].size(); ++i_cell) {
const CellId& cell_id = cell_list_to_send_by_proc[i_rank][i_cell];
const auto& cell_sub_item_list = cell_to_sub_item_matrix[cell_id];
for (size_t r = 0; r < cell_sub_item_list.size(); ++r) {
const ItemId& item_id = cell_sub_item_list[r];
if (item_is_owned[item_id] and (not tag[item_id])) {
for (size_t i_item = 0; i_item < cell_sub_item_list.size(); ++i_item) {
const ItemId& sub_item_id = cell_sub_item_list[i_item];
if (not item_is_owned[sub_item_id] and (not tag[sub_item_id])) {
request_not_owned_to_send_by_proc[item_owner[sub_item_id]].insert(
ToNumber{static_cast<int>(i_rank), item_number[sub_item_id]});
tag[sub_item_id] = true;
}
}
}
}
std::vector<Array<int>> not_owned_info_to_send_by_proc(parallel::size());
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
Array<int> info_to_send_by_proc(2 * request_not_owned_to_send_by_proc[i_rank].size());
size_t i = 0;
for (auto&& to_send_info : request_not_owned_to_send_by_proc[i_rank]) {
info_to_send_by_proc[i] = to_send_info.to;
info_to_send_by_proc[i + 1] = to_send_info.number;
i += 2;
}
not_owned_info_to_send_by_proc[i_rank] = info_to_send_by_proc;
}
std::vector<Array<int>> number_of_not_owned_to_send_by_proc(parallel::size());
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
number_of_not_owned_to_send_by_proc[i_rank] = Array<int>(1);
number_of_not_owned_to_send_by_proc[i_rank][0] = request_not_owned_to_send_by_proc[i_rank].size();
}
std::vector<Array<int>> number_of_possibly_item_to_send_by_proc(parallel::size());
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
number_of_possibly_item_to_send_by_proc[i_rank] = Array<int>(1);
}
parallel::exchange(number_of_not_owned_to_send_by_proc, number_of_possibly_item_to_send_by_proc);
std::vector<Array<int>> possible_info_to_send_demanded_by_rank(parallel::size());
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
possible_info_to_send_demanded_by_rank[i_rank] =
Array<int>(2 * number_of_possibly_item_to_send_by_proc[i_rank][0]);
}
parallel::exchange(not_owned_info_to_send_by_proc, possible_info_to_send_demanded_by_rank);
std::vector<std::vector<int>> possible_item_to_send_to_rank(parallel::size());
for (size_t i_demanding_rank = 0; i_demanding_rank < parallel::size(); ++i_demanding_rank) {
const auto& possible_info_to_send_to = possible_info_to_send_demanded_by_rank[i_demanding_rank];
for (size_t i = 0; i < possible_info_to_send_to.size(); i += 2) {
const int i_rank = possible_info_to_send_to[i];
const int number = possible_info_to_send_to[i + 1];
possible_item_to_send_to_rank[i_rank].push_back(number);
}
}
const bool has_possible_item_to_send = [&] {
for (auto&& item_to_send_list : possible_item_to_send_to_rank) {
if (item_to_send_list.size() > 0) {
return true;
}
}
return false;
}();
std::unordered_map<int, ItemId> owned_number_to_item_id_map;
if (has_possible_item_to_send) {
for (ItemId item_id = 0; item_id < m_connectivity.template numberOf<item_type>(); ++item_id) {
if (item_is_owned[item_id]) {
owned_number_to_item_id_map[item_number[item_id]] = item_id;
}
}
}
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
tag.fill(false);
std::vector<ItemId> item_id_vector;
const auto& possible_item_to_send = possible_item_to_send_to_rank[i_rank];
for (auto&& possible_item_number : possible_item_to_send) {
const auto& searched_item_id = owned_number_to_item_id_map.find(possible_item_number);
Assert(searched_item_id != owned_number_to_item_id_map.end());
const ItemId item_id = searched_item_id->second;
if (not tag[item_id]) {
item_id_vector.push_back(item_id);
tag[item_id] = true;
}
}
for (size_t i_cell = 0; i_cell < cell_list_to_send_by_proc[i_rank].size(); ++i_cell) {
const CellId& cell_id = cell_list_to_send_by_proc[i_rank][i_cell];
const auto& cell_sub_item_list = cell_to_sub_item_matrix[cell_id];
for (size_t i_item = 0; i_item < cell_sub_item_list.size(); ++i_item) {
const ItemId& sub_item_id = cell_sub_item_list[i_item];
if (item_is_owned[sub_item_id] and (not tag[sub_item_id])) {
item_id_vector.push_back(sub_item_id);
tag[sub_item_id] = true;
}
}
}
item_list_to_send_by_proc[i_rank] = convert_to_array(item_id_vector);
}
......@@ -364,6 +476,8 @@ ConnectivityDispatcher<Dimension>::_buildItemToSubItemDescriptor()
const auto& item_list_to_recv_size_by_proc = this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc;
const auto& recv_id_correspondance_by_proc = this->_dispatchedInfo<item_type>().m_recv_id_correspondance_by_proc;
const auto& number_of_sub_item_per_item_to_recv_by_proc =
this->_dispatchedInfo<ItemOfItemT>().m_number_of_sub_item_per_item_to_recv_by_proc;
......@@ -373,7 +487,14 @@ ConnectivityDispatcher<Dimension>::_buildItemToSubItemDescriptor()
this->_dispatchedInfo<ItemOfItemT>().m_sub_item_numbers_to_recv_by_proc;
std::vector<std::vector<unsigned int>> item_to_subitem_legacy;
size_t number_of_node_by_cell = 0;
item_to_subitem_legacy.resize([&] {
size_t size = 0;
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
size += item_list_to_recv_size_by_proc[i_rank];
}
return size;
}());
size_t number_of_subitem_by_item = 0;
for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
int l = 0;
for (size_t i = 0; i < item_list_to_recv_size_by_proc[i_rank]; ++i) {
......@@ -383,14 +504,14 @@ ConnectivityDispatcher<Dimension>::_buildItemToSubItemDescriptor()
Assert(searched_sub_item_id != sub_item_number_id_map.end());
sub_item_vector.push_back(searched_sub_item_id->second);
}
number_of_node_by_cell += sub_item_vector.size();
number_of_subitem_by_item += sub_item_vector.size();
item_to_subitem_legacy.emplace_back(sub_item_vector);
item_to_subitem_legacy[recv_id_correspondance_by_proc[i_rank][i]] = std::move(sub_item_vector);
}
}
Array<unsigned int> item_to_subitem_row_map(item_to_subitem_legacy.size() + 1);
Array<unsigned int> item_to_subitem_list(number_of_node_by_cell);
Array<unsigned int> item_to_subitem_list(number_of_subitem_by_item);
item_to_subitem_row_map[0] = 0;
for (size_t i = 0; i < item_to_subitem_legacy.size(); ++i) {
......@@ -469,11 +590,6 @@ ConnectivityDispatcher<Dimension>::_buildItemReferenceList()
}();
if (number_of_item_list_sender > 0) {
if (number_of_item_list_sender > 1) {
std::cerr << __FILE__ << ':' << __LINE__ << ": " << rang::fgB::red
<< "need to check that knowing procs know the same item_ref_lists!" << rang::fg::reset << '\n';
}
if (number_of_item_list_sender < parallel::size()) {
const size_t sender_rank = [&]() {
size_t i_rank = 0;
......@@ -634,6 +750,10 @@ ConnectivityDispatcher<Dimension>::_dispatchEdges()
this->_buildSubItemNumberToIdMap<EdgeOfCell>();
this->_buildItemToExchangeLists<ItemType::edge>();
this->_buildNumberOfSubItemPerItemToRecvByProc<NodeOfEdge>();
this->_buildSubItemNumbersToRecvByProc<NodeOfEdge>();
this->_buildItemToSubItemDescriptor<NodeOfEdge>();
m_new_descriptor.setEdgeNumberVector([&] {
Array<int> edge_number_vector;
this->_gatherFrom(m_connectivity.template number<ItemType::edge>(), edge_number_vector);
......@@ -642,10 +762,6 @@ ConnectivityDispatcher<Dimension>::_dispatchEdges()
this->_buildItemToSubItemDescriptor<EdgeOfCell>();
this->_buildNumberOfSubItemPerItemToRecvByProc<NodeOfEdge>();
this->_buildSubItemNumbersToRecvByProc<NodeOfEdge>();
this->_buildItemToSubItemDescriptor<NodeOfEdge>();
this->_buildNumberOfSubItemPerItemToRecvByProc<EdgeOfFace>();
this->_buildSubItemNumbersToRecvByProc<EdgeOfFace>();
this->_buildItemToSubItemDescriptor<EdgeOfFace>();
......@@ -716,12 +832,15 @@ ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType
}
this->_buildNewOwner<ItemType::cell>();
if constexpr (Dimension > 1) {
this->_buildNewOwner<ItemType::face>();
}
if constexpr (Dimension > 2) {
this->_buildNewOwner<ItemType::edge>();
}
this->_buildNewOwner<ItemType::node>();
this->_buildItemToExchangeLists<ItemType::cell>();
......
src/mesh/MeshBuilderBase.cpp
+ 1
− 21
View file @ 7efc43fa
......@@ -42,27 +42,7 @@ MeshBuilderBase::_dispatch()
std::shared_ptr dispatched_connectivity = p_dispatcher->dispatchedConnectivity();
NodeValue<Rd> dispatched_xr = p_dispatcher->dispatch(mesh.xr());
std::cout << rang::fgB::magenta << "dispatched xr=" << dispatched_xr << rang::fg::reset << '\n';
auto dispatched_mesh = std::make_shared<const MeshType>(dispatched_connectivity, dispatched_xr);
{
const auto& d_connectivity = dispatched_mesh->connectivity();
auto cell_to_node_matrix = d_connectivity.cellToNodeMatrix();
auto d_xr = dispatched_mesh->xr();
auto d_cell_number = d_connectivity.cellNumber();
for (CellId cell_id = 0; cell_id < dispatched_mesh->numberOfCells(); ++cell_id) {
std::cout << cell_id << "(" << d_cell_number[cell_id] << "):\n";
const auto cell_nodes = cell_to_node_matrix[cell_id];
for (size_t i_node = 0; i_node < cell_nodes.size(); ++i_node) {
const NodeId node_id = cell_nodes[i_node];
std::cout << i_node << ": " << d_xr[node_id] << '\n';
}
}
}
m_mesh = std::make_shared<MeshVariant>(dispatched_mesh);
// m_mesh = std::make_shared<MeshVariant>(std::make_shared<const MeshType>(dispatched_connectivity, dispatched_xr));
m_mesh = std::make_shared<MeshVariant>(std::make_shared<const MeshType>(dispatched_connectivity, dispatched_xr));
}
template void MeshBuilderBase::_dispatch<1>();
......
src/utils/ParMETISPartitioner.cpp
+ 0
− 12
View file @ 7efc43fa
......@@ -77,12 +77,6 @@ ParMETISPartitioner::partition(const CRSGraph& graph)
int* entries_ptr = const_cast<int*>(&(entries[0]));
int* neighbors_ptr = const_cast<int*>(&(neighbors[0]));
if (group_size > 1) {
#warning remove
std::cout << parallel::rank() << ": entries " << graph.entries() << "\n";
std::cout << parallel::rank() << ": neighbors " << graph.neighbors() << "\n";
}
int result =
ParMETIS_V3_PartKway(&(vtxdist[0]), entries_ptr, neighbors_ptr, NULL, NULL, &wgtflag, &numflag, &ncon, &npart,
&(tpwgts[0]), &(ubvec[0]), &(options[0]), &edgecut, &(partition[0]), &partitioning_comm);
......@@ -90,12 +84,6 @@ ParMETISPartitioner::partition(const CRSGraph& graph)
if (result == METIS_ERROR) {
throw UnexpectedError("Metis Error");
}
if (group_size > 1) {
#warning remove
std::cout << parallel::rank() << ": partition " << partition << "\n";
}
// LCOV_EXCL_STOP
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment