From e17ef06f04cc28c99366a872299039c85e87f2a6 Mon Sep 17 00:00:00 2001
From: Stephane Del Pino <stephane.delpino44@gmail.com>
Date: Tue, 9 Oct 2018 16:19:13 +0200
Subject: [PATCH] Progress in mesh dispatching
Replace broadcast by exchange of pertinent cell ids
---
src/mesh/GmshReader.cpp | 118 +++++++++++-----------------------------
1 file changed, 32 insertions(+), 86 deletions(-)
diff --git a/src/mesh/GmshReader.cpp b/src/mesh/GmshReader.cpp
index 483e98c8e..c258855f3 100644
--- a/src/mesh/GmshReader.cpp
+++ b/src/mesh/GmshReader.cpp
@@ -159,11 +159,9 @@ void GmshReader::_dispatch()
CSRGraph mesh_graph = mesh.cellToCellGraph();
- CellValue<int> cell_parts(mesh.connectivity());
-
const int reader_rank
= [&]() {
- Array<int> parts = allGather(static_cast<int>(cell_parts.size()));
+ Array<int> parts = allGather(static_cast<int>(mesh.numberOfCells()));
std::set<int> part_set;
for (size_t i=0; i<parts.size(); ++i) {
if (parts[i] != 0) { part_set.insert(i); }
@@ -179,108 +177,56 @@ void GmshReader::_dispatch()
Partitioner P;
Array<int> cell_new_owner = P.partition(mesh_graph);
+ const std::vector<Array<const int>> cell_to_send_by_proc
+ = [&] () {
+ std::vector<std::vector<int>> cell_vector_to_send_by_proc(commSize());
+ for (size_t i=0; i<cell_new_owner.size(); ++i) {
+ cell_vector_to_send_by_proc[cell_new_owner[i]].push_back(i);
+ }
+
+ std::vector<Array<const int>> cell_to_send_by_proc(commSize());
+ for (size_t i=0; i<commSize(); ++i) {
+ cell_to_send_by_proc[i] = convert_to_array(cell_vector_to_send_by_proc[i]);
+ }
+
+ return std::move(cell_to_send_by_proc);
+ } ();
+
+
Array<int> nb_cell_to_send_by_proc(commSize());
- for (size_t i=0; i<cell_new_owner.size(); ++i) {
- nb_cell_to_send_by_proc[cell_new_owner[i]]++;
+ for (size_t i=0; i<commSize(); ++i) {
+ nb_cell_to_send_by_proc[i] = cell_to_send_by_proc[i].size();
}
- // for (int i_rank=0; i_rank<commSize(); ++i_rank) {
- // if (commRank() == i_rank) {
- // for (size_t i_recv=0; i_recv<commSize(); ++i_recv) {
- // std::cout << commRank()
- // << " gives " << nb_cell_to_send_by_proc[i_recv]
- // << " cells to " << i_recv << '\n' << std::flush;
- // }
- // }
- // barrier();
- // }
Array<int> nb_cell_to_recv_by_proc = allToAll(nb_cell_to_send_by_proc);
- // for (int i_rank=0; i_rank<commSize(); ++i_rank) {
- // if (commRank() == i_rank) {
- // for (size_t i_send=0; i_send<commSize(); ++i_send) {
- // std::cout << commRank()
- // << " recv " << nb_cell_to_recv_by_proc[i_send]
- // << " cells of " << i_send << '\n';
- // }
- // }
- // barrier();
- // }
const size_t cell_number = mesh.numberOfCells();
const size_t new_cell_number
= cell_number
+ Sum(nb_cell_to_recv_by_proc)
- Sum(nb_cell_to_send_by_proc);
- std::cout << commRank() << ": " << cell_number << " -> " << new_cell_number << " cells\n";
-
- // now share owner table
- cell_new_owner = broadcast(cell_new_owner, reader_rank);
-
- Array<int> cell_new_owner_nb_cell(commSize());
- cell_new_owner_nb_cell.fill(0);
- for (size_t i=0; i<cell_new_owner.size(); ++i) {
- cell_new_owner_nb_cell[cell_new_owner[i]]++;
+ std::vector<Array<int>> recv_cell_by_proc(commSize());
+ for (size_t i=0; i<commSize(); ++i) {
+ recv_cell_by_proc[i] = Array<int>(nb_cell_to_recv_by_proc[i]);
}
+ exchange(cell_to_send_by_proc, recv_cell_by_proc);
- Array<int> node_new_owner(mesh.numberOfNodes());
- if (commRank() == reader_rank) {
- const auto& node_to_cell_matrix = mesh.connectivity().nodeToCellMatrix();
- parallel_for(mesh.numberOfNodes(), PASTIS_LAMBDA(const NodeId& r) {
- const auto& node_cells = node_to_cell_matrix[r];
- int new_owner = cell_new_owner[node_cells[0]];
- for (size_t j=1; j<node_cells.size(); ++j) {
- const int cell_owner = cell_new_owner[node_cells[j]];
- if (new_owner < cell_owner) {
- new_owner = cell_owner;
- }
- }
- node_new_owner[r] = new_owner;
- });
- }
+ Array<int> new_cell_id(new_cell_number);
- node_new_owner = broadcast(node_new_owner, reader_rank);
-
- for (int i_rank=0; i_rank<commSize(); ++i_rank) {
- if (commRank() == i_rank) {
- std::cout << i_rank << " cells -> ";
- for (size_t i=0; i<cell_new_owner.size(); ++i) {
- if (commRank() == cell_new_owner[i]) {
- std::cout << i << ' ';
- }
- }
- std::cout << " [" << cell_new_owner_nb_cell[i_rank] << "]\n" << std::flush;
- }
- // barrier();
- }
-
- for (int i_rank=0; i_rank<commSize(); ++i_rank) {
+ for (int i_rank=0; i_rank < commSize(); ++i_rank) {
if (commRank() == i_rank) {
- size_t cpt=0;
- std::cout << i_rank << " nodes -> ";
- for (size_t i=0; i<node_new_owner.size(); ++i) {
- if (commRank() == node_new_owner[i]) {
- std::cout << i << ' ';
- cpt++;
+ std::cout << "----- rank=" << i_rank << " -----\n";
+ for (int j_rank=0; j_rank < commSize(); ++j_rank) {
+ std::cout << "recv from " << j_rank << ':';
+ for (size_t i=0; i<recv_cell_by_proc[j_rank].size(); ++i) {
+ std::cout << ' ' << recv_cell_by_proc[j_rank][i];
}
+ std::cout << '\n' << std::flush;
}
- std::cout << " [" << cpt << "]\n" << std::flush;
- }
- // barrier();
- }
-
- for (int i_rank=0; i_rank<commSize(); ++i_rank) {
- if (commRank() == i_rank) {
- const auto cell_type
- = mesh.connectivity().cellType();
- std::cout << i_rank << " cell_type -> ";
- for (CellId j=0; j<cell_type.size(); ++j) {
- std::cout << static_cast<int>(cell_type[j]) << ' ';
- }
- std::cout << '\n';
}
- // barrier();
+ barrier();
}
Messenger::destroy();
--
GitLab