Skip to content
Snippets Groups Projects
Commit f8e3832e authored by Stéphane Del Pino's avatar Stéphane Del Pino
Browse files

Improve mesh dispatching in parallel

- connectivity is spread and node coordinates are shared
- it remains to
  - eventually share faces and edges
  - dispatch entity groups
  - set owners of entities
parent e91001ec
No related branches found
No related tags found
1 merge request!11Feature/mpi
......@@ -249,11 +249,6 @@ void GmshReader::_dispatch()
MeshDispatcher<Dimension> dispatcher(mesh);
// MeshData data(mesh);
// data.updateAllData();
// std::vector<Array<Rd>> recv_cell_center_by_proc
// = dispatcher.exchange(data.xj());
std::vector<Array<int>> recv_cell_number_by_proc
= dispatcher.exchange(mesh.connectivity().cellNumber());
......@@ -418,11 +413,82 @@ void GmshReader::_dispatch()
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
for (size_t i=0; i<recv_cell_number_by_proc[i_rank].size(); ++i) {
int cell_number = recv_cell_number_by_proc[i_rank][i];
const auto& searched_node_id = cell_number_id_map.find(cell_number);
const auto& searched_cell_id = cell_number_id_map.find(cell_number);
Assert(searched_cell_id != cell_number_id_map.end());
new_cell_type[searched_cell_id->second] = recv_cell_type_by_proc[i_rank][i];
}
}
std::vector<Array<const NodeId>> send_node_id_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
Array<bool> tag(mesh.numberOfNodes());
tag.fill(false);
std::vector<NodeId> node_id_vector;
for (size_t j=0; j<cell_list_to_send_by_proc[i_rank].size(); ++j) {
const CellId& cell_id = cell_list_to_send_by_proc[i_rank][j];
const auto& cell_node_list = cell_to_node_matrix[cell_id];
for (size_t r=0; r<cell_node_list.size(); ++r) {
const NodeId& node_id = cell_node_list[r];
if (not tag[node_id]) {
node_id_vector.push_back(node_id);
tag[node_id] = true;
}
}
}
send_node_id_by_proc[i_rank] = convert_to_array(node_id_vector);
}
std::vector<Array<const int>> send_node_number_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
Array<int> send_node_number(send_node_id_by_proc[i_rank].size());
const Array<const NodeId> send_node_id = send_node_id_by_proc[i_rank];
parallel_for(send_node_number.size(), PASTIS_LAMBDA(const size_t& j){
send_node_number[j] = node_number[send_node_id[j]];
});
send_node_number_by_proc[i_rank] = send_node_number;
}
Array<unsigned int> nb_node_to_send_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
nb_node_to_send_by_proc[i_rank] = send_node_id_by_proc[i_rank].size();
}
Array<const unsigned int> nb_node_to_recv_by_proc
= parallel::allToAll(nb_node_to_send_by_proc);
std::vector<Array<int>> recv_node_number_by_proc(parallel::size());
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
recv_node_number_by_proc[i_rank] = Array<int>(nb_node_to_recv_by_proc[i_rank]);
}
parallel::exchange(send_node_number_by_proc, recv_node_number_by_proc);
std::vector<Array<const NodeId>> recv_node_id_correspondance_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<nb_node_to_recv_by_proc.size(); ++i_rank) {
Array<NodeId> node_id_correspondace(nb_node_to_recv_by_proc[i_rank]);
for (size_t l=0; l<nb_node_to_recv_by_proc[i_rank]; ++l) {
const int& node_number = recv_node_number_by_proc[i_rank][l];
const auto& searched_node_id = node_number_id_map.find(node_number);
Assert(searched_node_id != node_number_id_map.end());
new_cell_type[searched_node_id->second] = recv_cell_type_by_proc[i_rank][i];
node_id_correspondace[l] = searched_node_id->second;
}
recv_node_id_correspondance_by_proc[i_rank] = node_id_correspondace;
}
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
if (parallel::rank() == i_rank) {
std::cout << "----- rank=" << i_rank << " -----\n";
std::cout << "nodes to receive from procs:\n";
for (size_t j_rank=0; j_rank<nb_node_to_recv_by_proc.size(); ++j_rank) {
std::cout << j_rank << " -> (" << nb_node_to_recv_by_proc[j_rank] << ")";
for (size_t l=0; l<nb_node_to_recv_by_proc[j_rank]; ++l) {
const int& node_number = recv_node_number_by_proc[j_rank][l];
std::cout << ' ' << rang::style::bold << node_number << rang::style::reset;
std::cout << "[" << recv_node_id_correspondance_by_proc[j_rank][l] << "]";
}
std::cout << '\n';
}
}
parallel::barrier();
}
using ConnectivityType = Connectivity<Dimension>;
......@@ -432,8 +498,33 @@ void GmshReader::_dispatch()
new_cell_number,
new_node_number);
parallel::Messenger::destroy();
std::exit(0);
const NodeValue<const Rd>& xr = mesh.xr();
std::vector<Array<const Rd>> send_node_coord_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
Array<Rd> send_node_coord(nb_node_to_send_by_proc[i_rank]);
const Array<const NodeId> send_node_id = send_node_id_by_proc[i_rank];
parallel_for(send_node_id.size(), PASTIS_LAMBDA(const size_t& r) {
const NodeId& node_id = send_node_id[r];
send_node_coord[r] = xr[node_id];
});
send_node_coord_by_proc[i_rank] = send_node_coord;
}
std::vector<Array<Rd>> recv_node_coord_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
recv_node_coord_by_proc[i_rank] = Array<Rd>(nb_node_to_recv_by_proc[i_rank]);
}
parallel::exchange(send_node_coord_by_proc, recv_node_coord_by_proc);
NodeValue<Rd> new_xr(*p_connectivity);
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
for (size_t r=0; r<recv_node_coord_by_proc[i_rank].size(); ++r) {
const NodeId& node_id = recv_node_id_correspondance_by_proc[i_rank][r];
new_xr[node_id] = recv_node_coord_by_proc[i_rank][r];
}
}
m_mesh = std::shared_ptr<IMesh>(new MeshType(p_connectivity, new_xr));
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment