Skip to content
Snippets Groups Projects
Commit dfec066e authored by Stéphane Del Pino's avatar Stéphane Del Pino
Browse files

Continue clean-up

- mostly code displacement
- still need to reorganize face dispatcher before completing clean-up and
  finalizing dispatcher code
parent d0331b27
No related branches found
No related tags found
1 merge request!11Feature/mpi
......@@ -107,11 +107,39 @@ ConnectivityDispatcher<Dimension>::_buildCellListToSend() const
return cell_list_to_send_by_proc;
}
template <int Dimension>
Array<int>
std::vector<Array<const NodeId>>
ConnectivityDispatcher<Dimension>::
_buildNodeListToSend() const
{
const auto& cell_to_node_matrix = m_connectivity.cellToNodeMatrix();
std::vector<Array<const NodeId>> node_list_to_send_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
Array<bool> tag(m_connectivity.numberOfNodes());
tag.fill(false);
std::vector<NodeId> node_id_vector;
for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) {
const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j];
const auto& cell_node_list = cell_to_node_matrix[cell_id];
for (size_t r=0; r<cell_node_list.size(); ++r) {
const NodeId& node_id = cell_node_list[r];
if (not tag[node_id]) {
node_id_vector.push_back(node_id);
tag[node_id] = true;
}
}
}
node_list_to_send_by_proc[i_rank] = convert_to_array(node_id_vector);
}
return node_list_to_send_by_proc;
}
template <int Dimension>
Array<const unsigned int>
ConnectivityDispatcher<Dimension>::_buildNbCellToSend()
{
Array<int> nb_cell_to_send_by_proc(parallel::size());
Array<unsigned int> nb_cell_to_send_by_proc(parallel::size());
for (size_t i=0; i<parallel::size(); ++i) {
nb_cell_to_send_by_proc[i] = m_cell_list_to_send_by_proc[i].size();
}
......@@ -123,7 +151,7 @@ void
ConnectivityDispatcher<Dimension>::_dispatchFaces()
{
if constexpr (Dimension>1) {
std::vector<Array<int>> recv_number_of_face_per_cell_by_proc =
std::vector<Array<const int>> recv_number_of_face_per_cell_by_proc =
[&] () {
CellValue<int> number_of_face_per_cell(m_connectivity);
const auto& cell_to_face_matrix = m_connectivity.cellToFaceMatrix();
......@@ -185,7 +213,7 @@ ConnectivityDispatcher<Dimension>::_dispatchFaces()
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
int l=0;
for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) {
for (size_t i=0; i<m_nb_cell_to_recv_by_proc[i_rank]; ++i) {
std::vector<unsigned int> face_vector;
for (int k=0; k<recv_number_of_face_per_cell_by_proc[i_rank][i]; ++k) {
const auto& searched_face_id = face_number_id_map.find(recv_cell_face_number_by_proc[i_rank][l++]);
......@@ -223,7 +251,7 @@ ConnectivityDispatcher<Dimension>::_dispatchFaces()
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
int l=0;
for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) {
for (size_t i=0; i<m_nb_cell_to_recv_by_proc[i_rank]; ++i) {
std::vector<bool> face_is_reversed_vector;
for (int k=0; k<recv_number_of_face_per_cell_by_proc[i_rank][i]; ++k) {
face_is_reversed_vector.push_back(recv_cell_face_is_reversed_by_proc[i_rank][l++]);
......@@ -552,88 +580,31 @@ ConnectivityDispatcher<Dimension>::_dispatchFaces()
template <int Dimension>
ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType& connectivity)
: m_connectivity(connectivity),
m_cell_new_owner(_getCellNewOwner()),
m_face_new_owner(_getFaceNewOwner()),
m_node_new_owner(_getNodeNewOwner()),
m_cell_list_to_send_by_proc(_buildCellListToSend()),
m_nb_cell_to_send_by_proc(_buildNbCellToSend()),
m_nb_cell_to_recv_by_proc(parallel::allToAll(m_nb_cell_to_send_by_proc))
template <typename CellValueType,
typename DataType>
void
ConnectivityDispatcher<Dimension>::_gatherFrom(const CellValueType& data_to_gather,
std::vector<DataType>& gathered_vector)
{
m_recv_cell_number_by_proc = this->exchange(m_connectivity.cellNumber());
static_assert(std::is_same_v<std::remove_const_t<typename CellValueType::data_type>, DataType>);
const auto& cell_to_node_matrix = m_connectivity.cellToNodeMatrix();
std::vector<Array<const DataType>> recv_cell_data_by_proc = this->exchange(data_to_gather);
std::vector<Array<int>> cell_node_number_to_send_by_proc =
[&] () {
const NodeValue<const int>& node_number = m_connectivity.nodeNumber();
std::vector<Array<int>> cell_node_number_to_send_by_proc(parallel::size());
gathered_vector.resize(m_cell_number_id_map.size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
std::vector<int> node_number_by_cell_vector;
for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) {
const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j];
const auto& cell_node_list = cell_to_node_matrix[cell_id];
for (size_t r=0; r<cell_node_list.size(); ++r) {
const NodeId& node_id = cell_node_list[r];
node_number_by_cell_vector.push_back(node_number[node_id]);
}
}
cell_node_number_to_send_by_proc[i_rank] = convert_to_array(node_number_by_cell_vector);
}
return cell_node_number_to_send_by_proc;
} ();
std::vector<Array<int>> recv_number_of_node_per_cell_by_proc
= [&] () {
CellValue<int> number_of_node_per_cell(m_connectivity);
parallel_for(m_connectivity.numberOfCells(), PASTIS_LAMBDA(const CellId& j){
number_of_node_per_cell[j] = cell_to_node_matrix[j].size();
});
return this->exchange(number_of_node_per_cell);
} ();
std::vector<Array<int>> recv_cell_node_number_by_proc(parallel::size());
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
recv_cell_node_number_by_proc[i_rank]
= Array<int>(sum(recv_number_of_node_per_cell_by_proc[i_rank]));
}
parallel::exchange(cell_node_number_to_send_by_proc, recv_cell_node_number_by_proc);
m_node_number_id_map =
[&] () {
std::unordered_map<int, int> node_number_id_map;
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
int cpt=0;
for (size_t i=0; i<recv_cell_node_number_by_proc[i_rank].size(); ++i) {
int node_number = recv_cell_node_number_by_proc[i_rank][i];
auto [iterator, inserted] = node_number_id_map.insert(std::make_pair(node_number, cpt));
if (inserted) cpt++;
}
}
return node_number_id_map;
} ();
m_new_descriptor.node_number_vector.resize(m_node_number_id_map.size());
for (const auto& [number, id] : m_node_number_id_map) {
m_new_descriptor.node_number_vector[id] = number;
}
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
int l=0;
for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) {
std::vector<unsigned int> node_vector;
for (int k=0; k<recv_number_of_node_per_cell_by_proc[i_rank][i]; ++k) {
const auto& searched_node_id = m_node_number_id_map.find(recv_cell_node_number_by_proc[i_rank][l++]);
Assert(searched_node_id != m_node_number_id_map.end());
node_vector.push_back(searched_node_id->second);
int cell_number = m_recv_cell_number_by_proc[i_rank][i];
const auto& searched_cell_id = m_cell_number_id_map.find(cell_number);
Assert(searched_cell_id != m_cell_number_id_map.end());
gathered_vector[searched_cell_id->second] = recv_cell_data_by_proc[i_rank][i];
}
m_new_descriptor.cell_by_node_vector.emplace_back(node_vector);
}
}
const std::unordered_map<int, int> cell_number_id_map
= [&] () {
template <int Dimension>
std::unordered_map<int, int>
ConnectivityDispatcher<Dimension>::_buildCellNumberIdMap()
{
std::unordered_map<int, int> cell_number_id_map;
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
int cpt=0;
......@@ -644,58 +615,81 @@ ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType
}
}
return cell_number_id_map;
} ();
m_new_descriptor.cell_number_vector.resize(cell_number_id_map.size());
for (const auto& [number, id] : cell_number_id_map) {
m_new_descriptor.cell_number_vector[id] = number;
}
template <int Dimension>
std::vector<Array<const int>>
ConnectivityDispatcher<Dimension>::_getRecvNumberOfNodePerCellByProc() const
{
std::vector<Array<CellType>> recv_cell_type_by_proc
= this->exchange(m_connectivity.cellType());
m_new_descriptor.cell_type_vector.resize(cell_number_id_map.size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) {
int cell_number = m_recv_cell_number_by_proc[i_rank][i];
const auto& searched_cell_id = cell_number_id_map.find(cell_number);
Assert(searched_cell_id != cell_number_id_map.end());
m_new_descriptor.cell_type_vector[searched_cell_id->second] = recv_cell_type_by_proc[i_rank][i];
}
}
const auto& cell_to_node_matrix = m_connectivity.cellToNodeMatrix();
CellValue<int> number_of_node_per_cell(m_connectivity);
parallel_for(m_connectivity.numberOfCells(), PASTIS_LAMBDA(const CellId& j){
number_of_node_per_cell[j] = cell_to_node_matrix[j].size();
});
return this->exchange(number_of_node_per_cell);
}
m_node_list_to_send_by_proc =
template <int Dimension>
std::vector<Array<const int>>
ConnectivityDispatcher<Dimension>::
_getRecvCellNodeNumberByProc(const std::vector<Array<const int>>& recv_number_of_node_per_cell_by_proc) const
{
std::vector<Array<const int>> cell_node_number_to_send_by_proc =
[&] () {
std::vector<Array<const NodeId>> node_list_to_send_by_proc(parallel::size());
const auto& cell_to_node_matrix = m_connectivity.cellToNodeMatrix();
const NodeValue<const int>& node_number = m_connectivity.nodeNumber();
std::vector<Array<const int>> cell_node_number_to_send_by_proc(parallel::size());
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
Array<bool> tag(m_connectivity.numberOfNodes());
tag.fill(false);
std::vector<NodeId> node_id_vector;
std::vector<int> node_number_by_cell_vector;
for (size_t j=0; j<m_cell_list_to_send_by_proc[i_rank].size(); ++j) {
const CellId& cell_id = m_cell_list_to_send_by_proc[i_rank][j];
const auto& cell_node_list = cell_to_node_matrix[cell_id];
for (size_t r=0; r<cell_node_list.size(); ++r) {
const NodeId& node_id = cell_node_list[r];
if (not tag[node_id]) {
node_id_vector.push_back(node_id);
tag[node_id] = true;
}
node_number_by_cell_vector.push_back(node_number[node_id]);
}
}
node_list_to_send_by_proc[i_rank] = convert_to_array(node_id_vector);
cell_node_number_to_send_by_proc[i_rank] = convert_to_array(node_number_by_cell_vector);
}
return node_list_to_send_by_proc;
return cell_node_number_to_send_by_proc;
} ();
Array<unsigned int> nb_node_to_send_by_proc(parallel::size());
std::vector<Array<int>> recv_cell_node_number_by_proc(parallel::size());
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
nb_node_to_send_by_proc[i_rank] = m_node_list_to_send_by_proc[i_rank].size();
recv_cell_node_number_by_proc[i_rank]
= Array<int>(sum(recv_number_of_node_per_cell_by_proc[i_rank]));
}
m_nb_node_to_recv_by_proc = parallel::allToAll(nb_node_to_send_by_proc);
parallel::exchange(cell_node_number_to_send_by_proc, recv_cell_node_number_by_proc);
m_recv_node_id_correspondance_by_proc =
[&] () {
std::vector<Array<const int>> const_recv_cell_node_number_by_proc(parallel::size());
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
const_recv_cell_node_number_by_proc[i_rank] = recv_cell_node_number_by_proc[i_rank];
}
return const_recv_cell_node_number_by_proc;
}
template <int Dimension>
std::unordered_map<int, int>
ConnectivityDispatcher<Dimension>::
_buildNodeNumberIdMap(const std::vector<Array<const int>>& recv_cell_node_number_by_proc)
{
std::unordered_map<int, int> node_number_id_map;
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
int node_id=0;
for (size_t i=0; i<recv_cell_node_number_by_proc[i_rank].size(); ++i) {
int node_number = recv_cell_node_number_by_proc[i_rank][i];
auto [iterator, inserted] = node_number_id_map.insert(std::make_pair(node_number, node_id));
if (inserted) node_id++;
}
}
return node_number_id_map;
}
template <int Dimension>
std::vector<Array<const NodeId>>
ConnectivityDispatcher<Dimension>::
_buildRecvNodeIdCorrespondanceByProc() const
{
std::vector<Array<const NodeId>> recv_node_id_correspondance_by_proc(parallel::size());
const NodeValue<const int>& node_number = m_connectivity.nodeNumber();
std::vector<Array<const int>> send_node_number_by_proc(parallel::size());
......@@ -725,20 +719,50 @@ ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType
recv_node_id_correspondance_by_proc[i_rank] = node_id_correspondace;
}
return recv_node_id_correspondance_by_proc;
} ();
}
template <int Dimension>
ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType& connectivity)
: m_connectivity(connectivity),
m_cell_new_owner(this->_getCellNewOwner()),
m_face_new_owner(this->_getFaceNewOwner()),
m_node_new_owner(this->_getNodeNewOwner()),
m_cell_list_to_send_by_proc(this->_buildCellListToSend()),
m_nb_cell_to_recv_by_proc(parallel::allToAll(this->_buildNbCellToSend())),
m_recv_cell_number_by_proc(this->exchange(m_connectivity.cellNumber())),
m_cell_number_id_map(this->_buildCellNumberIdMap())
{
std::vector<Array<int>> recv_cell_new_owner_by_proc
= this->exchange(m_cell_new_owner);
m_new_descriptor.cell_owner_vector.resize(cell_number_id_map.size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
for (size_t i=0; i<m_recv_cell_number_by_proc[i_rank].size(); ++i) {
int cell_number = m_recv_cell_number_by_proc[i_rank][i];
const auto& searched_cell_id = cell_number_id_map.find(cell_number);
Assert(searched_cell_id != cell_number_id_map.end());
m_new_descriptor.cell_owner_vector[searched_cell_id->second] = recv_cell_new_owner_by_proc[i_rank][i];
const std::vector<Array<const int>> recv_number_of_node_per_cell_by_proc
= this->_getRecvNumberOfNodePerCellByProc();
const std::vector<Array<const int>> recv_cell_node_number_by_proc =
this->_getRecvCellNodeNumberByProc(recv_number_of_node_per_cell_by_proc);
this->_buildNodeNumberIdMap(recv_cell_node_number_by_proc);
m_node_number_id_map = _buildNodeNumberIdMap(recv_cell_node_number_by_proc);
m_new_descriptor.cell_number_vector.resize(m_cell_number_id_map.size());
for (const auto& [number, id] : m_cell_number_id_map) {
m_new_descriptor.cell_number_vector[id] = number;
}
m_node_list_to_send_by_proc = this->_buildNodeListToSend();
Array<unsigned int> nb_node_to_send_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
nb_node_to_send_by_proc[i_rank] = m_node_list_to_send_by_proc[i_rank].size();
}
m_nb_node_to_recv_by_proc = parallel::allToAll(nb_node_to_send_by_proc);
m_recv_node_id_correspondance_by_proc = this->_buildRecvNodeIdCorrespondanceByProc();
// Fill new descriptor
this->_gatherFrom(m_connectivity.cellType(), m_new_descriptor.cell_type_vector);
this->_gatherFrom(m_cell_new_owner, m_new_descriptor.cell_owner_vector);
m_new_descriptor.node_number_vector.resize(m_node_number_id_map.size());
for (const auto& [number, id] : m_node_number_id_map) {
m_new_descriptor.node_number_vector[id] = number;
}
{
......@@ -753,6 +777,7 @@ ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType
send_node_owner_by_proc[i_rank] = send_node_owner;
}
#warning use a _gatherFrom-like function
std::vector<Array<int>> recv_node_owner_by_proc(parallel::size());
for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
recv_node_owner_by_proc[i_rank] = Array<int>(m_nb_node_to_recv_by_proc[i_rank]);
......@@ -768,6 +793,20 @@ ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType
}
}
for (size_t i_rank=0; i_rank < parallel::size(); ++i_rank) {
int l=0;
for (size_t i=0; i<m_nb_cell_to_recv_by_proc[i_rank]; ++i) {
std::vector<unsigned int> node_vector;
for (int k=0; k<recv_number_of_node_per_cell_by_proc[i_rank][i]; ++k) {
const auto& searched_node_id = m_node_number_id_map.find(recv_cell_node_number_by_proc[i_rank][l++]);
Assert(searched_node_id != m_node_number_id_map.end());
node_vector.push_back(searched_node_id->second);
}
m_new_descriptor.cell_by_node_vector.emplace_back(node_vector);
}
}
this->_dispatchFaces();
m_dispatched_connectivity = ConnectivityType::build(m_new_descriptor);
......
......@@ -20,33 +20,56 @@ class ConnectivityDispatcher
std::shared_ptr<ConnectivityType> m_dispatched_connectivity;
CellValue<const int> m_cell_new_owner;
FaceValue<const int> m_face_new_owner;
NodeValue<const int> m_node_new_owner;
const CellValue<const int> m_cell_new_owner;
const FaceValue<const int> m_face_new_owner;
const NodeValue<const int> m_node_new_owner;
const std::vector<Array<const CellId>> m_cell_list_to_send_by_proc;
#warning use a const sementic too
std::vector<Array<const NodeId>> m_node_list_to_send_by_proc;
Array<const int> m_nb_cell_to_send_by_proc;
Array<const int> m_nb_cell_to_recv_by_proc;
const Array<const unsigned int> m_nb_cell_to_recv_by_proc;
const std::vector<Array<const int>> m_recv_cell_number_by_proc;
const std::unordered_map<int, int> m_cell_number_id_map;
Array<const unsigned int> m_nb_node_to_recv_by_proc;
std::vector<Array<const NodeId>> m_recv_node_id_correspondance_by_proc;
std::vector<Array<int>> m_recv_cell_number_by_proc;
std::unordered_map<int, int> m_node_number_id_map;
std::vector<Array<const NodeId>> m_recv_node_id_correspondance_by_proc;
CellValue<int> _getCellNewOwner();
FaceValue<int> _getFaceNewOwner();
NodeValue<int> _getNodeNewOwner();
std::vector<Array<const CellId>> _buildCellListToSend() const;
Array<int> _buildNbCellToSend();
std::vector<Array<const NodeId>> _buildNodeListToSend() const;
Array<const unsigned int> _buildNbCellToSend();
std::unordered_map<int, int>
_buildCellNumberIdMap();
std::unordered_map<int, int>
_buildNodeNumberIdMap(const std::vector<Array<const int>>& recv_cell_node_number_by_proc);
void _dispatchFaces();
template <typename CellValueType,
typename DataType>
void _gatherFrom(const CellValueType& data_to_gather,
std::vector<DataType>& gathered_vector);
std::vector<Array<const int>>
_getRecvNumberOfNodePerCellByProc() const;
std::vector<Array<const int>>
_getRecvCellNodeNumberByProc(const std::vector<Array<const int>>& recv_number_of_node_per_cell_by_proc) const;
std::vector<Array<const NodeId>>
_buildRecvNodeIdCorrespondanceByProc() const;
public:
std::shared_ptr<const ConnectivityType>
dispatchedConnectivity() const
......@@ -73,11 +96,11 @@ class ConnectivityDispatcher
}
template<typename DataType, typename ConnectivityPtr>
std::vector<Array<std::remove_const_t<DataType>>>
std::vector<Array<const DataType>>
exchange(ItemValue<DataType, ItemType::cell, ConnectivityPtr> cell_value) const
{
using MutableDataType = std::remove_const_t<DataType>;
std::vector<Array<DataType>> cell_value_to_send_by_proc(parallel::size());
std::vector<Array<const DataType>> cell_value_to_send_by_proc(parallel::size());
for (size_t i=0; i<parallel::size(); ++i) {
const Array<const CellId>& cell_list = m_cell_list_to_send_by_proc[i];
Array<MutableDataType> cell_value_list(cell_list.size());
......@@ -93,12 +116,18 @@ class ConnectivityDispatcher
}
parallel::exchange(cell_value_to_send_by_proc, recv_cell_value_by_proc);
return recv_cell_value_by_proc;
std::vector<Array<const DataType>> const_recv_cell_value_by_proc(parallel::size());
for (size_t i=0; i<parallel::size(); ++i) {
const_recv_cell_value_by_proc[i] = recv_cell_value_by_proc[i];
}
return const_recv_cell_value_by_proc;
}
#warning Should write this function generically w.r. to the item type
template<typename DataType, typename ConnectivityPtr>
ItemValue<DataType, ItemType::node, ConnectivityPtr>
ItemValue<std::remove_const_t<DataType>, ItemType::node, ConnectivityPtr>
dispatch(ItemValue<DataType, ItemType::node, ConnectivityPtr> node_value) const
{
Assert(m_dispatched_connectivity.use_count()> 0,
......
......@@ -149,7 +149,7 @@ void GmshReader::_dispatch()
ConnectivityDispatcher<Dimension> dispatcher(mesh.connectivity());
std::shared_ptr dispatched_connectivity = dispatcher.dispatchedConnectivity();
NodeValue<Rd> dispatched_xr = dispatcher.dispatch(mesh.mutableXr());
NodeValue<Rd> dispatched_xr = dispatcher.dispatch(mesh.xr());
m_mesh = std::make_shared<MeshType>(dispatched_connectivity, dispatched_xr);
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment