diff --git a/src/mesh/GmshReader.cpp b/src/mesh/GmshReader.cpp index babd97e8cf21ba09ff87bb98448adf45b4960b2f..d45e6c90b844082fef7708ba55924be2f599e3e2 100644 --- a/src/mesh/GmshReader.cpp +++ b/src/mesh/GmshReader.cpp @@ -143,7 +143,7 @@ ErrorHandler(const std::string& filename, GmshReader::GmshReader(const std::string& filename) : m_filename(filename) { - if (commRank() == 0) { + if (commRank() == 1) { try { m_fin.open(m_filename); if (not m_fin) { @@ -307,29 +307,49 @@ GmshReader::GmshReader(const std::string& filename) if (commSize() > 1) { pout() << "Sequential mesh read! Need to be dispatched\n" << std::flush; - CSRGraph mesh_graph; - CellValue<int> cell_parts; - if (commRank() == 0) { - mesh_graph = m_mesh->cellToCellGraph(); - - switch(m_mesh->meshDimension()) - { + const int mesh_dimension + = [&]() { + int mesh_dimension = -1; // unknown mesh dimension + if (m_mesh) { + mesh_dimension = m_mesh->meshDimension(); + } + + Array<int> dimensions = allGather(mesh_dimension); + std::set<int> dimension_set; + for (size_t i=0; i<dimensions.size(); ++i) { + const int i_dimension = dimensions[i]; + if (i_dimension != -1) { + dimension_set.insert(i_dimension); + } + } + if (dimension_set.size() != 1) { + std::cerr << "error dimensions of read mesh parts differ!\n"; + std::exit(1); + } + + return *begin(dimension_set); + }(); + if (not m_mesh) { + switch(mesh_dimension) { case 1: { - Mesh<Connectivity1D>& mesh = dynamic_cast<Mesh<Connectivity1D>&>(*m_mesh); - CellValue<int> mesh_cell_parts(mesh.connectivity()); - cell_parts = mesh_cell_parts; + std::shared_ptr<Connectivity1D> connectivity(new Connectivity1D({},{})); + using Rd = TinyVector<1>; + NodeValue<Rd> xr; + m_mesh = std::shared_ptr<IMesh>(new Mesh<Connectivity1D>(connectivity, xr)); break; } case 2: { - Mesh<Connectivity2D>& mesh = dynamic_cast<Mesh<Connectivity2D>&>(*m_mesh); - CellValue<int> mesh_cell_parts(mesh.connectivity()); - cell_parts = mesh_cell_parts; + std::shared_ptr<Connectivity2D> connectivity(new Connectivity2D({},{})); + using Rd = TinyVector<2>; + NodeValue<Rd> xr; + m_mesh = std::shared_ptr<IMesh>(new Mesh<Connectivity2D>(connectivity, xr)); break; } case 3: { - Mesh<Connectivity3D>& mesh = dynamic_cast<Mesh<Connectivity3D>&>(*m_mesh); - CellValue<int> mesh_cell_parts(mesh.connectivity()); - cell_parts = mesh_cell_parts; + std::shared_ptr<Connectivity3D> connectivity(new Connectivity3D({},{})); + using Rd = TinyVector<3>; + NodeValue<Rd> xr; + m_mesh = std::shared_ptr<IMesh>(new Mesh<Connectivity3D>(connectivity, xr)); break; } default:{ @@ -339,9 +359,56 @@ GmshReader::GmshReader(const std::string& filename) } } } + CSRGraph mesh_graph; + mesh_graph = m_mesh->cellToCellGraph(); + + CellValue<int> cell_parts; + switch(m_mesh->meshDimension()) + { + case 1: { + Mesh<Connectivity1D>& mesh = dynamic_cast<Mesh<Connectivity1D>&>(*m_mesh); + CellValue<int> mesh_cell_parts(mesh.connectivity()); + cell_parts = mesh_cell_parts; + break; + } + case 2: { + Mesh<Connectivity2D>& mesh = dynamic_cast<Mesh<Connectivity2D>&>(*m_mesh); + CellValue<int> mesh_cell_parts(mesh.connectivity()); + cell_parts = mesh_cell_parts; + break; + } + case 3: { + Mesh<Connectivity3D>& mesh = dynamic_cast<Mesh<Connectivity3D>&>(*m_mesh); + CellValue<int> mesh_cell_parts(mesh.connectivity()); + cell_parts = mesh_cell_parts; + break; + } + default:{ + perr() << "unexpected mesh dimension\n"; + Messenger::destroy(); + std::exit(1); + } + } + + const int reader_rank + = [&]() { + Array<int> parts = allGather(static_cast<int>(cell_parts.size())); + std::set<int> part_set; + for (size_t i=0; i<parts.size(); ++i) { + if (parts[i] != 0) { + part_set.insert(i); + } + } + if (part_set.size() != 1) { + perr() << "mesh should be read by one processor\n"; + } + return *begin(part_set); + } (); + + pout() << "Mesh read by process " << rang::style::bold << reader_rank << rang::style::reset << '\n'; Partitioner P; - Array<int> cell_new_owner = broadcast(P.partition(mesh_graph), 0); + Array<int> cell_new_owner = broadcast(P.partition(mesh_graph), reader_rank); for (int i_rank=0; i_rank<commSize(); ++i_rank) { if (commRank() == i_rank) { diff --git a/src/utils/Partitioner.cpp b/src/utils/Partitioner.cpp index 410d873a2947d9314f168fe9d669ed430465cc10..29451cc145939524184de0d1e37fc70464ce1f7e 100644 --- a/src/utils/Partitioner.cpp +++ b/src/utils/Partitioner.cpp @@ -24,10 +24,8 @@ Array<int> Partitioner::partition(const CSRGraph& graph) int numflag = 0; int ncon = 1; int npart= commSize(); - std::vector<float> tpwgts; - for (int i_part=0; i_part<npart; ++i_part) { - tpwgts.push_back(1./npart); - } + std::vector<float> tpwgts(npart, 1./npart); + std::vector<float> ubvec{1.05}; std::vector<int> options{1,1,0}; int edgecut = 0; @@ -37,17 +35,30 @@ Array<int> Partitioner::partition(const CSRGraph& graph) MPI_Comm_group(MPI_COMM_WORLD, &world_group); MPI_Group mesh_group; - std::vector<int> group_ranks{0}; + std::vector<int> group_ranks + = [&]() { + Array<int> graph_node_owners + = allGather(static_cast<int>(graph.numberOfNodes())); + std::vector<int> group_ranks; + group_ranks.reserve(graph_node_owners.size()); + for (size_t i=0; i<graph_node_owners.size(); ++i) { + if (graph_node_owners[i] > 0) { + group_ranks.push_back(i); + } + } + return group_ranks; + } (); + MPI_Group_incl(world_group, group_ranks.size(), &(group_ranks[0]), &mesh_group); MPI_Comm parmetis_comm; MPI_Comm_create_group(MPI_COMM_WORLD, mesh_group, 1, &parmetis_comm); - int local_number_of_cells = graph.entries().size()-1; + int local_number_of_nodes = graph.numberOfNodes(); - if (commRank() ==0) { - part = Array<int>(local_number_of_cells); - std::vector<int> vtxdist{0,local_number_of_cells}; + if (graph.numberOfNodes() > 0) { + part = Array<int>(local_number_of_nodes); + std::vector<int> vtxdist{0,local_number_of_nodes}; static_assert(std::is_same<int, int>());