diff --git a/CMakeLists.txt b/CMakeLists.txt index 52579983f4dc7deaa42079eae59532695cdcf9ae..7c0f69fcf6395a96f88e74e9143e8e85730ba66e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,11 +147,36 @@ if(PARMETIS_LIBRARIES) set_property(TARGET PkgConfig::ParMETIS PROPERTY IMPORTED_LOCATION "${PARMETIS_LIBRARIES}") + set(PUGS_HAS_PARMETIS TRUE) set(PARMETIS_TARGET PkgConfig::ParMETIS) +else() + unset(PUGS_HAS_PARMETIS) endif() +#------------------------------------------------------ +# Search for PTScotch + +find_package(PTScotch) + +set(PUGS_HAS_PTSCOTCH ${PTScotch_FOUND}) + +if (PTScotch_FOUND) + add_library(PkgConfig::PTScotch STATIC IMPORTED) + set_property(TARGET PkgConfig::PTScotch PROPERTY + IMPORTED_LOCATION "${PTSCOTCH_LIBRARY}") + set_property(TARGET PkgConfig::PTScotch PROPERTY + INTERFACE_INCLUDE_DIRECTORIES "${PTSCOTCH_INCLUDE_DIR}") + + set(PTSCOTCH_TARGET PkgConfig::PTScotch) + include_directories(SYSTEM "${PTSCOTCH_INCLUDE_DIR}") +else() + set(PTSCOTCH_LIBRARY "") +endif() + +#------------------------------------------------------ + if(${MPI_FOUND}) - if (NOT PARMETIS_LIBRARIES) + if ((NOT PARMETIS_LIBRARIES) AND (NOT PTSCOTCH_LIBRARIES)) if(PUGS_ENABLE_MPI MATCHES "^AUTO$") message(STATUS "MPI support deactivated: ParMETIS cannot be found!") unset(MPI_FOUND) diff --git a/cmake/FindPTScotch.cmake b/cmake/FindPTScotch.cmake new file mode 100644 index 0000000000000000000000000000000000000000..da6286e3e8ea110460a068f8e40839b15080319c --- /dev/null +++ b/cmake/FindPTScotch.cmake @@ -0,0 +1,29 @@ +# Looking for PTScotch + +find_package(PkgConfig) +pkg_check_modules(PC_PTSCOTCH QUIET PTSCOTCH) + +find_path(PTSCOTCH_INCLUDE_DIR + NAMES ptscotch.h + PATH_SUFFIXES "include" "include/scotch" + HINTS "$ENV{PTSCOTCH_INCDIR}") + +if(EXISTS "${PTSCOTCH_INCLUDE_DIR}/ptscotch.h") + message(STATUS "Found ptscotch.h in ${PTSCOTCH_INCLUDE_DIR}") + find_library(LIB_PTSCOTCH ptscotch $ENV{PTSCOTCH_LIBDIR}) + if("${LIB_PTSCOTCH}" STREQUAL "LIB_PTSCOTCH-NOTFOUND") + message(WARNING "** Could not find ptscotch library.\n** Is PTSCOTCH_LIBDIR correctly set (Actual: \"$ENV{PTSCOTCH_LIBDIR}\")?") + endif() + set(PTSCOTCH_LIBRARIES ${LIB_PTSCOTCH}) + message(STATUS "Found ptscotch/scotch libraries ${PTSCOTCH_LIBRARIES}") +else() + message(WARNING "** Could not find ptscotch.h.\n** Is PTSCOTCH_INCDIR correctly set (Actual: \"$ENV{PTSCOTCH_INCDIR}\")?") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(PTScotch + FOUND_VAR + PTSCOTCH_FOUND + REQUIRED_VARS + PTSCOTCH_LIBRARIES + PTSCOTCH_INCLUDE_DIR) diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 4d74c5e195fe0361819315cd3732661ba7589e56..333d37768f20fa91f47c342a64e809f526142095 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -14,7 +14,9 @@ add_library( FPEManager.cpp GlobalVariableManager.cpp Messenger.cpp + ParMETISPartitioner.cpp Partitioner.cpp + PartitionerOptions.cpp PETScWrapper.cpp PluginsLoader.cpp PugsUtils.cpp diff --git a/src/utils/ParMETISPartitioner.cpp b/src/utils/ParMETISPartitioner.cpp new file mode 100644 index 0000000000000000000000000000000000000000..06feb9b258a10498ca771e0988bd1dc252a2f9a9 --- /dev/null +++ b/src/utils/ParMETISPartitioner.cpp @@ -0,0 +1,93 @@ +#include <utils/ParMETISPartitioner.hpp> + +#include <utils/pugs_config.hpp> + +#ifdef PUGS_HAS_PARMETIS + +#include <utils/Exceptions.hpp> +#include <utils/Messenger.hpp> + +#include <parmetis.h> + +#include <iostream> +#include <vector> + +Array<int> +ParMETISPartitioner::partition(const CRSGraph& graph) +{ + std::cout << "Partitioning graph into " << rang::style::bold << parallel::size() << rang::style::reset + << " parts using " << rang::fgB::green << "ParMETIS" << rang::fg::reset << '\n'; + + int wgtflag = 0; + int numflag = 0; + int ncon = 1; + int npart = parallel::size(); + std::vector<float> tpwgts(npart, 1. / npart); + + std::vector<float> ubvec{1.05}; + std::vector<int> options{1, 0, 0}; + int edgecut = 0; + Array<int> part(0); + + MPI_Group world_group; + + MPI_Comm_group(parallel::Messenger::getInstance().comm(), &world_group); + + MPI_Group mesh_group; + std::vector<int> group_ranks = [&]() { + Array<int> graph_node_owners = parallel::allGather(static_cast<int>(graph.numberOfNodes())); + std::vector<int> grp_ranks; + grp_ranks.reserve(graph_node_owners.size()); + for (size_t i = 0; i < graph_node_owners.size(); ++i) { + if (graph_node_owners[i] > 0) { + grp_ranks.push_back(i); + } + } + return grp_ranks; + }(); + + MPI_Group_incl(world_group, group_ranks.size(), &(group_ranks[0]), &mesh_group); + + MPI_Comm parmetis_comm; + MPI_Comm_create_group(parallel::Messenger::getInstance().comm(), mesh_group, 1, &parmetis_comm); + + int local_number_of_nodes = graph.numberOfNodes(); + + if (graph.numberOfNodes() > 0) { + part = Array<int>(local_number_of_nodes); + std::vector<int> vtxdist{0, local_number_of_nodes}; + + const Array<const int>& entries = graph.entries(); + const Array<const int>& neighbors = graph.neighbors(); + + int* entries_ptr = const_cast<int*>(&(entries[0])); + int* neighbors_ptr = const_cast<int*>(&(neighbors[0])); + + int result = + ParMETIS_V3_PartKway(&(vtxdist[0]), entries_ptr, neighbors_ptr, NULL, NULL, &wgtflag, &numflag, &ncon, &npart, + &(tpwgts[0]), &(ubvec[0]), &(options[0]), &edgecut, &(part[0]), &parmetis_comm); + // LCOV_EXCL_START + if (result == METIS_ERROR) { + throw UnexpectedError("Metis Error"); + } + // LCOV_EXCL_STOP + + MPI_Comm_free(&parmetis_comm); + } + + MPI_Group_free(&mesh_group); + + return part; +} + +#else // PUGS_HAS_PARMETIS + +Array<int> +ParMETISPartitioner::partition(const CRSGraph& graph) +{ + Array<int> partition{graph.entries().size() - 1}; + partition.fill(0); + return partition; +} + +#endif // PUGS_HAS_PARMETIS diff --git a/src/utils/ParMETISPartitioner.hpp b/src/utils/ParMETISPartitioner.hpp new file mode 100644 index 0000000000000000000000000000000000000000..53c1d2dc50e96c85a2c57d0abfb59735b471cc6f --- /dev/null +++ b/src/utils/ParMETISPartitioner.hpp @@ -0,0 +1,19 @@ +#ifndef PARMETIS_PARTITIONER_HPP +#define PARMETIS_PARTITIONER_HPP + +#include <utils/Array.hpp> +#include <utils/CRSGraph.hpp> + +class ParMETISPartitioner +{ + private: + friend class Partitioner; + + // Forbids direct calls + static Array<int> partition(const CRSGraph& graph); + + public: + ParMETISPartitioner() = delete; +}; + +#endif // PARMETIS_PARTITIONER_HPP diff --git a/src/utils/Partitioner.cpp b/src/utils/Partitioner.cpp index 85e13c22d80f449c211f4c37ee20f3107556d6ba..4c7145cf247c3e67bb5e2df802cf4c64e2050d66 100644 --- a/src/utils/Partitioner.cpp +++ b/src/utils/Partitioner.cpp @@ -1,94 +1,21 @@ #include <utils/Partitioner.hpp> -#include <utils/Messenger.hpp> -#include <utils/pugs_config.hpp> +#include <utils/ParMETISPartitioner.hpp> -#ifdef PUGS_HAS_MPI - -#define IDXTYPEWIDTH 64 -#define REALTYPEWIDTH 64 -#include <parmetis.h> - -#include <iostream> -#include <vector> - -#include <utils/Exceptions.hpp> +Partitioner::Partitioner(const PartitionerOptions& options) : m_partitioner_options{options} {} Array<int> Partitioner::partition(const CRSGraph& graph) { - std::cout << "Partitioning graph into " << rang::style::bold << parallel::size() << rang::style::reset << " parts\n"; - - int wgtflag = 0; - int numflag = 0; - int ncon = 1; - int npart = parallel::size(); - std::vector<float> tpwgts(npart, 1. / npart); - - std::vector<float> ubvec{1.05}; - std::vector<int> options{1, 0, 0}; - int edgecut = 0; - Array<int> part(0); - - MPI_Group world_group; - - MPI_Comm_group(parallel::Messenger::getInstance().comm(), &world_group); - - MPI_Group mesh_group; - std::vector<int> group_ranks = [&]() { - Array<int> graph_node_owners = parallel::allGather(static_cast<int>(graph.numberOfNodes())); - std::vector<int> grp_ranks; - grp_ranks.reserve(graph_node_owners.size()); - for (size_t i = 0; i < graph_node_owners.size(); ++i) { - if (graph_node_owners[i] > 0) { - grp_ranks.push_back(i); - } - } - return grp_ranks; - }(); - - MPI_Group_incl(world_group, group_ranks.size(), &(group_ranks[0]), &mesh_group); - - MPI_Comm parmetis_comm; - MPI_Comm_create_group(parallel::Messenger::getInstance().comm(), mesh_group, 1, &parmetis_comm); - - int local_number_of_nodes = graph.numberOfNodes(); - - if (graph.numberOfNodes() > 0) { - part = Array<int>(local_number_of_nodes); - std::vector<int> vtxdist{0, local_number_of_nodes}; - - const Array<const int>& entries = graph.entries(); - const Array<const int>& neighbors = graph.neighbors(); - - int* entries_ptr = const_cast<int*>(&(entries[0])); - int* neighbors_ptr = const_cast<int*>(&(neighbors[0])); - - int result = - ParMETIS_V3_PartKway(&(vtxdist[0]), entries_ptr, neighbors_ptr, NULL, NULL, &wgtflag, &numflag, &ncon, &npart, - &(tpwgts[0]), &(ubvec[0]), &(options[0]), &edgecut, &(part[0]), &parmetis_comm); - // LCOV_EXCL_START - if (result == METIS_ERROR) { - throw UnexpectedError("Metis Error"); - } - // LCOV_EXCL_STOP - - MPI_Comm_free(&parmetis_comm); + switch (m_partitioner_options.library()) { + case PartitionerLibrary::parmetis: { + return ParMETISPartitioner::partition(graph); + } + case PartitionerLibrary::ptscotch: { + throw NotImplementedError("PTScotch"); + } + default: { + throw UnexpectedError("invalid partition library"); + } } - - MPI_Group_free(&mesh_group); - - return part; -} - -#else // PUGS_HAS_MPI - -Array<int> -Partitioner::partition(const CRSGraph& graph) -{ - Array<int> partition{graph.entries().size() - 1}; - partition.fill(0); - return partition; } - -#endif // PUGS_HAS_MPI diff --git a/src/utils/Partitioner.hpp b/src/utils/Partitioner.hpp index 2c720bfd87e7853e12cd0736a46c5eda246f085f..a923d2301635d43f346159aeff6de4331e7dceef 100644 --- a/src/utils/Partitioner.hpp +++ b/src/utils/Partitioner.hpp @@ -2,11 +2,15 @@ #define PARTITIONER_HPP #include <utils/CRSGraph.hpp> +#include <utils/PartitionerOptions.hpp> class Partitioner { + private: + PartitionerOptions m_partitioner_options; + public: - Partitioner() = default; + Partitioner(const PartitionerOptions& options = PartitionerOptions::default_options); Partitioner(const Partitioner&) = default; ~Partitioner() = default; diff --git a/src/utils/PartitionerOptions.cpp b/src/utils/PartitionerOptions.cpp new file mode 100644 index 0000000000000000000000000000000000000000..207d649803be16ac79795084517eb90feb97f283 --- /dev/null +++ b/src/utils/PartitionerOptions.cpp @@ -0,0 +1,10 @@ +#include <utils/PartitionerOptions.hpp> + +#include <rang.hpp> + +std::ostream& +operator<<(std::ostream& os, const PartitionerOptions& options) +{ + os << " library: " << rang::style::bold << name(options.library()) << rang::style::reset << '\n'; + return os; +} diff --git a/src/utils/PartitionerOptions.hpp b/src/utils/PartitionerOptions.hpp new file mode 100644 index 0000000000000000000000000000000000000000..8ce9263b02168f497ea29decfd304865dd8bdab3 --- /dev/null +++ b/src/utils/PartitionerOptions.hpp @@ -0,0 +1,72 @@ +#ifndef PARTITIONER_OPTIONS_HPP +#define PARTITIONER_OPTIONS_HPP + +#include <utils/Exceptions.hpp> +#include <utils/pugs_config.hpp> + +#include <iostream> + +enum class PartitionerLibrary : int8_t +{ + PT__begin = 0, + // + parmetis = PT__begin, + ptscotch, + // + PT__end +}; + +inline std::string +name(const PartitionerLibrary library) +{ + switch (library) { + case PartitionerLibrary::parmetis: { + return "ParMETIS"; + } + case PartitionerLibrary::ptscotch: { + return "PTScotch"; + } + case PartitionerLibrary::PT__end: { + } + } + throw UnexpectedError("Linear system library name is not defined!"); +} + +class PartitionerOptions +{ + private: + PartitionerLibrary m_library = []() { +#if !defined(PUGS_HAS_PARMETIS) && defined(PUGS_HAS_PTSCOTCH) + return PartitionerLibrary::ptscotch; +#else // sets parmetis as default if no alternative is available + return PartitionerLibrary::parmetis; +#endif + }(); + + public: + static PartitionerOptions default_options; + + friend std::ostream& operator<<(std::ostream& os, const PartitionerOptions& options); + + PartitionerLibrary& + library() + { + return m_library; + } + + PartitionerLibrary + library() const + { + return m_library; + } + + PartitionerOptions(const PartitionerOptions&) = default; + PartitionerOptions(PartitionerOptions&&) = default; + + PartitionerOptions() = default; + ~PartitionerOptions() = default; +}; + +inline PartitionerOptions PartitionerOptions::default_options; + +#endif // PARTITIONER_OPTIONS_HPP diff --git a/src/utils/pugs_config.hpp.in b/src/utils/pugs_config.hpp.in index a07d8c314da95f8db595886e3df5b29cdc45907a..9560f7101d75033749dd89801cd560123cfbfb77 100644 --- a/src/utils/pugs_config.hpp.in +++ b/src/utils/pugs_config.hpp.in @@ -1,12 +1,14 @@ #ifndef PUGS_CONFIG_HPP #define PUGS_CONFIG_HPP +#cmakedefine PUGS_HAS_EIGEN3 #cmakedefine PUGS_HAS_FENV_H +#cmakedefine PUGS_HAS_HDF5 #cmakedefine PUGS_HAS_MPI +#cmakedefine PUGS_HAS_PARMETIS #cmakedefine PUGS_HAS_PETSC +#cmakedefine PUGS_HAS_PTSCOTCH #cmakedefine PUGS_HAS_SLEPC -#cmakedefine PUGS_HAS_EIGEN3 -#cmakedefine PUGS_HAS_HDF5 #cmakedefine PUGS_HAS_SLURM #cmakedefine SYSTEM_IS_LINUX