Prepare PTScotch integration

- Add simple CMake recipe
- Add PartitionerOptions
- Separate ParMETIS partitioner from Partitioner

CMakeLists.txt

@@ -147,11 +147,36 @@ if(PARMETIS_LIBRARIES)
   set_property(TARGET PkgConfig::ParMETIS PROPERTY
     IMPORTED_LOCATION "${PARMETIS_LIBRARIES}")
 
+  set(PUGS_HAS_PARMETIS TRUE)
   set(PARMETIS_TARGET PkgConfig::ParMETIS)
+else()
+  unset(PUGS_HAS_PARMETIS)
 endif()
 
+#------------------------------------------------------
+# Search for PTScotch
+
+find_package(PTScotch)
+
+set(PUGS_HAS_PTSCOTCH ${PTScotch_FOUND})
+
+if (PTScotch_FOUND)
+  add_library(PkgConfig::PTScotch STATIC IMPORTED)
+  set_property(TARGET PkgConfig::PTScotch PROPERTY
+    IMPORTED_LOCATION "${PTSCOTCH_LIBRARY}")
+  set_property(TARGET PkgConfig::PTScotch PROPERTY
+    INTERFACE_INCLUDE_DIRECTORIES "${PTSCOTCH_INCLUDE_DIR}")
+
+  set(PTSCOTCH_TARGET PkgConfig::PTScotch)
+  include_directories(SYSTEM "${PTSCOTCH_INCLUDE_DIR}")
+else()
+  set(PTSCOTCH_LIBRARY "")
+endif()
+
+#------------------------------------------------------
+
 if(${MPI_FOUND})
-  if (NOT PARMETIS_LIBRARIES)
+  if ((NOT PARMETIS_LIBRARIES) AND (NOT PTSCOTCH_LIBRARIES))
     if(PUGS_ENABLE_MPI MATCHES "^AUTO$")
       message(STATUS "MPI support deactivated: ParMETIS cannot be found!")
       unset(MPI_FOUND)

cmake/FindPTScotch.cmake

+# Looking for PTScotch
+
+find_package(PkgConfig)
+pkg_check_modules(PC_PTSCOTCH QUIET PTSCOTCH)
+
+find_path(PTSCOTCH_INCLUDE_DIR
+  NAMES ptscotch.h
+  PATH_SUFFIXES  "include" "include/scotch"
+  HINTS "$ENV{PTSCOTCH_INCDIR}")
+
+if(EXISTS "${PTSCOTCH_INCLUDE_DIR}/ptscotch.h")
+  message(STATUS "Found ptscotch.h in ${PTSCOTCH_INCLUDE_DIR}")
+  find_library(LIB_PTSCOTCH ptscotch $ENV{PTSCOTCH_LIBDIR})
+  if("${LIB_PTSCOTCH}" STREQUAL "LIB_PTSCOTCH-NOTFOUND")
+    message(WARNING "** Could not find ptscotch library.\n** Is PTSCOTCH_LIBDIR correctly set (Actual: \"$ENV{PTSCOTCH_LIBDIR}\")?")
+  endif()
+  set(PTSCOTCH_LIBRARIES ${LIB_PTSCOTCH})
+  message(STATUS "Found ptscotch/scotch libraries ${PTSCOTCH_LIBRARIES}")
+else()
+  message(WARNING "** Could not find ptscotch.h.\n** Is PTSCOTCH_INCDIR correctly set (Actual: \"$ENV{PTSCOTCH_INCDIR}\")?")
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(PTScotch
+  FOUND_VAR
+    PTSCOTCH_FOUND
+  REQUIRED_VARS
+    PTSCOTCH_LIBRARIES
+    PTSCOTCH_INCLUDE_DIR)

src/utils/CMakeLists.txt

@@ -14,7 +14,9 @@ add_library(
   FPEManager.cpp
   GlobalVariableManager.cpp
   Messenger.cpp
+  ParMETISPartitioner.cpp
   Partitioner.cpp
+  PartitionerOptions.cpp
   PETScWrapper.cpp
   PluginsLoader.cpp
   PugsUtils.cpp

src/utils/ParMETISPartitioner.cpp

+#include <utils/ParMETISPartitioner.hpp>
+
+#include <utils/pugs_config.hpp>
+
+#ifdef PUGS_HAS_PARMETIS
+
+#include <utils/Exceptions.hpp>
+#include <utils/Messenger.hpp>
+
+#include <parmetis.h>
+
+#include <iostream>
+#include <vector>
+
+Array<int>
+ParMETISPartitioner::partition(const CRSGraph& graph)
+{
+  std::cout << "Partitioning graph into " << rang::style::bold << parallel::size() << rang::style::reset
+            << " parts using " << rang::fgB::green << "ParMETIS" << rang::fg::reset << '\n';
+
+  int wgtflag = 0;
+  int numflag = 0;
+  int ncon    = 1;
+  int npart   = parallel::size();
+  std::vector<float> tpwgts(npart, 1. / npart);
+
+  std::vector<float> ubvec{1.05};
+  std::vector<int> options{1, 0, 0};
+  int edgecut = 0;
+  Array<int> part(0);
+
+  MPI_Group world_group;
+
+  MPI_Comm_group(parallel::Messenger::getInstance().comm(), &world_group);
+
+  MPI_Group mesh_group;
+  std::vector<int> group_ranks = [&]() {
+    Array<int> graph_node_owners = parallel::allGather(static_cast<int>(graph.numberOfNodes()));
+    std::vector<int> grp_ranks;
+    grp_ranks.reserve(graph_node_owners.size());
+    for (size_t i = 0; i < graph_node_owners.size(); ++i) {
+      if (graph_node_owners[i] > 0) {
+        grp_ranks.push_back(i);
+      }
+    }
+    return grp_ranks;
+  }();
+
+  MPI_Group_incl(world_group, group_ranks.size(), &(group_ranks[0]), &mesh_group);
+
+  MPI_Comm parmetis_comm;
+  MPI_Comm_create_group(parallel::Messenger::getInstance().comm(), mesh_group, 1, &parmetis_comm);
+
+  int local_number_of_nodes = graph.numberOfNodes();
+
+  if (graph.numberOfNodes() > 0) {
+    part = Array<int>(local_number_of_nodes);
+    std::vector<int> vtxdist{0, local_number_of_nodes};
+
+    const Array<const int>& entries   = graph.entries();
+    const Array<const int>& neighbors = graph.neighbors();
+
+    int* entries_ptr   = const_cast<int*>(&(entries[0]));
+    int* neighbors_ptr = const_cast<int*>(&(neighbors[0]));
+
+    int result =
+      ParMETIS_V3_PartKway(&(vtxdist[0]), entries_ptr, neighbors_ptr, NULL, NULL, &wgtflag, &numflag, &ncon, &npart,
+                           &(tpwgts[0]), &(ubvec[0]), &(options[0]), &edgecut, &(part[0]), &parmetis_comm);
+    // LCOV_EXCL_START
+    if (result == METIS_ERROR) {
+      throw UnexpectedError("Metis Error");
+    }
+    // LCOV_EXCL_STOP
+
+    MPI_Comm_free(&parmetis_comm);
+  }
+
+  MPI_Group_free(&mesh_group);
+
+  return part;
+}
+
+#else   // PUGS_HAS_PARMETIS
+
+Array<int>
+ParMETISPartitioner::partition(const CRSGraph& graph)
+{
+  Array<int> partition{graph.entries().size() - 1};
+  partition.fill(0);
+  return partition;
+}
+
+#endif   // PUGS_HAS_PARMETIS

src/utils/ParMETISPartitioner.hpp

+#ifndef PARMETIS_PARTITIONER_HPP
+#define PARMETIS_PARTITIONER_HPP
+
+#include <utils/Array.hpp>
+#include <utils/CRSGraph.hpp>
+
+class ParMETISPartitioner
+{
+ private:
+  friend class Partitioner;
+
+  // Forbids direct calls
+  static Array<int> partition(const CRSGraph& graph);
+
+ public:
+  ParMETISPartitioner() = delete;
+};
+
+#endif   // PARMETIS_PARTITIONER_HPP

src/utils/Partitioner.cpp

 #include <utils/Partitioner.hpp>
 
-#include <utils/Messenger.hpp>
-#include <utils/pugs_config.hpp>
+#include <utils/ParMETISPartitioner.hpp>
 
-#ifdef PUGS_HAS_MPI
-
-#define IDXTYPEWIDTH 64
-#define REALTYPEWIDTH 64
-#include <parmetis.h>
-
-#include <iostream>
-#include <vector>
-
-#include <utils/Exceptions.hpp>
+Partitioner::Partitioner(const PartitionerOptions& options) : m_partitioner_options{options} {}
 
 Array<int>
 Partitioner::partition(const CRSGraph& graph)
 {
-  std::cout << "Partitioning graph into " << rang::style::bold << parallel::size() << rang::style::reset << " parts\n";
-
-  int wgtflag = 0;
-  int numflag = 0;
-  int ncon    = 1;
-  int npart   = parallel::size();
-  std::vector<float> tpwgts(npart, 1. / npart);
-
-  std::vector<float> ubvec{1.05};
-  std::vector<int> options{1, 0, 0};
-  int edgecut = 0;
-  Array<int> part(0);
-
-  MPI_Group world_group;
-
-  MPI_Comm_group(parallel::Messenger::getInstance().comm(), &world_group);
-
-  MPI_Group mesh_group;
-  std::vector<int> group_ranks = [&]() {
-    Array<int> graph_node_owners = parallel::allGather(static_cast<int>(graph.numberOfNodes()));
-    std::vector<int> grp_ranks;
-    grp_ranks.reserve(graph_node_owners.size());
-    for (size_t i = 0; i < graph_node_owners.size(); ++i) {
-      if (graph_node_owners[i] > 0) {
-        grp_ranks.push_back(i);
+  switch (m_partitioner_options.library()) {
+  case PartitionerLibrary::parmetis: {
+    return ParMETISPartitioner::partition(graph);
   }
+  case PartitionerLibrary::ptscotch: {
+    throw NotImplementedError("PTScotch");
   }
-    return grp_ranks;
-  }();
-
-  MPI_Group_incl(world_group, group_ranks.size(), &(group_ranks[0]), &mesh_group);
-
-  MPI_Comm parmetis_comm;
-  MPI_Comm_create_group(parallel::Messenger::getInstance().comm(), mesh_group, 1, &parmetis_comm);
-
-  int local_number_of_nodes = graph.numberOfNodes();
-
-  if (graph.numberOfNodes() > 0) {
-    part = Array<int>(local_number_of_nodes);
-    std::vector<int> vtxdist{0, local_number_of_nodes};
-
-    const Array<const int>& entries   = graph.entries();
-    const Array<const int>& neighbors = graph.neighbors();
-
-    int* entries_ptr   = const_cast<int*>(&(entries[0]));
-    int* neighbors_ptr = const_cast<int*>(&(neighbors[0]));
-
-    int result =
-      ParMETIS_V3_PartKway(&(vtxdist[0]), entries_ptr, neighbors_ptr, NULL, NULL, &wgtflag, &numflag, &ncon, &npart,
-                           &(tpwgts[0]), &(ubvec[0]), &(options[0]), &edgecut, &(part[0]), &parmetis_comm);
-    // LCOV_EXCL_START
-    if (result == METIS_ERROR) {
-      throw UnexpectedError("Metis Error");
-    }
-    // LCOV_EXCL_STOP
-
-    MPI_Comm_free(&parmetis_comm);
+  default: {
+    throw UnexpectedError("invalid partition library");
   }
-
-  MPI_Group_free(&mesh_group);
-
-  return part;
   }
-
-#else   // PUGS_HAS_MPI
-
-Array<int>
-Partitioner::partition(const CRSGraph& graph)
-{
-  Array<int> partition{graph.entries().size() - 1};
-  partition.fill(0);
-  return partition;
 }
-
-#endif   // PUGS_HAS_MPI

src/utils/Partitioner.hpp

@@ -2,11 +2,15 @@
 #define PARTITIONER_HPP
 
 #include <utils/CRSGraph.hpp>
+#include <utils/PartitionerOptions.hpp>
 
 class Partitioner
 {
+ private:
+  PartitionerOptions m_partitioner_options;
+
  public:
-  Partitioner()                   = default;
+  Partitioner(const PartitionerOptions& options = PartitionerOptions::default_options);
   Partitioner(const Partitioner&) = default;
   ~Partitioner()                  = default;
 

src/utils/PartitionerOptions.cpp

+#include <utils/PartitionerOptions.hpp>
+
+#include <rang.hpp>
+
+std::ostream&
+operator<<(std::ostream& os, const PartitionerOptions& options)
+{
+  os << "  library: " << rang::style::bold << name(options.library()) << rang::style::reset << '\n';
+  return os;
+}

src/utils/PartitionerOptions.hpp

+#ifndef PARTITIONER_OPTIONS_HPP
+#define PARTITIONER_OPTIONS_HPP
+
+#include <utils/Exceptions.hpp>
+#include <utils/pugs_config.hpp>
+
+#include <iostream>
+
+enum class PartitionerLibrary : int8_t
+{
+  PT__begin = 0,
+  //
+  parmetis = PT__begin,
+  ptscotch,
+  //
+  PT__end
+};
+
+inline std::string
+name(const PartitionerLibrary library)
+{
+  switch (library) {
+  case PartitionerLibrary::parmetis: {
+    return "ParMETIS";
+  }
+  case PartitionerLibrary::ptscotch: {
+    return "PTScotch";
+  }
+  case PartitionerLibrary::PT__end: {
+  }
+  }
+  throw UnexpectedError("Linear system library name is not defined!");
+}
+
+class PartitionerOptions
+{
+ private:
+  PartitionerLibrary m_library = []() {
+#if !defined(PUGS_HAS_PARMETIS) && defined(PUGS_HAS_PTSCOTCH)
+    return PartitionerLibrary::ptscotch;
+#else   // sets parmetis as default if no alternative is available
+    return PartitionerLibrary::parmetis;
+#endif
+  }();
+
+ public:
+  static PartitionerOptions default_options;
+
+  friend std::ostream& operator<<(std::ostream& os, const PartitionerOptions& options);
+
+  PartitionerLibrary&
+  library()
+  {
+    return m_library;
+  }
+
+  PartitionerLibrary
+  library() const
+  {
+    return m_library;
+  }
+
+  PartitionerOptions(const PartitionerOptions&) = default;
+  PartitionerOptions(PartitionerOptions&&)      = default;
+
+  PartitionerOptions()  = default;
+  ~PartitionerOptions() = default;
+};
+
+inline PartitionerOptions PartitionerOptions::default_options;
+
+#endif   // PARTITIONER_OPTIONS_HPP

src/utils/pugs_config.hpp.in

 #ifndef PUGS_CONFIG_HPP
 #define PUGS_CONFIG_HPP
 
+#cmakedefine PUGS_HAS_EIGEN3
 #cmakedefine PUGS_HAS_FENV_H
+#cmakedefine PUGS_HAS_HDF5
 #cmakedefine PUGS_HAS_MPI
+#cmakedefine PUGS_HAS_PARMETIS
 #cmakedefine PUGS_HAS_PETSC
+#cmakedefine PUGS_HAS_PTSCOTCH
 #cmakedefine PUGS_HAS_SLEPC
-#cmakedefine PUGS_HAS_EIGEN3
-#cmakedefine PUGS_HAS_HDF5
 #cmakedefine PUGS_HAS_SLURM
 
 #cmakedefine SYSTEM_IS_LINUX