From 3cb1ee257a67c3a8596c11d71c05b7c0f3ce367d Mon Sep 17 00:00:00 2001
From: Stephane Del Pino <stephane.delpino44@gmail.com>
Date: Mon, 25 Feb 2019 12:55:51 +0100
Subject: [PATCH] Prepare SubItemValuePerItem to parallelism (ie:
 synchronization)

SubItemValuePerItem mimics recent developments of ItemValue:
- use a {shared, weak}_ptr to store connectivity
- "WeakSubItemValuePerItem" are available but should only be used to avoid
  circular dependencies

Change some Connectivity member accesses to avoid forced conversion to
shared_ptr-like objects
---
 src/mesh/Connectivity.hpp        |  53 +++++++-------
 src/mesh/GmshReader.cpp          |   4 +-
 src/mesh/IConnectivity.hpp       |   1 +
 src/mesh/SubItemValuePerItem.hpp | 115 +++++++++++++++++++++++++------
 4 files changed, 124 insertions(+), 49 deletions(-)

diff --git a/src/mesh/Connectivity.hpp b/src/mesh/Connectivity.hpp
index 3f1d83e37..261437091 100644
--- a/src/mesh/Connectivity.hpp
+++ b/src/mesh/Connectivity.hpp
@@ -125,23 +125,23 @@ class Connectivity final
   WeakNodeValue<const int> m_node_owner;
   WeakNodeValue<const bool> m_node_is_owned;
 
-  FaceValuePerCell<const bool> m_cell_face_is_reversed;
+  WeakFaceValuePerCell<const bool> m_cell_face_is_reversed;
 
-  NodeValuePerCell<const unsigned short> m_cell_local_numbers_in_their_nodes;
-  EdgeValuePerCell<const unsigned short> m_cell_local_numbers_in_their_edges;
-  FaceValuePerCell<const unsigned short> m_cell_local_numbers_in_their_faces;
+  WeakNodeValuePerCell<const unsigned short> m_cell_local_numbers_in_their_nodes;
+  WeakEdgeValuePerCell<const unsigned short> m_cell_local_numbers_in_their_edges;
+  WeakFaceValuePerCell<const unsigned short> m_cell_local_numbers_in_their_faces;
 
-  CellValuePerFace<const unsigned short> m_face_local_numbers_in_their_cells;
-  EdgeValuePerFace<const unsigned short> m_face_local_numbers_in_their_edges;
-  NodeValuePerFace<const unsigned short> m_face_local_numbers_in_their_nodes;
+  WeakCellValuePerFace<const unsigned short> m_face_local_numbers_in_their_cells;
+  WeakEdgeValuePerFace<const unsigned short> m_face_local_numbers_in_their_edges;
+  WeakNodeValuePerFace<const unsigned short> m_face_local_numbers_in_their_nodes;
 
-  CellValuePerEdge<const unsigned short> m_edge_local_numbers_in_their_cells;
-  FaceValuePerEdge<const unsigned short> m_edge_local_numbers_in_their_faces;
-  NodeValuePerEdge<const unsigned short> m_edge_local_numbers_in_their_nodes;
+  WeakCellValuePerEdge<const unsigned short> m_edge_local_numbers_in_their_cells;
+  WeakFaceValuePerEdge<const unsigned short> m_edge_local_numbers_in_their_faces;
+  WeakNodeValuePerEdge<const unsigned short> m_edge_local_numbers_in_their_nodes;
 
-  CellValuePerNode<const unsigned short> m_node_local_numbers_in_their_cells;
-  EdgeValuePerNode<const unsigned short> m_node_local_numbers_in_their_edges;
-  FaceValuePerNode<const unsigned short> m_node_local_numbers_in_their_faces;
+  WeakCellValuePerNode<const unsigned short> m_node_local_numbers_in_their_cells;
+  WeakEdgeValuePerNode<const unsigned short> m_node_local_numbers_in_their_edges;
+  WeakFaceValuePerNode<const unsigned short> m_node_local_numbers_in_their_faces;
 
   ConnectivityComputer m_connectivity_computer;
 
@@ -185,49 +185,49 @@ class Connectivity final
 
  public:
   PASTIS_INLINE
-  CellValue<const CellType> cellType() const
+  const auto& cellType() const
   {
     return m_cell_type;
   }
 
   PASTIS_INLINE
-  CellValue<const int> cellNumber() const
+  const auto& cellNumber() const
   {
     return m_cell_number;
   }
 
   PASTIS_INLINE
-  FaceValue<const int> faceNumber() const
+  const auto& faceNumber() const
   {
     return m_face_number;
   }
 
   PASTIS_INLINE
-  EdgeValue<const int> edgeNumber() const
+  const auto& edgeNumber() const
   {
     return m_edge_number;
   }
 
   PASTIS_INLINE
-  NodeValue<const int> nodeNumber() const
+  const auto& nodeNumber() const
   {
     return m_node_number;
   }
 
   PASTIS_INLINE
-  CellValue<const int> cellOwner() const
+  const auto& cellOwner() const
   {
     return m_cell_owner;
   }
 
   PASTIS_INLINE
-  FaceValue<const int> faceOwner() const
+  const auto& faceOwner() const
   {
     return m_face_owner;
   }
 
   PASTIS_INLINE
-  EdgeValue<const int> edgeOwner() const
+  const auto& edgeOwner() const
   {
     perr() << __FILE__ << ':' << __LINE__ << ": edge owner not built\n";
     std::terminate();
@@ -235,7 +235,7 @@ class Connectivity final
   }
 
   PASTIS_INLINE
-  NodeValue<const int> nodeOwner() const
+  const auto& nodeOwner() const
   {
     return m_node_owner;
   }
@@ -259,19 +259,19 @@ class Connectivity final
   }
 
   PASTIS_INLINE
-  CellValue<const bool> cellIsOwned() const
+  const auto& cellIsOwned() const
   {
     return m_cell_is_owned;
   }
 
   PASTIS_INLINE
-  FaceValue<const bool> faceIsOwned() const
+  const auto& faceIsOwned() const
   {
     return m_face_is_owned;
   }
 
   PASTIS_INLINE
-  EdgeValue<const bool> edgeIsOwned() const
+  const auto& edgeIsOwned() const
   {
     perr() << __FILE__ << ':' << __LINE__ << ": edge is owned not built\n";
     std::terminate();
@@ -279,7 +279,7 @@ class Connectivity final
   }
 
   PASTIS_INLINE
-  NodeValue<const bool> nodeIsOwned() const
+  const auto& nodeIsOwned() const
   {
     return m_node_is_owned;
   }
@@ -374,7 +374,6 @@ class Connectivity final
     return this->template getItemToItemMatrix<ItemType::node, ItemType::edge>();
   }
 
-
   PASTIS_INLINE
   const auto& cellFaceIsReversed() const
   {
diff --git a/src/mesh/GmshReader.cpp b/src/mesh/GmshReader.cpp
index 98ce886b3..a1a4f6220 100644
--- a/src/mesh/GmshReader.cpp
+++ b/src/mesh/GmshReader.cpp
@@ -472,9 +472,9 @@ class MeshDispatcher
     return m_node_new_owner;
   }
 
-  template<typename DataType>
+  template<typename DataType, typename ConnectivityPtr>
   std::vector<Array<std::remove_const_t<DataType>>>
-  exchange(const CellValue<DataType>& cell_value) const
+  exchange(ItemValue<DataType, ItemType::cell, ConnectivityPtr> cell_value) const
   {
     using MutableDataType = std::remove_const_t<DataType>;
     std::vector<Array<DataType>> cell_value_to_send_by_proc(parallel::size());
diff --git a/src/mesh/IConnectivity.hpp b/src/mesh/IConnectivity.hpp
index a7735f4df..359c9c75c 100644
--- a/src/mesh/IConnectivity.hpp
+++ b/src/mesh/IConnectivity.hpp
@@ -11,6 +11,7 @@ class IConnectivity : public std::enable_shared_from_this<IConnectivity>
   template <typename DataType,
             ItemType sub_item_type,
             ItemType item_type,
+            typename ConnectivityPtr,
             typename Allowed>
   friend
   class SubItemValuePerItem;
diff --git a/src/mesh/SubItemValuePerItem.hpp b/src/mesh/SubItemValuePerItem.hpp
index edb73dd5c..95d0d6949 100644
--- a/src/mesh/SubItemValuePerItem.hpp
+++ b/src/mesh/SubItemValuePerItem.hpp
@@ -12,19 +12,23 @@
 
 #include <ConnectivityMatrix.hpp>
 #include <IConnectivity.hpp>
+#include <memory>
 
 template <typename DataType,
           ItemType sub_item_type,
           ItemType item_type,
+          typename ConnectivityPtr = std::shared_ptr<const IConnectivity>,
           typename Allowed=void>
 class SubItemValuePerItem;
 
 template <typename DataType,
           ItemType sub_item_type,
-          ItemType item_type>
+          ItemType item_type,
+          typename ConnectivityPtr>
 class SubItemValuePerItem<DataType,
                           sub_item_type,
                           item_type,
+                          ConnectivityPtr,
                           std::enable_if_t<sub_item_type != item_type>>
 {
  public:
@@ -36,17 +40,36 @@ class SubItemValuePerItem<DataType,
   using index_type = ItemId;
 
  private:
-  bool m_is_built{false};
+  using ConnectivitySharedPtr = std::shared_ptr<const IConnectivity>;
+  using ConnectivityWeakPtr = std::weak_ptr<const IConnectivity>;
+
+  static_assert(std::is_same_v<ConnectivityPtr, ConnectivitySharedPtr> or
+                std::is_same_v<ConnectivityPtr, ConnectivityWeakPtr>);
+
+  ConnectivityPtr m_connectivity_ptr;
 
   typename ConnectivityMatrix::HostRowType m_host_row_map;
   Array<DataType> m_values;
 
-  // Allows const version to access our data
+  // Allow const std:shared_ptr version to access our data
+  friend SubItemValuePerItem<std::add_const_t<DataType>,
+                             sub_item_type,
+                             item_type,
+                             ConnectivitySharedPtr>;
+
+  // Allow const std:weak_ptr version to access our data
   friend SubItemValuePerItem<std::add_const_t<DataType>,
                              sub_item_type,
-                             item_type>;
+                             item_type,
+                             ConnectivityWeakPtr>;
 
  public:
+  using ToShared = SubItemValuePerItem<DataType,
+                                       sub_item_type,
+                                       item_type,
+                                       ConnectivitySharedPtr,
+                                       std::enable_if_t<sub_item_type != item_type>>;
+
   class SubView
   {
    public:
@@ -94,10 +117,10 @@ class SubItemValuePerItem<DataType,
     }
   };
 
-  PASTIS_FORCEINLINE
-  const bool& isBuilt() const
+  PASTIS_INLINE
+  bool isBuilt() const
   {
-    return m_is_built;
+    return m_connectivity_ptr.use_count() != 0;
   }
 
   // Following Kokkos logic, these classes are view and const view does allow
@@ -105,7 +128,7 @@ class SubItemValuePerItem<DataType,
   PASTIS_FORCEINLINE
   DataType& operator()(const ItemId& j, const size_t& r) const noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     return m_values[m_host_row_map(size_t{j})+r];
   }
 
@@ -121,7 +144,7 @@ class SubItemValuePerItem<DataType,
   PASTIS_INLINE
   size_t numberOfValues() const noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     return m_values.size();
   }
 
@@ -130,7 +153,7 @@ class SubItemValuePerItem<DataType,
   PASTIS_FORCEINLINE
   DataType& operator[](const size_t& i) const noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     return m_values[i];
   }
 
@@ -145,7 +168,7 @@ class SubItemValuePerItem<DataType,
   PASTIS_INLINE
   size_t numberOfItems() const noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     Assert(m_host_row_map.extent(0) != 0);
     return m_host_row_map.extent(0);
   }
@@ -153,14 +176,14 @@ class SubItemValuePerItem<DataType,
   PASTIS_INLINE
   size_t numberOfSubValues(const size_t& i_cell) const noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     return m_host_row_map(i_cell+1)-m_host_row_map(i_cell);
   }
 
   PASTIS_INLINE
   SubView itemValues(const size_t& i_cell) noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     const auto& cell_begin = m_host_row_map(i_cell);
     const auto& cell_end = m_host_row_map(i_cell+1);
     return SubView(m_values, cell_begin, cell_end);
@@ -171,16 +194,17 @@ class SubItemValuePerItem<DataType,
   PASTIS_INLINE
   SubView itemValues(const size_t& i_cell) const noexcept(NO_ASSERT)
   {
-    Assert(m_is_built);
+    Assert(this->isBuilt());
     const auto& cell_begin = m_host_row_map(i_cell);
     const auto& cell_end = m_host_row_map(i_cell+1);
     return SubView(m_values, cell_begin, cell_end);
   }
 
-  template <typename DataType2>
+  template <typename DataType2,
+            typename ConnectivityPtr2>
   PASTIS_INLINE
   SubItemValuePerItem&
-  operator=(const SubItemValuePerItem<DataType2, sub_item_type, item_type>& sub_item_value_per_item)
+  operator=(const SubItemValuePerItem<DataType2, sub_item_type, item_type, ConnectivityPtr2>& sub_item_value_per_item)
   {
     // ensures that DataType is the same as source DataType2
     static_assert(std::is_same<std::remove_const_t<DataType>, std::remove_const_t<DataType2>>(),
@@ -192,14 +216,20 @@ class SubItemValuePerItem<DataType,
     m_host_row_map = sub_item_value_per_item.m_host_row_map;
     m_values = sub_item_value_per_item.m_values;
 
-    m_is_built = sub_item_value_per_item.m_is_built;
+    if constexpr (std::is_same_v<ConnectivityPtr, ConnectivitySharedPtr> and
+                  std::is_same_v<ConnectivityPtr2, ConnectivityWeakPtr>) {
+      m_connectivity_ptr = sub_item_value_per_item.m_connectivity_ptr.lock();
+    } else {
+      m_connectivity_ptr = sub_item_value_per_item.m_connectivity_ptr;
+    }
 
     return *this;
   }
 
-  template <typename DataType2>
+  template <typename DataType2,
+            typename ConnectivityPtr2>
   PASTIS_INLINE
-  SubItemValuePerItem(const SubItemValuePerItem<DataType2, sub_item_type, item_type>& sub_item_value_per_item)
+  SubItemValuePerItem(const SubItemValuePerItem<DataType2, sub_item_type, item_type, ConnectivityPtr2>& sub_item_value_per_item)
   {
     this->operator=(sub_item_value_per_item);
   }
@@ -207,7 +237,7 @@ class SubItemValuePerItem<DataType,
   SubItemValuePerItem() = default;
 
   SubItemValuePerItem(const IConnectivity& connectivity)
-      : m_is_built{true}
+      : m_connectivity_ptr{connectivity.shared_ptr()}
   {
     static_assert(not std::is_const<DataType>(),
                   "Cannot allocate SubItemValuePerItem of const data: only view is supported"); ;
@@ -266,4 +296,49 @@ using CellValuePerEdge = SubItemValuePerItem<DataType, ItemType::cell, ItemType:
 template <typename DataType>
 using CellValuePerFace = SubItemValuePerItem<DataType, ItemType::cell, ItemType::face>;
 
+// Weak versions: should not be used outside of Connectivity
+// Item values at nodes
+
+template <typename DataType>
+using WeakNodeValuePerEdge = SubItemValuePerItem<DataType, ItemType::node, ItemType::edge, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakNodeValuePerFace = SubItemValuePerItem<DataType, ItemType::node, ItemType::face, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakNodeValuePerCell = SubItemValuePerItem<DataType, ItemType::node, ItemType::cell, std::weak_ptr<const IConnectivity>>;
+
+// Item values at edges
+
+template <typename DataType>
+using WeakEdgeValuePerNode = SubItemValuePerItem<DataType, ItemType::edge, ItemType::node, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakEdgeValuePerFace = SubItemValuePerItem<DataType, ItemType::edge, ItemType::face, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakEdgeValuePerCell = SubItemValuePerItem<DataType, ItemType::edge, ItemType::cell, std::weak_ptr<const IConnectivity>>;
+
+// Item values at faces
+
+template <typename DataType>
+using WeakFaceValuePerNode = SubItemValuePerItem<DataType, ItemType::face, ItemType::node, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakFaceValuePerEdge = SubItemValuePerItem<DataType, ItemType::face, ItemType::edge, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakFaceValuePerCell = SubItemValuePerItem<DataType, ItemType::face, ItemType::cell, std::weak_ptr<const IConnectivity>>;
+
+// Item values at cells
+
+template <typename DataType>
+using WeakCellValuePerNode = SubItemValuePerItem<DataType, ItemType::cell, ItemType::node, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakCellValuePerEdge = SubItemValuePerItem<DataType, ItemType::cell, ItemType::edge, std::weak_ptr<const IConnectivity>>;
+
+template <typename DataType>
+using WeakCellValuePerFace = SubItemValuePerItem<DataType, ItemType::cell, ItemType::face, std::weak_ptr<const IConnectivity>>;
+
 #endif // SUBITEM_VALUE_PER_ITEM_HPP
-- 
GitLab