From 5a0f8744762c7736e29f3bdec031a760fcbbc335 Mon Sep 17 00:00:00 2001
From: Stephane Del Pino <stephane.delpino44@gmail.com>
Date: Sat, 7 Sep 2024 17:42:20 +0200
Subject: [PATCH] Add tests for SubItemArrayPerItemVariant checkpointing

---
 .../ReadSubItemArrayPerItemVariant.cpp        |   4 +
 tests/CMakeLists.txt                          |   1 +
 tests/test_checkpointing_ItemArrayVariant.cpp |   6 -
 ...eckpointing_SubItemArrayPerItemVariant.cpp | 396 ++++++++++++++++++
 4 files changed, 401 insertions(+), 6 deletions(-)
 create mode 100644 tests/test_checkpointing_SubItemArrayPerItemVariant.cpp

diff --git a/src/utils/checkpointing/ReadSubItemArrayPerItemVariant.cpp b/src/utils/checkpointing/ReadSubItemArrayPerItemVariant.cpp
index e2b6b6c90..274a79013 100644
--- a/src/utils/checkpointing/ReadSubItemArrayPerItemVariant.cpp
+++ b/src/utils/checkpointing/ReadSubItemArrayPerItemVariant.cpp
@@ -72,11 +72,15 @@ readSubItemArrayPerItemVariant(const HighFive::Group& sub_item_array_per_item_va
         readSubItemArrayPerItem<TinyMatrix<3>, item_type, sub_item_type>(sub_item_array_per_item_variant_group,
                                                                          "arrays", connectivity));
     } else {
+      // LCOV_EXCL_START
       throw UnexpectedError("unexpected discrete function data type: " + data_type);
+      // LCOV_EXCL_STOP
     }
     return p_sub_item_array_per_item_variant;
   } else {
+    // LCOV_EXCL_START
     throw UnexpectedError("item_type and sub_item_type must be different");
+    // LCOV_EXCL_STOP
   }
 }
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index cff9f33d2..aea0b508b 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -177,6 +177,7 @@ if(PUGS_HAS_HDF5)
     test_checkpointing_IWriter.cpp
     test_checkpointing_IZoneDescriptor.cpp
     test_checkpointing_Mesh.cpp
+    test_checkpointing_SubItemArrayPerItemVariant.cpp
     test_checkpointing_SubItemValuePerItemVariant.cpp
     test_checkpointing_Table.cpp
   )
diff --git a/tests/test_checkpointing_ItemArrayVariant.cpp b/tests/test_checkpointing_ItemArrayVariant.cpp
index 06a8d8dac..847d6d0e0 100644
--- a/tests/test_checkpointing_ItemArrayVariant.cpp
+++ b/tests/test_checkpointing_ItemArrayVariant.cpp
@@ -34,12 +34,6 @@ check_is_same(const ItemArray<DataType, item_type>& reference, const EmbeddedDat
           same &= (a(i, j) == b(i, j));
         }
       }
-    } else {
-      same = false;
-    }
-
-    if (not same) {
-      throw UnexpectedError("a!=b");
     }
 
     return parallel::allReduceAnd(same);
diff --git a/tests/test_checkpointing_SubItemArrayPerItemVariant.cpp b/tests/test_checkpointing_SubItemArrayPerItemVariant.cpp
new file mode 100644
index 000000000..bf13d74e0
--- /dev/null
+++ b/tests/test_checkpointing_SubItemArrayPerItemVariant.cpp
@@ -0,0 +1,396 @@
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_all.hpp>
+
+#include <utils/Messenger.hpp>
+
+#include <language/utils/DataHandler.hpp>
+#include <language/utils/EmbeddedData.hpp>
+#include <mesh/Mesh.hpp>
+#include <mesh/SubItemArrayPerItemVariant.hpp>
+#include <utils/GlobalVariableManager.hpp>
+#include <utils/checkpointing/ReadSubItemArrayPerItemVariant.hpp>
+#include <utils/checkpointing/ResumingData.hpp>
+#include <utils/checkpointing/WriteSubItemArrayPerItemVariant.hpp>
+
+#include <MeshDataBaseForTests.hpp>
+#include <checkpointing_Connectivity_utilities.hpp>
+
+#include <filesystem>
+
+// clazy:excludeall=non-pod-global-static
+
+namespace test_only
+{
+
+template <typename DataType, typename ItemOfItemTypeT>
+PUGS_INLINE void
+check_is_same(const SubItemArrayPerItem<DataType, ItemOfItemTypeT>& reference, const EmbeddedData& e_read_data)
+{
+  auto same_table = [](const auto& a, const auto& b) -> bool {
+    bool same = true;
+    if ((a.numberOfRows() == b.numberOfRows()) and (a.numberOfColumns() == b.numberOfColumns())) {
+      for (size_t i = 0; i < a.numberOfRows(); ++i) {
+        for (size_t j = 0; j < a.numberOfColumns(); ++j) {
+          same &= (a(i, j) == b(i, j));
+        }
+      }
+    }
+
+    return parallel::allReduceAnd(same);
+  };
+
+  REQUIRE_NOTHROW(dynamic_cast<const DataHandler<const SubItemArrayPerItemVariant>&>(e_read_data.get()));
+
+  std::shared_ptr<const SubItemArrayPerItemVariant> p_new_data_v =
+    dynamic_cast<const DataHandler<const SubItemArrayPerItemVariant>&>(e_read_data.get()).data_ptr();
+
+  using SubItemArrayPerItemT = SubItemArrayPerItem<const DataType, ItemOfItemTypeT>;
+
+  SubItemArrayPerItemT read_data = p_new_data_v->get<SubItemArrayPerItemT>();
+
+  switch (reference.connectivity_ptr()->dimension()) {
+  case 1: {
+    REQUIRE(test_only::isSameConnectivity(dynamic_cast<const Connectivity<1>&>(*reference.connectivity_ptr()),
+                                          dynamic_cast<const Connectivity<1>&>(*read_data.connectivity_ptr())));
+    break;
+  }
+  case 2: {
+    REQUIRE(test_only::isSameConnectivity(dynamic_cast<const Connectivity<2>&>(*reference.connectivity_ptr()),
+                                          dynamic_cast<const Connectivity<2>&>(*read_data.connectivity_ptr())));
+    break;
+  }
+  case 3: {
+    REQUIRE(test_only::isSameConnectivity(dynamic_cast<const Connectivity<3>&>(*reference.connectivity_ptr()),
+                                          dynamic_cast<const Connectivity<3>&>(*read_data.connectivity_ptr())));
+    break;
+  }
+  default: {
+    throw UnexpectedError("invalid connectivity dimension");
+  }
+  }
+
+  REQUIRE(same_table(reference.tableView(), read_data.tableView()));
+}
+
+}   // namespace test_only
+
+TEST_CASE("checkpointing_SubItemArrayPerItemVariant", "[utils/checkpointing]")
+{
+  std::string tmp_dirname;
+  {
+    {
+      if (parallel::rank() == 0) {
+        tmp_dirname = [&]() -> std::string {
+          std::string temp_filename = std::filesystem::temp_directory_path() / "pugs_checkpointing_XXXXXX";
+          return std::string{mkdtemp(&temp_filename[0])};
+        }();
+      }
+      parallel::broadcast(tmp_dirname, 0);
+    }
+    std::filesystem::path path = tmp_dirname;
+    const std::string filename = path / "checkpoint.h5";
+
+    HighFive::FileAccessProps fapl;
+    fapl.add(HighFive::MPIOFileAccess{MPI_COMM_WORLD, MPI_INFO_NULL});
+    fapl.add(HighFive::MPIOCollectiveMetadata{});
+    HighFive::File file = HighFive::File(filename, HighFive::File::Truncate, fapl);
+
+    const size_t initial_connectivity_id = GlobalVariableManager::instance().getConnectivityId();
+
+    SECTION("Connectivity")
+    {
+      using R1   = TinyVector<1>;
+      using R2   = TinyVector<2>;
+      using R3   = TinyVector<3>;
+      using R1x1 = TinyMatrix<1>;
+      using R2x2 = TinyMatrix<2>;
+      using R3x3 = TinyMatrix<3>;
+
+      HighFive::Group checkpoint_group   = file.createGroup("checkpoint");
+      HighFive::Group symbol_table_group = checkpoint_group.createGroup("symbol_table");
+
+      auto mesh_1d = MeshDataBaseForTests::get().unordered1DMesh()->get<Mesh<1>>();
+
+      NodeArrayPerCell<bool> node_B_per_cell_1d{mesh_1d->connectivity(), 2};
+      for (CellId cell_id = 0; cell_id < mesh_1d->numberOfCells(); ++cell_id) {
+        auto node_list = mesh_1d->connectivity().cellToNodeMatrix()[cell_id];
+        for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
+          for (size_t i = 0; i < node_B_per_cell_1d.sizeOfArrays(); ++i) {
+            node_B_per_cell_1d[cell_id][i_node][i] = (std::rand() / (RAND_MAX / mesh_1d->numberOfCells())) % 2;
+          }
+        }
+      }
+
+      NodeArrayPerCell<uint64_t> node_N_per_cell_1d{mesh_1d->connectivity(), 3};
+      for (CellId cell_id = 0; cell_id < mesh_1d->numberOfCells(); ++cell_id) {
+        auto node_list = mesh_1d->connectivity().cellToNodeMatrix()[cell_id];
+        for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
+          for (size_t i = 0; i < node_N_per_cell_1d.sizeOfArrays(); ++i) {
+            node_N_per_cell_1d[cell_id][i_node][i] = (std::rand() / (RAND_MAX / mesh_1d->numberOfCells()));
+          }
+        }
+      }
+
+      CellArrayPerNode<int64_t> cell_Z_per_node_1d{mesh_1d->connectivity(), 2};
+      for (NodeId node_id = 0; node_id < mesh_1d->numberOfNodes(); ++node_id) {
+        auto cell_list = mesh_1d->connectivity().nodeToCellMatrix()[node_id];
+        for (size_t i_cell = 0; i_cell < cell_list.size(); ++i_cell) {
+          for (size_t i = 0; i < cell_Z_per_node_1d.sizeOfArrays(); ++i) {
+            cell_Z_per_node_1d[node_id][i_cell][i] =
+              100 * (std::rand() - RAND_MAX / 2.) / (RAND_MAX / mesh_1d->numberOfNodes());
+          }
+        }
+      }
+
+      CellArrayPerNode<double> cell_R_per_node_1d{mesh_1d->connectivity(), 3};
+      for (NodeId node_id = 0; node_id < mesh_1d->numberOfNodes(); ++node_id) {
+        auto cell_list = mesh_1d->connectivity().nodeToCellMatrix()[node_id];
+        for (size_t i_cell = 0; i_cell < cell_list.size(); ++i_cell) {
+          for (size_t i = 0; i < cell_R_per_node_1d.sizeOfArrays(); ++i) {
+            cell_R_per_node_1d[node_id][i_cell][i] = std::rand() / (1. * RAND_MAX / mesh_1d->numberOfNodes());
+          }
+        }
+      }
+
+      NodeArrayPerCell<R1> node_R1_per_cell_1d{mesh_1d->connectivity(), 4};
+      for (CellId cell_id = 0; cell_id < mesh_1d->numberOfCells(); ++cell_id) {
+        auto node_list = mesh_1d->connectivity().cellToNodeMatrix()[cell_id];
+        for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
+          for (size_t i = 0; i < node_R1_per_cell_1d.sizeOfArrays(); ++i) {
+            node_R1_per_cell_1d[cell_id][i_node][i] = R1{std::rand() / (1. * RAND_MAX / mesh_1d->numberOfCells())};
+          }
+        }
+      }
+
+      CellArrayPerNode<R2> cell_R2_per_node_1d{mesh_1d->connectivity(), 2};
+      for (NodeId node_id = 0; node_id < mesh_1d->numberOfNodes(); ++node_id) {
+        auto cell_list = mesh_1d->connectivity().nodeToCellMatrix()[node_id];
+        for (size_t i_cell = 0; i_cell < cell_list.size(); ++i_cell) {
+          for (size_t i = 0; i < cell_R2_per_node_1d.sizeOfArrays(); ++i) {
+            cell_R2_per_node_1d[node_id][i_cell][i] = R2{std::rand() / (1. * RAND_MAX / mesh_1d->numberOfNodes()),
+                                                         std::rand() / (1. * RAND_MAX / mesh_1d->numberOfNodes())};
+          }
+        }
+      }
+
+      auto mesh_2d = MeshDataBaseForTests::get().hybrid2DMesh()->get<Mesh<2>>();
+
+      CellArrayPerFace<R3> cell_R3_per_face_2d{mesh_2d->connectivity(), 3};
+      for (FaceId face_id = 0; face_id < mesh_2d->numberOfFaces(); ++face_id) {
+        auto cell_list = mesh_2d->connectivity().faceToCellMatrix()[face_id];
+        for (size_t i_cell = 0; i_cell < cell_list.size(); ++i_cell) {
+          for (size_t i = 0; i < cell_R3_per_face_2d.sizeOfArrays(); ++i) {
+            cell_R3_per_face_2d[face_id][i_cell][i] = R3{std::rand() / (1. * RAND_MAX / mesh_2d->numberOfFaces()),
+                                                         std::rand() / (1. * RAND_MAX / mesh_2d->numberOfFaces()),
+                                                         std::rand() / (1. * RAND_MAX / mesh_2d->numberOfFaces())};
+          }
+        }
+      }
+
+      FaceArrayPerNode<R2x2> face_R2x2_per_node_2d{mesh_2d->connectivity(), 3};
+      for (NodeId node_id = 0; node_id < mesh_2d->numberOfNodes(); ++node_id) {
+        auto face_list = mesh_2d->connectivity().nodeToFaceMatrix()[node_id];
+        for (size_t i_face = 0; i_face < face_list.size(); ++i_face) {
+          for (size_t i = 0; i < face_R2x2_per_node_2d.sizeOfArrays(); ++i) {
+            face_R2x2_per_node_2d[node_id][i_face][i] = R2x2{std::rand() / (1. * RAND_MAX / mesh_2d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_2d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_2d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_2d->numberOfNodes())};
+          }
+        }
+      }
+
+      auto mesh_3d = MeshDataBaseForTests::get().hybrid3DMesh()->get<Mesh<3>>();
+
+      FaceArrayPerEdge<R3> face_R3_per_edge_3d{mesh_3d->connectivity(), 2};
+      for (EdgeId edge_id = 0; edge_id < mesh_3d->numberOfEdges(); ++edge_id) {
+        auto face_list = mesh_3d->connectivity().edgeToFaceMatrix()[edge_id];
+        for (size_t i_face = 0; i_face < face_list.size(); ++i_face) {
+          for (size_t i = 0; i < face_R3_per_edge_3d.sizeOfArrays(); ++i) {
+            face_R3_per_edge_3d[edge_id][i_face][i] = R3{std::rand() / (1. * RAND_MAX / mesh_3d->numberOfEdges()),
+                                                         std::rand() / (1. * RAND_MAX / mesh_3d->numberOfEdges()),
+                                                         std::rand() / (1. * RAND_MAX / mesh_3d->numberOfEdges())};
+          }
+        }
+      }
+
+      EdgeArrayPerFace<R1x1> edge_R1x1_per_face_3d{mesh_3d->connectivity(), 3};
+      for (FaceId face_id = 0; face_id < mesh_3d->numberOfFaces(); ++face_id) {
+        auto edge_list = mesh_3d->connectivity().faceToEdgeMatrix()[face_id];
+        for (size_t i_edge = 0; i_edge < edge_list.size(); ++i_edge) {
+          for (size_t i = 0; i < edge_R1x1_per_face_3d.sizeOfArrays(); ++i) {
+            edge_R1x1_per_face_3d[face_id][i_edge][i] = R1x1{std::rand() / (1. * RAND_MAX / mesh_3d->numberOfFaces())};
+          }
+        }
+      }
+
+      EdgeArrayPerNode<R3x3> edge_R3x3_per_node_3d{mesh_3d->connectivity(), 4};
+      for (NodeId node_id = 0; node_id < mesh_3d->numberOfNodes(); ++node_id) {
+        auto edge_list = mesh_3d->connectivity().nodeToEdgeMatrix()[node_id];
+        for (size_t i_edge = 0; i_edge < edge_list.size(); ++i_edge) {
+          for (size_t i = 0; i < edge_R3x3_per_node_3d.sizeOfArrays(); ++i) {
+            edge_R3x3_per_node_3d[node_id][i_edge][i] = R3x3{std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes()),
+                                                             std::rand() / (1. * RAND_MAX / mesh_3d->numberOfNodes())};
+          }
+        }
+      }
+
+      {   // Write
+        using DataHandlerT = DataHandler<const SubItemArrayPerItemVariant>;
+
+        auto new_connectivity_1d = test_only::duplicateConnectivity(mesh_1d->connectivity());
+
+        NodeArrayPerCell<const bool> node_B_per_cell_1d_new{*new_connectivity_1d, node_B_per_cell_1d.tableView()};
+        CellArrayPerNode<const int64_t> cell_Z_per_node_1d_new{*new_connectivity_1d, cell_Z_per_node_1d.tableView()};
+        NodeArrayPerCell<const uint64_t> node_N_per_cell_1d_new{*new_connectivity_1d, node_N_per_cell_1d.tableView()};
+        CellArrayPerNode<const double> cell_R_per_node_1d_new{*new_connectivity_1d, cell_R_per_node_1d.tableView()};
+        NodeArrayPerCell<const R1> node_R1_per_cell_1d_new{*new_connectivity_1d, node_R1_per_cell_1d.tableView()};
+        CellArrayPerNode<const R2> cell_R2_per_node_1d_new{*new_connectivity_1d, cell_R2_per_node_1d.tableView()};
+
+        checkpointing::writeSubItemArrayPerItemVariant("node_B_per_cell_1d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           node_B_per_cell_1d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("node_N_per_cell_1d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           node_N_per_cell_1d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("cell_Z_per_node_1d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           cell_Z_per_node_1d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("cell_R_per_node_1d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           cell_R_per_node_1d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("node_R1_per_cell_1d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           node_R1_per_cell_1d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("cell_R2_per_node_1d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           cell_R2_per_node_1d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        auto new_connectivity_2d = test_only::duplicateConnectivity(mesh_2d->connectivity());
+
+        CellArrayPerFace<const R3> cell_R3_per_face_2d_new{*new_connectivity_2d, cell_R3_per_face_2d.tableView()};
+        FaceArrayPerNode<const R2x2> face_R2x2_per_node_2d_new{*new_connectivity_2d, face_R2x2_per_node_2d.tableView()};
+
+        checkpointing::writeSubItemArrayPerItemVariant("cell_R3_per_face_2d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           cell_R3_per_face_2d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("face_R2x2_per_node_2d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           face_R2x2_per_node_2d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        auto new_connectivity_3d = test_only::duplicateConnectivity(mesh_3d->connectivity());
+
+        FaceArrayPerEdge<const R3> face_R3_per_edge_3d_new{*new_connectivity_3d, face_R3_per_edge_3d.tableView()};
+        EdgeArrayPerFace<const R1x1> edge_R1x1_per_face_3d_new{*new_connectivity_3d, edge_R1x1_per_face_3d.tableView()};
+        EdgeArrayPerNode<const R3x3> edge_R3x3_per_node_3d_new{*new_connectivity_3d, edge_R3x3_per_node_3d.tableView()};
+
+        checkpointing::writeSubItemArrayPerItemVariant("face_R3_per_edge_3d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           face_R3_per_edge_3d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("edge_R1x1_per_face_3d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           edge_R1x1_per_face_3d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        checkpointing::writeSubItemArrayPerItemVariant("edge_R3x3_per_node_3d",
+                                                       EmbeddedData{std::make_shared<DataHandlerT>(
+                                                         std::make_shared<const SubItemArrayPerItemVariant>(
+                                                           edge_R3x3_per_node_3d_new))},
+                                                       file, checkpoint_group, symbol_table_group);
+
+        HighFive::Group global_variables_group = checkpoint_group.createGroup("singleton/global_variables");
+        global_variables_group.createAttribute("connectivity_id",
+                                               GlobalVariableManager::instance().getConnectivityId());
+        global_variables_group.createAttribute("mesh_id", GlobalVariableManager::instance().getMeshId());
+      }
+
+      // reset to reuse after resuming
+      GlobalVariableManager::instance().setConnectivityId(initial_connectivity_id);
+
+      file.flush();
+
+      checkpointing::ResumingData::create();
+      checkpointing::ResumingData::instance().readData(checkpoint_group, nullptr);
+
+      GlobalVariableManager::instance().setConnectivityId(initial_connectivity_id);
+      {   // Read
+        auto e_cell_B_per_node_1d =
+          checkpointing::readSubItemArrayPerItemVariant("node_B_per_cell_1d", symbol_table_group);
+        test_only::check_is_same(node_B_per_cell_1d, e_cell_B_per_node_1d);
+
+        auto e_cell_N_per_node_1d =
+          checkpointing::readSubItemArrayPerItemVariant("node_N_per_cell_1d", symbol_table_group);
+        test_only::check_is_same(node_N_per_cell_1d, e_cell_N_per_node_1d);
+
+        auto e_node_Z_1d = checkpointing::readSubItemArrayPerItemVariant("cell_Z_per_node_1d", symbol_table_group);
+        test_only::check_is_same(cell_Z_per_node_1d, e_node_Z_1d);
+
+        auto e_node_R_1d = checkpointing::readSubItemArrayPerItemVariant("cell_R_per_node_1d", symbol_table_group);
+        test_only::check_is_same(cell_R_per_node_1d, e_node_R_1d);
+
+        auto e_cell_R1_1d = checkpointing::readSubItemArrayPerItemVariant("node_R1_per_cell_1d", symbol_table_group);
+        test_only::check_is_same(node_R1_per_cell_1d, e_cell_R1_1d);
+
+        auto e_node_R2_1d = checkpointing::readSubItemArrayPerItemVariant("cell_R2_per_node_1d", symbol_table_group);
+        test_only::check_is_same(cell_R2_per_node_1d, e_node_R2_1d);
+
+        auto e_face_R3_2d = checkpointing::readSubItemArrayPerItemVariant("cell_R3_per_face_2d", symbol_table_group);
+        test_only::check_is_same(cell_R3_per_face_2d, e_face_R3_2d);
+
+        auto e_node_R2x2_2d =
+          checkpointing::readSubItemArrayPerItemVariant("face_R2x2_per_node_2d", symbol_table_group);
+        test_only::check_is_same(face_R2x2_per_node_2d, e_node_R2x2_2d);
+
+        auto e_edge_R3_3d = checkpointing::readSubItemArrayPerItemVariant("face_R3_per_edge_3d", symbol_table_group);
+        test_only::check_is_same(face_R3_per_edge_3d, e_edge_R3_3d);
+
+        auto e_face_R1x1_3d =
+          checkpointing::readSubItemArrayPerItemVariant("edge_R1x1_per_face_3d", symbol_table_group);
+        test_only::check_is_same(edge_R1x1_per_face_3d, e_face_R1x1_3d);
+
+        auto e_node_R3x3_3d =
+          checkpointing::readSubItemArrayPerItemVariant("edge_R3x3_per_node_3d", symbol_table_group);
+        test_only::check_is_same(edge_R3x3_per_node_3d, e_node_R3x3_3d);
+      }
+      checkpointing::ResumingData::destroy();
+    }
+  }
+
+  parallel::barrier();
+  if (parallel::rank() == 0) {
+    std::filesystem::remove_all(std::filesystem::path{tmp_dirname});
+  }
+}
-- 
GitLab