Skip to content
Snippets Groups Projects
Select Git revision
  • 437955468c428b3c2a892ee192e35957cb117845
  • develop default protected
  • feature/advection
  • feature/composite-scheme-other-fluxes
  • origin/stage/bouguettaia
  • save_clemence
  • feature/local-dt-fsi
  • feature/variational-hydro
  • feature/gmsh-reader
  • feature/reconstruction
  • feature/kinetic-schemes
  • feature/composite-scheme-sources
  • feature/serraille
  • feature/composite-scheme
  • hyperplastic
  • feature/polynomials
  • feature/gks
  • feature/implicit-solver-o2
  • feature/coupling_module
  • feature/implicit-solver
  • feature/merge-local-dt-fsi
  • v0.5.0 protected
  • v0.4.1 protected
  • v0.4.0 protected
  • v0.3.0 protected
  • v0.2.0 protected
  • v0.1.0 protected
  • Kidder
  • v0.0.4 protected
  • v0.0.3 protected
  • v0.0.2 protected
  • v0 protected
  • v0.0.1 protected
33 results

test_ParallelChecker_read.cpp

Blame
  • test_ParallelChecker_read.cpp 83.51 KiB
    #include <catch2/catch_test_macros.hpp>
    #include <catch2/matchers/catch_matchers_all.hpp>
    
    #include <dev/ParallelChecker.hpp>
    
    #include <MeshDataBaseForTests.hpp>
    
    #include <mesh/ItemArrayUtils.hpp>
    #include <mesh/SubItemArrayPerItemUtils.hpp>
    #include <mesh/SubItemValuePerItemUtils.hpp>
    
    #include <filesystem>
    
    // clazy:excludeall=non-pod-global-static
    
    #ifdef PUGS_HAS_HDF5
    
    #include <ParallelCheckerTester.hpp>
    
    template <typename T>
    struct test_TinyVectorDataType;
    
    template <size_t Dimension, typename DataT>
    struct test_TinyVectorDataType<TinyVector<Dimension, DataT>> : public HighFive::DataType
    {
      test_TinyVectorDataType()
      {
        hsize_t dim[]     = {Dimension};
        auto h5_data_type = HighFive::create_datatype<DataT>();
        _hid              = H5Tarray_create(h5_data_type.getId(), 1, dim);
      }
    };
    
    template <typename T>
    struct test_TinyMatrixDataType;
    
    template <size_t M, size_t N, typename DataT>
    struct test_TinyMatrixDataType<TinyMatrix<M, N, DataT>> : public HighFive::DataType
    {
      test_TinyMatrixDataType()
      {
        hsize_t dim[]     = {M, N};
        auto h5_data_type = HighFive::create_datatype<DataT>();
        _hid              = H5Tarray_create(h5_data_type.getId(), 2, dim);
      }
    };
    
    TEST_CASE("ParallelChecker_read", "[dev]")
    {
      {
        ParallelCheckerTester pc_tester;
        if (pc_tester.isCreated()) {
          REQUIRE_NOTHROW(ParallelChecker::destroy());
        }
      }
      REQUIRE_NOTHROW(ParallelChecker::create());
      REQUIRE_NOTHROW(ParallelChecker::instance().setMode(ParallelChecker::Mode::read));
    
      std::string tmp_dirname;
    
      {
        if (parallel::rank() == 0) {
          tmp_dirname = [&]() -> std::string {
            std::string temp_filename = std::filesystem::temp_directory_path() / "pugs_test_read_h5_XXXXXX";
            return std::string{mkdtemp(&temp_filename[0])};
          }();
        }
        parallel::broadcast(tmp_dirname, 0);
    
        std::filesystem::path path = tmp_dirname;
        REQUIRE_NOTHROW(ParallelChecker::instance().setFilename(path / "parallel_check.h5"));
      }
    
      auto get_item_numbers = []<typename ConnectivityT>(const ConnectivityT& connectivity, ItemType item_type) {
        Array<const int> number;
        Array<const bool> is_owned;
    
        switch (item_type) {
        case ItemType::cell: {
          number   = connectivity.cellNumber().arrayView();
          is_owned = connectivity.cellIsOwned().arrayView();
          break;
        }
        case ItemType::face: {
          number   = connectivity.faceNumber().arrayView();
          is_owned = connectivity.faceIsOwned().arrayView();
          break;
        }
        case ItemType::edge: {
          number   = connectivity.edgeNumber().arrayView();
          is_owned = connectivity.edgeIsOwned().arrayView();
          break;
        }
        case ItemType::node: {
          number   = connectivity.nodeNumber().arrayView();
          is_owned = connectivity.nodeIsOwned().arrayView();
          break;
        }
        }
    
        if (parallel::size() > 1) {
          const size_t nb_local_item = [is_owned]() {
            size_t count = 0;
            for (size_t i = 0; i < is_owned.size(); ++i) {
              count += is_owned[i];
            }
            return count;
          }();
    
          Array<int> owned_number{nb_local_item};
          for (size_t i = 0, l = 0; i < is_owned.size(); ++i) {
            if (is_owned[i]) {
              owned_number[l++] = number[i];
            }
          }
    
          number = parallel::allGatherVariable(owned_number);
        }
    
        return number;
      };
    
      auto get_subitem_rows_map =
        []<typename ConnectivityT>(const ConnectivityT& connectivity, ItemType item_type,
                                   ItemType subitem_type) -> Array<const typename ConnectivityMatrix::IndexType> {
        Array rows_map = connectivity.getMatrix(item_type, subitem_type).rowsMap();
    
        if (parallel::size() == 1) {
          return rows_map;
        } else {
          Array<const bool> is_owned;
          switch (item_type) {
          case ItemType::cell: {
            is_owned = connectivity.cellIsOwned().arrayView();
            break;
          }
          case ItemType::face: {
            is_owned = connectivity.faceIsOwned().arrayView();
            break;
          }
          case ItemType::edge: {
            is_owned = connectivity.edgeIsOwned().arrayView();
            break;
          }
          case ItemType::node: {
            is_owned = connectivity.nodeIsOwned().arrayView();
            break;
          }
          }
    
          Array<size_t> nb_subitem_per_item(rows_map.size() - 1);
    
          for (size_t i = 0; i < nb_subitem_per_item.size(); ++i) {
            nb_subitem_per_item[i] = rows_map[i + 1] - rows_map[i];
          }
    
          const size_t nb_local_item = [is_owned]() {
            size_t count = 0;
            for (size_t i = 0; i < is_owned.size(); ++i) {
              count += is_owned[i];
            }
            return count;
          }();
    
          {
            Array<size_t> owned_nb_subitem_per_item{nb_local_item};
            for (size_t i = 0, l = 0; i < is_owned.size(); ++i) {
              if (is_owned[i]) {
                owned_nb_subitem_per_item[l++] = nb_subitem_per_item[i];
              }
            }
            nb_subitem_per_item = parallel::allGatherVariable(owned_nb_subitem_per_item);
          }
    
          Array<typename ConnectivityMatrix::IndexType> global_rows_map{nb_subitem_per_item.size() + 1};
          global_rows_map[0] = 0;
          for (size_t i = 0; i < nb_subitem_per_item.size(); ++i) {
            global_rows_map[i + 1] = global_rows_map[i] + nb_subitem_per_item[i];
          }
          return global_rows_map;
        }
      };
    
      SECTION("check parallel write implementation")
      {
        if (parallel::size() == 1) {
          REQUIRE_NOTHROW(ParallelChecker::instance().setMode(ParallelChecker::Mode::write));
        } else {
          REQUIRE_THROWS_WITH(ParallelChecker::instance().setMode(ParallelChecker::Mode::write),
                              "not implemented yet: parallel check write in parallel");
        }
    
        ParallelCheckerTester pc_tester;
        pc_tester.setMode(ParallelChecker::Mode::automatic);
        REQUIRE(ParallelChecker::instance().isWriting() == (parallel::size() == 1));
      }
    
      SECTION("check ItemValue/ItemArray")
      {
        // ItemValues
        {   // 1d
          auto mesh                           = MeshDataBaseForTests::get().unordered1DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<1>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          Array numbers = get_item_numbers(connectivity, ItemType::cell);
    
          int tag = 12;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Array<double> values{numbers.size()};
            for (size_t i = 0; i < numbers.size(); ++i) {
              values[i] = std::sin(numbers[i]);
            }
            group.createDataSet<double>(name, HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<double>(&(values[0]));
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::cell)});
            group.createAttribute("data_type", demangle<double>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          CellValue<double> values{connectivity};
          CellValue<const int> cell_number = connectivity.cellNumber();
          for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
            values[cell_id] = std::sin(cell_number[cell_id]);
          }
    
          REQUIRE_NOTHROW(parallel_check(values, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(values, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            CellValue<double> not_sync     = copy(values);
            CellValue<const bool> is_owned = connectivity.cellIsOwned();
            if (parallel::rank() == 0) {
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (not is_owned[cell_id]) {
                  not_sync[cell_id] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            CellValue<double> different = copy(values);
            bool has_difference         = false;
            if (parallel::rank() == 0) {
              CellValue<const bool> is_owned = connectivity.cellIsOwned();
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (is_owned[cell_id]) {
                  different[cell_id] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            CellValue<int> other_data_type{connectivity};
            other_data_type.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArray<double> arrays{connectivity, 1};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArray<double> arrays{connectivity, 2};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("dimension").write(size_t{2});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("dimension").write(connectivity.dimension());
            }
            parallel::barrier();
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("item_type").write(std::string{itemName(ItemType::node)});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("item_type").write(std::string{itemName(ItemType::cell)});
            }
            parallel::barrier();
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian1DMesh();
            const Connectivity<1>& other_connectivity = other_mesh->connectivity();
    
            CellValue<double> other_shape{other_connectivity};
            other_shape.fill(1);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: some item numbers are not defined in reference");
          }
        }
    
        // ItemArray
        {   // 1d
          auto mesh                           = MeshDataBaseForTests::get().unordered1DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<1>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          Array numbers = get_item_numbers(connectivity, ItemType::cell);
    
          int tag = 12;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Table<double> arrays{numbers.size(), 2};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t j = 0; j < 2; ++j) {
                arrays[i][j] = std::sin(2 * numbers[i] + j);
              }
            }
            group
              .createDataSet<double>(name, HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(),
                                                                                   arrays.numberOfColumns()}})
              .write_raw<double>(&(arrays(0, 0)));
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::cell)});
            group.createAttribute("data_type", demangle<double>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          CellArray<double> arrays{connectivity, 2};
          CellValue<const int> cell_number = connectivity.cellNumber();
          for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
            for (size_t j = 0; j < 2; ++j) {
              arrays[cell_id][j] = std::sin(2 * cell_number[cell_id] + j);
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            CellArray<double> not_sync     = copy(arrays);
            CellValue<const bool> is_owned = connectivity.cellIsOwned();
            if (parallel::rank() == 0) {
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (not is_owned[cell_id]) {
                  not_sync[cell_id][0] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            CellArray<double> different = copy(arrays);
            bool has_difference         = false;
            if (parallel::rank() == 0) {
              CellValue<const bool> is_owned = connectivity.cellIsOwned();
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (is_owned[cell_id]) {
                  different[cell_id][0] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            CellValue<int> other_data_type{connectivity};
            other_data_type.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArray<double> arrays{connectivity, 1};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian1DMesh();
            const Connectivity<1>& other_connectivity = other_mesh->connectivity();
    
            CellArray<double> other_shape{other_connectivity, 2};
            other_shape.fill(1);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: some item numbers are not defined in reference");
          }
        }
    
        // ItemValues
        {   // 2d
          auto mesh                           = MeshDataBaseForTests::get().hybrid2DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<2>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          Array numbers = get_item_numbers(connectivity, ItemType::node);
    
          using DataType = TinyVector<3>;
    
          int tag = 9;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Array<DataType> values{numbers.size()};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t j = 0; j < DataType::Dimension; ++j) {
                values[i][j] = std::sin(numbers[i] + j);
              }
            }
            group
              .createDataSet(name, HighFive::DataSpace{std::vector<size_t>{numbers.size()}},
                             test_TinyVectorDataType<DataType>{})
              .template write_raw<double>(&(values[0][0]), test_TinyVectorDataType<DataType>{});
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::node)});
            group.createAttribute("data_type", demangle<DataType>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          NodeValue<DataType> values{connectivity};
          NodeValue<const int> node_number = connectivity.nodeNumber();
          for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
            for (size_t j = 0; j < DataType::Dimension; ++j) {
              values[node_id][j] = std::sin(node_number[node_id] + j);
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(values, "sin", source_location));
    
          if (parallel::size() > 1) {
            NodeValue<DataType> not_sync   = copy(values);
            NodeValue<const bool> is_owned = connectivity.nodeIsOwned();
            if (parallel::rank() == 0) {
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (not is_owned[node_id]) {
                  not_sync[node_id][0] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            NodeValue<DataType> different = copy(values);
            bool has_difference           = false;
            if (parallel::rank() == 0) {
              NodeValue<const bool> is_owned = connectivity.nodeIsOwned();
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (is_owned[node_id]) {
                  different[node_id][0] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
        }
    
        // ItemArray
        {   // 2d
          auto mesh                           = MeshDataBaseForTests::get().hybrid2DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<2>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          using DataType = TinyMatrix<3, 2>;
    
          Array numbers = get_item_numbers(connectivity, ItemType::face);
    
          int tag = 12;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Table<DataType> arrays{numbers.size(), 2};
            for (size_t i = 0; i < arrays.numberOfRows(); ++i) {
              for (size_t j = 0; j < arrays.numberOfColumns(); ++j) {
                for (size_t k = 0; k < DataType::NumberOfRows; ++k) {
                  for (size_t l = 0; l < DataType::NumberOfColumns; ++l) {
                    arrays[i][j](k, l) = std::sin(2 * numbers[i] + j + 3 * k + 2 * l);
                  }
                }
              }
            }
            group
              .createDataSet(name,
                             HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(), arrays.numberOfColumns()}},
                             test_TinyMatrixDataType<DataType>{})
              .template write_raw<double>(&(arrays[0][0](0, 0)), test_TinyMatrixDataType<DataType>{});
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::face)});
            group.createAttribute("data_type", demangle<DataType>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          FaceArray<DataType> arrays{connectivity, 2};
          FaceValue<const int> face_number = connectivity.faceNumber();
          for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
            for (size_t j = 0; j < arrays.sizeOfArrays(); ++j) {
              for (size_t k = 0; k < DataType::NumberOfRows; ++k) {
                for (size_t l = 0; l < DataType::NumberOfColumns; ++l) {
                  arrays[face_id][j](k, l) = std::sin(2 * face_number[face_id] + j + 3 * k + 2 * l);
                }
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            FaceArray<DataType> not_sync   = copy(arrays);
            FaceValue<const bool> is_owned = connectivity.faceIsOwned();
            if (parallel::rank() == 0) {
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (not is_owned[face_id]) {
                  not_sync[face_id][0](0, 0) += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            FaceArray<DataType> different = copy(arrays);
            bool has_difference           = false;
            if (parallel::rank() == 0) {
              FaceValue<const bool> is_owned = connectivity.faceIsOwned();
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (is_owned[face_id]) {
                  different[face_id][0](0, 0) += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            FaceValue<TinyVector<6>> other_data_type{connectivity};
            other_data_type.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            FaceArray<DataType> arrays{connectivity, 1};
            arrays.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian2DMesh();
            const Connectivity<2>& other_connectivity = other_mesh->connectivity();
    
            FaceArray<DataType> other_shape{other_connectivity, 2};
            other_shape.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: number of items differs from reference");
          }
        }
    
        // ItemValues
        {   // 3d
          auto mesh                           = MeshDataBaseForTests::get().hybrid3DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<3>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          Array numbers = get_item_numbers(connectivity, ItemType::node);
    
          using DataType = TinyMatrix<2, 3>;
    
          int tag = 9;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Array<DataType> values{numbers.size()};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t j = 0; j < DataType::NumberOfRows; ++j) {
                for (size_t k = 0; k < DataType::NumberOfColumns; ++k) {
                  values[i](j, k) = std::sin(numbers[i] + j + 2 * k);
                }
              }
            }
            group
              .createDataSet(name, HighFive::DataSpace{std::vector<size_t>{numbers.size()}},
                             test_TinyMatrixDataType<DataType>{})
              .template write_raw<double>(&(values[0](0, 0)), test_TinyMatrixDataType<DataType>{});
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::node)});
            group.createAttribute("data_type", demangle<DataType>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          NodeValue<DataType> values{connectivity};
          NodeValue<const int> node_number = connectivity.nodeNumber();
          for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
            for (size_t j = 0; j < DataType::NumberOfRows; ++j) {
              for (size_t k = 0; k < DataType::NumberOfColumns; ++k) {
                values[node_id](j, k) = std::sin(node_number[node_id] + j + 2 * k);
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(values, "sin", source_location));
    
          if (parallel::size() > 1) {
            NodeValue<DataType> not_sync   = copy(values);
            NodeValue<const bool> is_owned = connectivity.nodeIsOwned();
            if (parallel::rank() == 0) {
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (not is_owned[node_id]) {
                  not_sync[node_id](0, 0) += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            NodeValue<DataType> different = copy(values);
            bool has_difference           = false;
            if (parallel::rank() == 0) {
              NodeValue<const bool> is_owned = connectivity.nodeIsOwned();
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (is_owned[node_id]) {
                  different[node_id](0, 0) += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
        }
    
        // ItemArray
        {   // 3d
          auto mesh                           = MeshDataBaseForTests::get().hybrid3DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<3>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          using DataType = TinyVector<2>;
    
          Array numbers = get_item_numbers(connectivity, ItemType::face);
    
          int tag = 7;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Table<DataType> arrays{numbers.size(), 2};
            for (size_t i = 0; i < arrays.numberOfRows(); ++i) {
              for (size_t j = 0; j < arrays.numberOfColumns(); ++j) {
                for (size_t k = 0; k < DataType::Dimension; ++k) {
                  arrays[i][j][k] = std::sin(2 * numbers[i] + j + 3 * k);
                }
              }
            }
            group
              .createDataSet(name,
                             HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(), arrays.numberOfColumns()}},
                             test_TinyVectorDataType<DataType>{})
              .template write_raw<double>(&(arrays[0][0][0]), test_TinyVectorDataType<DataType>{});
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::face)});
            group.createAttribute("data_type", demangle<DataType>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          FaceArray<DataType> arrays{connectivity, 2};
          FaceValue<const int> face_number = connectivity.faceNumber();
          for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
            for (size_t j = 0; j < arrays.sizeOfArrays(); ++j) {
              for (size_t k = 0; k < DataType::Dimension; ++k) {
                arrays[face_id][j][k] = std::sin(2 * face_number[face_id] + j + 3 * k);
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            FaceArray<DataType> not_sync   = copy(arrays);
            FaceValue<const bool> is_owned = connectivity.faceIsOwned();
            if (parallel::rank() == 0) {
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (not is_owned[face_id]) {
                  not_sync[face_id][0][0] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            FaceArray<DataType> different = copy(arrays);
            bool has_difference           = false;
            if (parallel::rank() == 0) {
              FaceValue<const bool> is_owned = connectivity.faceIsOwned();
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (is_owned[face_id]) {
                  different[face_id][0][0] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            FaceValue<TinyVector<6>> other_data_type{connectivity};
            other_data_type.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            FaceArray<DataType> arrays{connectivity, 1};
            arrays.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian2DMesh();
            const Connectivity<2>& other_connectivity = other_mesh->connectivity();
    
            FaceArray<DataType> other_shape{other_connectivity, 2};
            other_shape.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location), "error: cannot compare data");
          }
        }
      }
    
      SECTION("check SubItemValuePerItem/SubItemArrayPerItem")
      {
        // SubItemValuePerItem
        {   // 1d
          auto mesh                           = MeshDataBaseForTests::get().unordered1DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<1>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
          Array numbers = get_item_numbers(connectivity, ItemType::node);
          Array<const typename ConnectivityMatrix::IndexType> rows_map =
            get_subitem_rows_map(connectivity, ItemType::node, ItemType::cell);
    
          int tag = 6;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
            group
              .createDataSet<typename ConnectivityMatrix::IndexType>("rows_map", HighFive::DataSpace{std::vector<size_t>{
                                                                                   rows_map.size()}})
              .write_raw<typename ConnectivityMatrix::IndexType>(&(rows_map[0]));
    
            Array<double> values{rows_map[rows_map.size() - 1]};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t i_row = rows_map[i]; i_row < rows_map[i + 1]; ++i_row) {
                const size_t j = i_row - rows_map[i];
                values[i_row]  = std::sin(numbers[i] + 2 * j);
              }
            }
    
            group.createDataSet<double>(name, HighFive::DataSpace{std::vector<size_t>{values.size()}})
              .write_raw<double>(&(values[0]));
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::node)});
            group.createAttribute("subitem_type", std::string{itemName(ItemType::cell)});
            group.createAttribute("data_type", demangle<double>());
          }
    
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          auto node_to_cell_matrix = connectivity.nodeToCellMatrix();
    
          CellValuePerNode<double> values{connectivity};
          NodeValue<const int> node_number = connectivity.nodeNumber();
          for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
            auto cell_list = node_to_cell_matrix[node_id];
            for (size_t i_cell = 0; i_cell < cell_list.size(); ++i_cell) {
              values[node_id][i_cell] = std::sin(node_number[node_id] + 2 * i_cell);
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(values, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(values, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            CellValuePerNode<double> not_sync = copy(values);
            NodeValue<const bool> is_owned    = connectivity.nodeIsOwned();
            if (parallel::rank() == 0) {
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (not is_owned[node_id]) {
                  not_sync[node_id][0] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            CellValuePerNode<double> different = copy(values);
            bool has_difference                = false;
            if (parallel::rank() == 0) {
              NodeValue<const bool> is_owned = connectivity.nodeIsOwned();
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (is_owned[node_id]) {
                  different[node_id][0] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            CellValuePerNode<int> other_data_type{connectivity};
            other_data_type.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArrayPerNode<double> arrays{connectivity, 1};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("dimension").write(size_t{2});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("dimension").write(connectivity.dimension());
            }
            parallel::barrier();
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("item_type").write(std::string{itemName(ItemType::face)});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("item_type").write(std::string{itemName(ItemType::node)});
            }
            parallel::barrier();
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("subitem_type").write(std::string{itemName(ItemType::face)});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("subitem_type").write(std::string{itemName(ItemType::cell)});
            }
            parallel::barrier();
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian1DMesh();
            const Connectivity<1>& other_connectivity = other_mesh->connectivity();
    
            CellValuePerNode<double> other_shape{other_connectivity};
            other_shape.fill(1);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: some item numbers are not defined in reference");
          }
        }
    
        // SubItemArrayPerItem
        {   // 1d
          auto mesh                           = MeshDataBaseForTests::get().unordered1DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<1>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          Array numbers = get_item_numbers(connectivity, ItemType::cell);
          Array<const typename ConnectivityMatrix::IndexType> rows_map =
            get_subitem_rows_map(connectivity, ItemType::cell, ItemType::node);
    
          int tag = 12;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
            group
              .createDataSet<typename ConnectivityMatrix::IndexType>("rows_map", HighFive::DataSpace{std::vector<size_t>{
                                                                                   rows_map.size()}})
              .write_raw<typename ConnectivityMatrix::IndexType>(&(rows_map[0]));
    
            Table<double> arrays{rows_map[rows_map.size() - 1], 2};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t i_row = rows_map[i]; i_row < rows_map[i + 1]; ++i_row) {
                const size_t j = i_row - rows_map[i];
                for (size_t k = 0; k < 2; ++k) {
                  arrays[i_row][k] = std::sin(2 * numbers[i] + (1 + k) * j);
                }
              }
            }
            group
              .createDataSet<double>(name, HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(),
                                                                                   arrays.numberOfColumns()}})
              .write_raw<double>(&(arrays(0, 0)));
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::cell)});
            group.createAttribute("subitem_type", std::string{itemName(ItemType::node)});
            group.createAttribute("data_type", demangle<double>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          auto cell_to_node_matrix = connectivity.cellToNodeMatrix();
    
          NodeArrayPerCell<double> arrays{connectivity, 2};
          CellValue<const int> cell_number = connectivity.cellNumber();
          for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
            for (size_t j = 0; j < cell_to_node_matrix[cell_id].size(); ++j) {
              for (size_t k = 0; k < 2; ++k) {
                arrays[cell_id][j][k] = std::sin(2 * cell_number[cell_id] + (1 + k) * j);
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            NodeArrayPerCell<double> not_sync = copy(arrays);
            CellValue<const bool> is_owned    = connectivity.cellIsOwned();
            if (parallel::rank() == 0) {
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (not is_owned[cell_id]) {
                  not_sync[cell_id][0][1] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            NodeArrayPerCell<double> different = copy(arrays);
            bool has_difference                = false;
            if (parallel::rank() == 0) {
              CellValue<const bool> is_owned = connectivity.cellIsOwned();
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (is_owned[cell_id]) {
                  different[cell_id][0][1] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            CellValue<int> other_data_type{connectivity};
            other_data_type.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArray<double> arrays{connectivity, 1};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian1DMesh();
            const Connectivity<1>& other_connectivity = other_mesh->connectivity();
    
            CellArray<double> other_shape{other_connectivity, 2};
            other_shape.fill(1);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: some item numbers are not defined in reference");
          }
        }
    
        // SubItemValuePerItem
        {   // 2d
          auto mesh                           = MeshDataBaseForTests::get().hybrid2DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<2>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          using DataType = TinyMatrix<3, 2>;
    
          Array numbers = get_item_numbers(connectivity, ItemType::face);
    
          int tag = 12;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Table<DataType> arrays{numbers.size(), 2};
            for (size_t i = 0; i < arrays.numberOfRows(); ++i) {
              for (size_t j = 0; j < arrays.numberOfColumns(); ++j) {
                for (size_t k = 0; k < DataType::NumberOfRows; ++k) {
                  for (size_t l = 0; l < DataType::NumberOfColumns; ++l) {
                    arrays[i][j](k, l) = std::sin(2 * numbers[i] + j + 3 * k + 2 * l);
                  }
                }
              }
            }
            group
              .createDataSet(name,
                             HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(), arrays.numberOfColumns()}},
                             test_TinyMatrixDataType<DataType>{})
              .template write_raw<double>(&(arrays[0][0](0, 0)), test_TinyMatrixDataType<DataType>{});
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::face)});
            group.createAttribute("data_type", demangle<DataType>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          FaceArray<DataType> arrays{connectivity, 2};
          FaceValue<const int> face_number = connectivity.faceNumber();
          for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
            for (size_t j = 0; j < arrays.sizeOfArrays(); ++j) {
              for (size_t k = 0; k < DataType::NumberOfRows; ++k) {
                for (size_t l = 0; l < DataType::NumberOfColumns; ++l) {
                  arrays[face_id][j](k, l) = std::sin(2 * face_number[face_id] + j + 3 * k + 2 * l);
                }
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            FaceArray<DataType> not_sync   = copy(arrays);
            FaceValue<const bool> is_owned = connectivity.faceIsOwned();
            if (parallel::rank() == 0) {
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (not is_owned[face_id]) {
                  not_sync[face_id][0](0, 0) += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            FaceArray<DataType> different = copy(arrays);
            bool has_difference           = false;
            if (parallel::rank() == 0) {
              FaceValue<const bool> is_owned = connectivity.faceIsOwned();
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (is_owned[face_id]) {
                  different[face_id][0](0, 0) += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            FaceValue<TinyVector<6>> other_data_type{connectivity};
            other_data_type.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            FaceArray<DataType> arrays{connectivity, 1};
            arrays.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian2DMesh();
            const Connectivity<2>& other_connectivity = other_mesh->connectivity();
    
            FaceArray<DataType> other_shape{other_connectivity, 2};
            other_shape.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: number of items differs from reference");
          }
        }
    
        // SubItemArrayPerItem
        {   // 2d
          auto mesh                           = MeshDataBaseForTests::get().hybrid2DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<2>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
          Array numbers = get_item_numbers(connectivity, ItemType::node);
          Array<const typename ConnectivityMatrix::IndexType> rows_map =
            get_subitem_rows_map(connectivity, ItemType::node, ItemType::cell);
    
          int tag = 6;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
            group
              .createDataSet<typename ConnectivityMatrix::IndexType>("rows_map", HighFive::DataSpace{std::vector<size_t>{
                                                                                   rows_map.size()}})
              .write_raw<typename ConnectivityMatrix::IndexType>(&(rows_map[0]));
    
            Array<double> values{rows_map[rows_map.size() - 1]};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t i_row = rows_map[i]; i_row < rows_map[i + 1]; ++i_row) {
                const size_t j = i_row - rows_map[i];
                values[i_row]  = std::sin(numbers[i] + 2 * j);
              }
            }
    
            group.createDataSet<double>(name, HighFive::DataSpace{std::vector<size_t>{values.size()}})
              .write_raw<double>(&(values[0]));
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::node)});
            group.createAttribute("subitem_type", std::string{itemName(ItemType::cell)});
            group.createAttribute("data_type", demangle<double>());
          }
    
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          auto node_to_cell_matrix = connectivity.nodeToCellMatrix();
    
          CellValuePerNode<double> values{connectivity};
          NodeValue<const int> node_number = connectivity.nodeNumber();
          for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
            auto cell_list = node_to_cell_matrix[node_id];
            for (size_t i_cell = 0; i_cell < cell_list.size(); ++i_cell) {
              values[node_id][i_cell] = std::sin(node_number[node_id] + 2 * i_cell);
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(values, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(values, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(values, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            CellValuePerNode<double> not_sync = copy(values);
            NodeValue<const bool> is_owned    = connectivity.nodeIsOwned();
            if (parallel::rank() == 0) {
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (not is_owned[node_id]) {
                  not_sync[node_id][0] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            CellValuePerNode<double> different = copy(values);
            bool has_difference                = false;
            if (parallel::rank() == 0) {
              NodeValue<const bool> is_owned = connectivity.nodeIsOwned();
              for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
                if (is_owned[node_id]) {
                  different[node_id][0] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            CellValuePerNode<int> other_data_type{connectivity};
            other_data_type.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArrayPerNode<double> arrays{connectivity, 1};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("dimension").write(size_t{1});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("dimension").write(connectivity.dimension());
            }
            parallel::barrier();
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("item_type").write(std::string{itemName(ItemType::face)});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("item_type").write(std::string{itemName(ItemType::node)});
            }
            parallel::barrier();
          }
    
          {
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("subitem_type").write(std::string{itemName(ItemType::face)});
            }
            parallel::barrier();
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(values, "sin", source_location), "error: cannot compare data");
    
            if (parallel::rank() == 0) {
              HighFive::File file(filename, HighFive::File::ReadWrite);
              HighFive::Group group = file.getGroup("/values/" + std::to_string(tag));
              group.getAttribute("subitem_type").write(std::string{itemName(ItemType::cell)});
            }
            parallel::barrier();
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian2DMesh();
            const Connectivity<2>& other_connectivity = other_mesh->connectivity();
    
            CellValuePerNode<double> other_shape{other_connectivity};
            other_shape.fill(1);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: some item numbers are not defined in reference");
          }
        }
    
        // SubItemValuePerItem
        {   // 3d
          auto mesh                           = MeshDataBaseForTests::get().hybrid3DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<3>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          using DataType = TinyVector<2>;
    
          Array numbers = get_item_numbers(connectivity, ItemType::face);
    
          int tag = 7;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
    
            Table<DataType> arrays{numbers.size(), 2};
            for (size_t i = 0; i < arrays.numberOfRows(); ++i) {
              for (size_t j = 0; j < arrays.numberOfColumns(); ++j) {
                for (size_t k = 0; k < DataType::Dimension; ++k) {
                  arrays[i][j][k] = std::sin(2 * numbers[i] + j + 3 * k);
                }
              }
            }
            group
              .createDataSet(name,
                             HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(), arrays.numberOfColumns()}},
                             test_TinyVectorDataType<DataType>{})
              .template write_raw<double>(&(arrays[0][0][0]), test_TinyVectorDataType<DataType>{});
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::face)});
            group.createAttribute("data_type", demangle<DataType>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          FaceArray<DataType> arrays{connectivity, 2};
          FaceValue<const int> face_number = connectivity.faceNumber();
          for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
            for (size_t j = 0; j < arrays.sizeOfArrays(); ++j) {
              for (size_t k = 0; k < DataType::Dimension; ++k) {
                arrays[face_id][j][k] = std::sin(2 * face_number[face_id] + j + 3 * k);
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            FaceArray<DataType> not_sync   = copy(arrays);
            FaceValue<const bool> is_owned = connectivity.faceIsOwned();
            if (parallel::rank() == 0) {
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (not is_owned[face_id]) {
                  not_sync[face_id][0][0] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            FaceArray<DataType> different = copy(arrays);
            bool has_difference           = false;
            if (parallel::rank() == 0) {
              FaceValue<const bool> is_owned = connectivity.faceIsOwned();
              for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
                if (is_owned[face_id]) {
                  different[face_id][0][0] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            FaceValue<TinyVector<6>> other_data_type{connectivity};
            other_data_type.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            FaceArray<DataType> arrays{connectivity, 1};
            arrays.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian2DMesh();
            const Connectivity<2>& other_connectivity = other_mesh->connectivity();
    
            FaceArray<DataType> other_shape{other_connectivity, 2};
            other_shape.fill(zero);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location), "error: cannot compare data");
          }
        }
    
        // SubItemArrayPerItem
        {   // 3d
          auto mesh                           = MeshDataBaseForTests::get().hybrid3DMesh();
          std::string filename                = ParallelChecker::instance().filename();
          const Connectivity<3>& connectivity = mesh->connectivity();
    
          const std::string name = "sin";
    
          SourceLocation source_location;
    
          Array numbers = get_item_numbers(connectivity, ItemType::cell);
          Array<const typename ConnectivityMatrix::IndexType> rows_map =
            get_subitem_rows_map(connectivity, ItemType::cell, ItemType::node);
    
          int tag = 12;
          if (parallel::rank() == 0) {
            HighFive::File file(filename, HighFive::File::Overwrite);
            HighFive::Group group = file.createGroup("/values/" + std::to_string(tag));
    
            group.createDataSet<int>("numbers", HighFive::DataSpace{std::vector<size_t>{numbers.size()}})
              .write_raw<int>(&(numbers[0]));
            group
              .createDataSet<typename ConnectivityMatrix::IndexType>("rows_map", HighFive::DataSpace{std::vector<size_t>{
                                                                                   rows_map.size()}})
              .write_raw<typename ConnectivityMatrix::IndexType>(&(rows_map[0]));
    
            Table<double> arrays{rows_map[rows_map.size() - 1], 2};
            for (size_t i = 0; i < numbers.size(); ++i) {
              for (size_t i_row = rows_map[i]; i_row < rows_map[i + 1]; ++i_row) {
                const size_t j = i_row - rows_map[i];
                for (size_t k = 0; k < 2; ++k) {
                  arrays[i_row][k] = std::sin(2 * numbers[i] + (1 + k) * j);
                }
              }
            }
            group
              .createDataSet<double>(name, HighFive::DataSpace{std::vector<size_t>{arrays.numberOfRows(),
                                                                                   arrays.numberOfColumns()}})
              .write_raw<double>(&(arrays(0, 0)));
    
            group.createAttribute("filename", source_location.filename());
            group.createAttribute("line", source_location.line());
            group.createAttribute("function", source_location.function());
            group.createAttribute("name", name);
            group.createAttribute("dimension", connectivity.dimension());
            group.createAttribute("item_type", std::string{itemName(ItemType::cell)});
            group.createAttribute("subitem_type", std::string{itemName(ItemType::node)});
            group.createAttribute("data_type", demangle<double>());
          }
          parallel::barrier();
          ParallelCheckerTester pc_tester;
          pc_tester.setTag(tag);
    
          auto cell_to_node_matrix = connectivity.cellToNodeMatrix();
    
          NodeArrayPerCell<double> arrays{connectivity, 2};
          CellValue<const int> cell_number = connectivity.cellNumber();
          for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
            for (size_t j = 0; j < cell_to_node_matrix[cell_id].size(); ++j) {
              for (size_t k = 0; k < 2; ++k) {
                arrays[cell_id][j][k] = std::sin(2 * cell_number[cell_id] + (1 + k) * j);
              }
            }
          }
    
          REQUIRE_NOTHROW(parallel_check(arrays, "sin", source_location));
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different name in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "not_sin", source_location));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source file in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{"other-source-file", source_location.line(),
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source line in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line() + 100,
                                                        source_location.column(), source_location.function()}));
    
          pc_tester.setTag(tag);
          UNSCOPED_INFO("can have different source function in ref");
          REQUIRE_NOTHROW(parallel_check(arrays, "sin",
                                         SourceLocation{source_location.filename(), source_location.line(),
                                                        source_location.column(), "foo"}));
    
          if (parallel::size() > 1) {
            NodeArrayPerCell<double> not_sync = copy(arrays);
            CellValue<const bool> is_owned    = connectivity.cellIsOwned();
            if (parallel::rank() == 0) {
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (not is_owned[cell_id]) {
                  not_sync[cell_id][0][1] += 3.2;
                  break;
                }
              }
            }
            REQUIRE(not isSynchronized(not_sync));
            pc_tester.setTag(tag);
            UNSCOPED_INFO("can have different ghost values in ref (no exception)");
            REQUIRE_NOTHROW(parallel_check(not_sync, "sin", source_location));
          }
    
          {
            NodeArrayPerCell<double> different = copy(arrays);
            bool has_difference                = false;
            if (parallel::rank() == 0) {
              CellValue<const bool> is_owned = connectivity.cellIsOwned();
              for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
                if (is_owned[cell_id]) {
                  different[cell_id][0][1] += 3.2;
                  has_difference = true;
                  break;
                }
              }
            }
            has_difference = parallel::allReduceOr(has_difference);
    
            REQUIRE(has_difference);
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(different, "sin", source_location), "error: calculations differ!");
          }
    
          {
            CellValue<int> other_data_type{connectivity};
            other_data_type.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_data_type, "sin", source_location), "error: cannot compare data");
          }
    
          {
            CellArray<double> arrays{connectivity, 1};
            arrays.fill(0);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(arrays, "sin", source_location), "error: cannot compare data");
          }
    
          {
            auto other_mesh                           = MeshDataBaseForTests::get().cartesian3DMesh();
            const Connectivity<3>& other_connectivity = other_mesh->connectivity();
    
            CellArray<double> other_shape{other_connectivity, 2};
            other_shape.fill(1);
    
            pc_tester.setTag(tag);
            REQUIRE_THROWS_WITH(parallel_check(other_shape, "sin", source_location),
                                "error: some item numbers are not defined in reference");
          }
        }
      }
    
      std::error_code err_code;
      std::filesystem::remove_all(tmp_dirname, err_code);
      // error is not handled to avoid exception throws if the directory
      // has been removed by another processor
    
      REQUIRE_NOTHROW(ParallelChecker::destroy());
    }
    
    #else   // PUGS_HAS_HDF5
    
    TEST_CASE("ParallelChecker_read", "[dev]")
    {
      REQUIRE_NOTHROW(ParallelChecker::create());
      if (parallel::size() > 1) {
        REQUIRE_THROWS_WITH(ParallelChecker::instance().setMode(ParallelChecker::Mode::write),
                            "not implemented yet: parallel check write in parallel");
      } else {
        REQUIRE_NOTHROW(ParallelChecker::instance().setMode(ParallelChecker::Mode::write));
        REQUIRE_NOTHROW(ParallelChecker::instance().isWriting());
      }
      REQUIRE_NOTHROW(ParallelChecker::instance().setMode(ParallelChecker::Mode::automatic));
      REQUIRE_NOTHROW(ParallelChecker::instance().isWriting() == (parallel::size() > 1));
      REQUIRE_NOTHROW(ParallelChecker::instance().setMode(ParallelChecker::Mode::read));
      REQUIRE_NOTHROW(not ParallelChecker::instance().isWriting());
    
      auto mesh = MeshDataBaseForTests::get().unordered1DMesh();
    
      const Connectivity<1>& connectivity = mesh->connectivity();
    
      NodeValue<double> nv{connectivity};
      REQUIRE_THROWS_WITH(parallel_check(nv, "test"), "error: parallel checker cannot be used without HDF5 support");
    
      REQUIRE_THROWS_WITH(parallel_check(ItemValueVariant{nv}, "test"),
                          "error: parallel checker cannot be used without HDF5 support");
    
      NodeArray<double> na{connectivity, 2};
      REQUIRE_THROWS_WITH(parallel_check(na, "test"), "error: parallel checker cannot be used without HDF5 support");
    
      REQUIRE_THROWS_WITH(parallel_check(ItemArrayVariant{na}, "test"),
                          "error: parallel checker cannot be used without HDF5 support");
    
      NodeValuePerCell<double> nvpc{connectivity};
      REQUIRE_THROWS_WITH(parallel_check(nvpc, "test"), "error: parallel checker cannot be used without HDF5 support");
    
      REQUIRE_THROWS_WITH(parallel_check(SubItemValuePerItemVariant{nvpc}, "test"),
                          "error: parallel checker cannot be used without HDF5 support");
    
      NodeArrayPerCell<double> napc{connectivity, 2};
      REQUIRE_THROWS_WITH(parallel_check(napc, "test"), "error: parallel checker cannot be used without HDF5 support");
    
      REQUIRE_THROWS_WITH(parallel_check(SubItemArrayPerItemVariant{napc}, "test"),
                          "error: parallel checker cannot be used without HDF5 support");
    
      REQUIRE_NOTHROW(ParallelChecker::destroy());
    }
    
    #endif   // PUGS_HAS_HDF5