From d36b8a0d3009bb8a9b0c3e6948baf9cea6607ad5 Mon Sep 17 00:00:00 2001 From: Stephane Del Pino <stephane.delpino44@gmail.com> Date: Mon, 16 Oct 2023 23:03:30 +0200 Subject: [PATCH] Add parallel checker support for SubItem{Array,Value}PerItem --- src/dev/CMakeLists.txt | 5 + src/dev/ParallelChecker.cpp | 18 - src/dev/ParallelChecker.hpp | 600 +++++++++++++++++- ...ParallelCheckerDiscreteFunctionVariant.cpp | 10 + src/dev/ParallelCheckerItemArrayVariant.cpp | 10 + src/dev/ParallelCheckerItemValueVariant.cpp | 10 + ...allelCheckerSubItemArrayPerItemVariant.cpp | 10 + ...allelCheckerSubItemValuePerItemVariant.cpp | 10 + src/language/modules/DevUtilsModule.cpp | 41 ++ src/mesh/SubItemArrayPerItem.hpp | 7 + src/mesh/SubItemArrayPerItemVariant.hpp | 4 +- src/mesh/SubItemValuePerItem.hpp | 8 + src/mesh/SubItemValuePerItemVariant.hpp | 4 +- src/utils/PugsUtils.cpp | 4 +- 14 files changed, 700 insertions(+), 41 deletions(-) create mode 100644 src/dev/ParallelCheckerDiscreteFunctionVariant.cpp create mode 100644 src/dev/ParallelCheckerItemArrayVariant.cpp create mode 100644 src/dev/ParallelCheckerItemValueVariant.cpp create mode 100644 src/dev/ParallelCheckerSubItemArrayPerItemVariant.cpp create mode 100644 src/dev/ParallelCheckerSubItemValuePerItemVariant.cpp diff --git a/src/dev/CMakeLists.txt b/src/dev/CMakeLists.txt index 22264f7ed..a212a76fa 100644 --- a/src/dev/CMakeLists.txt +++ b/src/dev/CMakeLists.txt @@ -3,6 +3,11 @@ add_library( PugsDev ParallelChecker.cpp + ParallelCheckerDiscreteFunctionVariant.cpp + ParallelCheckerItemArrayVariant.cpp + ParallelCheckerItemValueVariant.cpp + ParallelCheckerSubItemArrayPerItemVariant.cpp + ParallelCheckerSubItemValuePerItemVariant.cpp ) target_link_libraries( diff --git a/src/dev/ParallelChecker.cpp b/src/dev/ParallelChecker.cpp index 680cd92e6..2e6880fb8 100644 --- a/src/dev/ParallelChecker.cpp +++ b/src/dev/ParallelChecker.cpp @@ -15,21 +15,3 @@ ParallelChecker::destroy() Assert(ParallelChecker::m_instance != nullptr, "ParallelChecker has already been destroyed"); delete ParallelChecker::m_instance; } - -void -parallel_check(const ItemValueVariant& item_value_variant, - const std::string& name, - const SourceLocation& source_location) -{ - std::visit([&](auto&& item_value) { parallel_check(item_value, name, source_location); }, - item_value_variant.itemValue()); -} - -void -parallel_check(const DiscreteFunctionVariant& discrete_function_variant, - const std::string& name, - const SourceLocation& source_location) -{ - std::visit([&](auto&& discrete_function) { parallel_check(discrete_function, name, source_location); }, - discrete_function_variant.discreteFunction()); -} diff --git a/src/dev/ParallelChecker.hpp b/src/dev/ParallelChecker.hpp index b19689a63..e195a4280 100644 --- a/src/dev/ParallelChecker.hpp +++ b/src/dev/ParallelChecker.hpp @@ -9,6 +9,8 @@ #include <mesh/Connectivity.hpp> #include <mesh/ItemArrayVariant.hpp> #include <mesh/ItemValueVariant.hpp> +#include <mesh/SubItemArrayPerItemVariant.hpp> +#include <mesh/SubItemValuePerItemVariant.hpp> #include <scheme/DiscreteFunctionVariant.hpp> #include <utils/Demangle.hpp> #include <utils/Filesystem.hpp> @@ -23,7 +25,17 @@ void parallel_check(const ItemValue<DataType, item_type, ConnectivityPtr>& item_ const SourceLocation& source_location = SourceLocation{}); template <typename DataType, ItemType item_type, typename ConnectivityPtr> -void parallel_check(const ItemArray<DataType, item_type, ConnectivityPtr>& item_value, +void parallel_check(const ItemArray<DataType, item_type, ConnectivityPtr>& item_array, + const std::string& name, + const SourceLocation& source_location = SourceLocation{}); + +template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> +void parallel_check(const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_value_per_item, + const std::string& name, + const SourceLocation& source_location = SourceLocation{}); + +template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> +void parallel_check(const SubItemArrayPerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_array_per_item, const std::string& name, const SourceLocation& source_location = SourceLocation{}); @@ -58,6 +70,16 @@ class ParallelChecker const std::string& name, const SourceLocation& source_location); + template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> + friend void parallel_check(const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_value_per_item, + const std::string& name, + const SourceLocation& source_location); + + template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> + friend void parallel_check(const SubItemArrayPerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_array_per_item, + const std::string& name, + const SourceLocation& source_location); + #ifdef PUGS_HAS_HDF5 private: template <typename T> @@ -237,6 +259,29 @@ class ParallelChecker } } + template <ItemType item_type, ItemType sub_item_type> + Array<const typename ConnectivityMatrix::IndexType> + _getSubItemRowsMap(const std::shared_ptr<const IConnectivity>& i_connectivity) + { + switch (i_connectivity->dimension()) { + case 1: { + const Connectivity<1>& connectivity = dynamic_cast<const Connectivity<1>&>(*i_connectivity); + return connectivity.getMatrix(item_type, sub_item_type).rowsMap(); + } + case 2: { + const Connectivity<2>& connectivity = dynamic_cast<const Connectivity<2>&>(*i_connectivity); + return connectivity.getMatrix(item_type, sub_item_type).rowsMap(); + } + case 3: { + const Connectivity<3>& connectivity = dynamic_cast<const Connectivity<3>&>(*i_connectivity); + return connectivity.getMatrix(item_type, sub_item_type).rowsMap(); + } + default: { + throw UnexpectedError("unexpected connectivity dimension"); + } + } + } + template <ItemType item_type> Array<const int> _getItemOwner(const std::shared_ptr<const IConnectivity>& i_connectivity) const @@ -278,8 +323,31 @@ class ParallelChecker group.createHardLink("numbers", item_numbers); } - template <typename DataType, ItemType item_type> + template <typename ItemOfItem> void + _writeSubItemRowsMap(const std::shared_ptr<const IConnectivity> i_connectivity, + HighFive::File file, + HighFive::Group group) + { + constexpr ItemType item_type = ItemOfItem::item_type; + constexpr ItemType sub_item_type = ItemOfItem::sub_item_type; + + std::string subitem_of_item_row_map_path = + "/connectivities/" + std::to_string(this->_getConnectivityId(i_connectivity)) + '/' + + std::string{itemName(sub_item_type)} + "of" + std::string{itemName(item_type)}; + + if (not file.exist(subitem_of_item_row_map_path)) { + HighFive::Group subitem_of_item_row_map_group = file.createGroup(subitem_of_item_row_map_path); + this->_writeArray(subitem_of_item_row_map_group, "rows_map", + this->_getSubItemRowsMap<item_type, sub_item_type>(i_connectivity)); + } + + HighFive::DataSet subitem_of_item_row_map = file.getDataSet(subitem_of_item_row_map_path + "/rows_map"); + group.createHardLink("rows_map", subitem_of_item_row_map); + } + + template <typename DataType, ItemType item_type> + bool _checkIsComparable(const std::string& name, const SourceLocation& source_location, const std::vector<size_t> data_shape, @@ -343,8 +411,8 @@ class ParallelChecker // Just warn for different labels (maybe useful for some kind of // debugging...) std::cout << rang::fg::cyan << " | " << rang::fgB::magenta << "different names: reference (" << rang::fgB::yellow - << reference_name << rang::fgB::magenta << ") / target (" << rang::fgB::yellow << name - << rang::fg::reset << ")\n"; + << rang::style::reversed << reference_name << rang::style::reset << rang::fgB::magenta << ") / target (" + << rang::fgB::yellow << rang::style::reversed << name << rang::style::reset << rang::fg::reset << ")\n"; std::cout << rang::fg::cyan << " | " << rang::fgB::magenta << "reference from " << rang::fgB::blue << reference_file_name << rang::fg::reset << ':' << rang::style::bold << reference_line_number << rang::style::reset << '\n'; @@ -356,6 +424,53 @@ class ParallelChecker } } + return is_comparable; + } + + template <ItemType sub_item_type> + bool + _checkIsComparable(HighFive::Group group) const + { + const std::string reference_sub_item_type = group.getAttribute("subitem_type").read<std::string>(); + + bool is_comparable = true; + if (itemName(sub_item_type) != reference_sub_item_type) { + std::cout << rang::fg::cyan << " | " << rang::fgB::red << "different sub_item types: reference (" + << rang::fgB::yellow << reference_sub_item_type << rang::fgB::red << ") / target (" << rang::fgB::yellow + << itemName(sub_item_type) << rang::fg::reset << ")\n"; + is_comparable = false; + } + + return is_comparable; + } + + template <typename DataType, ItemType item_type> + void + _throwIfNotComparable(const std::string& name, + const SourceLocation& source_location, + const std::vector<size_t> data_shape, + const std::shared_ptr<const IConnectivity>& i_connectivity, + HighFive::Group group) const + { + bool is_comparable = + this->_checkIsComparable<DataType, item_type>(name, source_location, data_shape, i_connectivity, group); + if (not parallel::allReduceAnd(is_comparable)) { + throw NormalError("cannot compare data"); + } + } + + template <typename DataType, typename ItemOfItem> + void + _throwIfNotComparable(const std::string& name, + const SourceLocation& source_location, + const std::vector<size_t> data_shape, + const std::shared_ptr<const IConnectivity>& i_connectivity, + HighFive::Group group) const + { + bool is_comparable = this->_checkIsComparable<DataType, ItemOfItem::item_type>(name, source_location, data_shape, + i_connectivity, group) // + and this->_checkIsComparable<ItemOfItem::sub_item_type>(group); + if (not parallel::allReduceAnd(is_comparable)) { throw NormalError("cannot compare data"); } @@ -436,6 +551,92 @@ class ParallelChecker } } + template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> + void + write(const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_value_per_item, + const std::string& name, + const SourceLocation& source_location) + { + constexpr ItemType item_type = ItemOfItem::item_type; + constexpr ItemType sub_item_type = ItemOfItem::sub_item_type; + + HighFive::SilenceHDF5 m_silence_hdf5{true}; + this->_printHeader(name, source_location); + + try { + HighFive::File file = this->_createOrOpenFileRW(); + + auto group = file.createGroup("values/" + std::to_string(m_tag)); + + group.createAttribute("filename", std::string{source_location.filename()}); + group.createAttribute("function", std::string{source_location.function()}); + group.createAttribute("line", static_cast<size_t>(source_location.line())); + group.createAttribute("name", name); + + std::shared_ptr<const IConnectivity> i_connectivity = subitem_value_per_item.connectivity_ptr(); + group.createAttribute("dimension", static_cast<size_t>(i_connectivity->dimension())); + group.createAttribute("item_type", std::string{itemName(item_type)}); + group.createAttribute("subitem_type", std::string{itemName(sub_item_type)}); + + group.createAttribute("data_type", demangle<DataType>()); + + this->_writeArray(group, name, subitem_value_per_item.arrayView()); + + this->_writeItemNumbers<item_type>(i_connectivity, file, group); + this->_writeSubItemRowsMap<ItemOfItem>(i_connectivity, file, group); + + ++m_tag; + + std::cout << rang::fg::cyan << " | writing " << rang::fgB::green << "success" << rang::fg::reset << '\n'; + } + catch (HighFive::Exception& e) { + throw NormalError(e.what()); + } + } + + template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> + void + write(const SubItemArrayPerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_value_per_item, + const std::string& name, + const SourceLocation& source_location) + { + constexpr ItemType item_type = ItemOfItem::item_type; + constexpr ItemType sub_item_type = ItemOfItem::sub_item_type; + + HighFive::SilenceHDF5 m_silence_hdf5{true}; + this->_printHeader(name, source_location); + + try { + HighFive::File file = this->_createOrOpenFileRW(); + + auto group = file.createGroup("values/" + std::to_string(m_tag)); + + group.createAttribute("filename", std::string{source_location.filename()}); + group.createAttribute("function", std::string{source_location.function()}); + group.createAttribute("line", static_cast<size_t>(source_location.line())); + group.createAttribute("name", name); + + std::shared_ptr<const IConnectivity> i_connectivity = subitem_value_per_item.connectivity_ptr(); + group.createAttribute("dimension", static_cast<size_t>(i_connectivity->dimension())); + group.createAttribute("item_type", std::string{itemName(item_type)}); + group.createAttribute("subitem_type", std::string{itemName(sub_item_type)}); + + group.createAttribute("data_type", demangle<DataType>()); + + this->_writeTable(group, name, subitem_value_per_item.tableView()); + + this->_writeItemNumbers<item_type>(i_connectivity, file, group); + this->_writeSubItemRowsMap<ItemOfItem>(i_connectivity, file, group); + + ++m_tag; + + std::cout << rang::fg::cyan << " | writing " << rang::fgB::green << "success" << rang::fg::reset << '\n'; + } + catch (HighFive::Exception& e) { + throw NormalError(e.what()); + } + } + template <typename DataType, ItemType item_type, typename ConnectivityPtr> void compare(const ItemValue<DataType, item_type, ConnectivityPtr>& item_value, @@ -454,9 +655,9 @@ class ParallelChecker std::shared_ptr<const IConnectivity> i_connectivity = item_value.connectivity_ptr(); - this->_checkIsComparable<DataType, item_type>(name, source_location, - std::vector<size_t>{item_value.numberOfItems()}, i_connectivity, - group); + this->_throwIfNotComparable<DataType, item_type>(name, source_location, + std::vector<size_t>{item_value.numberOfItems()}, i_connectivity, + group); Array<const int> reference_item_numbers = this->_readArray<int>(group, "numbers"); @@ -575,10 +776,10 @@ class ParallelChecker std::shared_ptr<const IConnectivity> i_connectivity = item_array.connectivity_ptr(); - this->_checkIsComparable<DataType, item_type>(name, source_location, - std::vector<size_t>{item_array.numberOfItems(), - item_array.sizeOfArrays()}, - i_connectivity, group); + this->_throwIfNotComparable<DataType, item_type>(name, source_location, + std::vector<size_t>{item_array.numberOfItems(), + item_array.sizeOfArrays()}, + i_connectivity, group); Array<const int> reference_item_numbers = this->_readArray<int>(group, "numbers"); Table<const DataType> reference_item_array = this->_readTable<DataType>(group, reference_name); @@ -683,20 +884,347 @@ class ParallelChecker } } + template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> + void + compare(const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_value_per_item, + const std::string& name, + const SourceLocation& source_location) + { + HighFive::SilenceHDF5 m_silence_hdf5{true}; + this->_printHeader(name, source_location); + + try { + HighFive::File file{m_filename, HighFive::File::ReadOnly}; + + auto group = file.getGroup("values/" + std::to_string(m_tag)); + + const std::string reference_name = group.getAttribute("name").read<std::string>(); + + std::shared_ptr<const IConnectivity> i_connectivity = subitem_value_per_item.connectivity_ptr(); + + this->_throwIfNotComparable<DataType, ItemOfItem>(name, source_location, + std::vector<size_t>{subitem_value_per_item.numberOfItems()}, + i_connectivity, group); + + constexpr ItemType item_type = ItemOfItem::item_type; + constexpr ItemType sub_item_type = ItemOfItem::sub_item_type; + using IndexType = typename ConnectivityMatrix::IndexType; + + Array<const int> reference_item_numbers = this->_readArray<int>(group, "numbers"); + Array<const IndexType> reference_subitem_rows_map = this->_readArray<IndexType>(group, "rows_map"); + + Array<const DataType> reference_subitem_value_per_item = this->_readArray<DataType>(group, reference_name); + + Array<const int> item_numbers = this->_getItemNumber<item_type>(i_connectivity); + Array<const IndexType> sub_item_row_map = this->_getSubItemRowsMap<item_type, sub_item_type>(i_connectivity); + + using ItemId = ItemIdT<item_type>; + + std::unordered_map<int, ItemId> item_number_to_item_id_map; + + for (ItemId item_id = 0; item_id < item_numbers.size(); ++item_id) { + const auto& [iterator, success] = + item_number_to_item_id_map.insert(std::make_pair(item_numbers[item_id], item_id)); + + if (not success) { + throw UnexpectedError("item numbers have duplicate values"); + } + } + + Assert(item_number_to_item_id_map.size() == item_numbers.size()); + + Array<int> item_index_in_reference(item_numbers.size()); + item_index_in_reference.fill(-1); + for (size_t i = 0; i < reference_item_numbers.size(); ++i) { + const auto& i_number_to_item_id = item_number_to_item_id_map.find(reference_item_numbers[i]); + if (i_number_to_item_id != item_number_to_item_id_map.end()) { + item_index_in_reference[i_number_to_item_id->second] = i; + } + } + + if (parallel::allReduceMin(min(item_index_in_reference)) < 0) { + throw NormalError("some item numbers are not defined in reference"); + } + + Array<const int> owner = this->_getItemOwner<item_type>(i_connectivity); + + bool has_own_differences = false; + bool is_same = true; + + for (ItemId item_id = 0; item_id < subitem_value_per_item.numberOfItems(); ++item_id) { + const size_t reference_item_index = item_index_in_reference[item_id]; + const size_t index_begin_in_reference = reference_subitem_rows_map[reference_item_index]; + const size_t index_end_in_reference = reference_subitem_rows_map[reference_item_index + 1]; + + bool item_is_same = true; + if ((index_end_in_reference - index_begin_in_reference) != subitem_value_per_item[item_id].size()) { + item_is_same = false; + } else { + for (size_t i_sub_item = 0; i_sub_item < subitem_value_per_item[item_id].size(); ++i_sub_item) { + if (reference_subitem_value_per_item[index_begin_in_reference + i_sub_item] != + subitem_value_per_item[item_id][i_sub_item]) { + item_is_same = false; + } + } + } + if (not item_is_same) { + is_same = false; + if (static_cast<size_t>(owner[item_id]) == parallel::rank()) { + has_own_differences = true; + } + } + } + + is_same = parallel::allReduceAnd(is_same); + has_own_differences = parallel::allReduceOr(has_own_differences); + + if (is_same) { + std::cout << rang::fg::cyan << " | compare: " << rang::fgB::green << "success" << rang::fg::reset << '\n'; + } else { + if (has_own_differences) { + std::cout << rang::fg::cyan << " | compare: " << rang::fgB::red << "failed!" << rang::fg::reset; + } else { + std::cout << rang::fg::cyan << " | compare: " << rang::fgB::yellow << "not synchronized" << rang::fg::reset; + } + std::cout << rang::fg::cyan << " [see \"" << rang::fgB::blue << "parallel_differences_" << m_tag << "_*" + << rang::fg::cyan << "\" files for details]" << rang::fg::reset << '\n'; + + { + std::ofstream fout(std::string{"parallel_differences_"} + stringify(m_tag) + std::string{"_"} + + stringify(parallel::rank())); + + fout.precision(15); + for (ItemId item_id = 0; item_id < subitem_value_per_item.numberOfItems(); ++item_id) { + const size_t reference_item_index = item_index_in_reference[item_id]; + const size_t index_begin_in_reference = reference_subitem_rows_map[reference_item_index]; + const size_t index_end_in_reference = reference_subitem_rows_map[reference_item_index + 1]; + if ((index_end_in_reference - index_begin_in_reference) != subitem_value_per_item[item_id].size()) { + const bool is_own_difference = (parallel::rank() == static_cast<size_t>(owner[item_id])); + if (is_own_difference) { + fout << rang::fgB::red << "[ own ]" << rang::fg::reset; + } else { + fout << rang::fgB::yellow << "[ghost]" << rang::fg::reset; + } + fout << " rank=" << parallel::rank() << " owner=" << owner[item_id] << " item_id=" << item_id + << " number=" << item_numbers[item_id] + << " reference[subitems number]=" << index_end_in_reference - index_begin_in_reference + << " target[subitems number]=" << subitem_value_per_item[item_id].size() << '\n'; + + } else { + for (size_t i_sub_item = 0; i_sub_item < subitem_value_per_item[item_id].size(); ++i_sub_item) { + if (reference_subitem_value_per_item[index_begin_in_reference + i_sub_item] != + subitem_value_per_item[item_id][i_sub_item]) { + const bool is_own_difference = (parallel::rank() == static_cast<size_t>(owner[item_id])); + if (is_own_difference) { + fout << rang::fgB::red << "[ own ]" << rang::fg::reset; + } else { + fout << rang::fgB::yellow << "[ghost]" << rang::fg::reset; + } + fout << " rank=" << parallel::rank() << " owner=" << owner[item_id] << " item_id=" << item_id + << " number=" << item_numbers[item_id] << " i_subitem=" << i_sub_item + << " reference=" << reference_subitem_value_per_item[index_begin_in_reference + i_sub_item] + << " target=" << subitem_value_per_item[item_id][i_sub_item] << " difference=" + << reference_subitem_value_per_item[index_begin_in_reference + i_sub_item] - + subitem_value_per_item[item_id][i_sub_item] + << '\n'; + } + } + } + } + } + + if (has_own_differences) { + throw NormalError("calculations differ!"); + } + } + + ++m_tag; + } + catch (HighFive::Exception& e) { + throw NormalError(e.what()); + } + } + + template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> + void + compare(const SubItemArrayPerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_array_per_item, + const std::string& name, + const SourceLocation& source_location) + { + HighFive::SilenceHDF5 m_silence_hdf5{true}; + this->_printHeader(name, source_location); + + try { + HighFive::File file{m_filename, HighFive::File::ReadOnly}; + + auto group = file.getGroup("values/" + std::to_string(m_tag)); + + const std::string reference_name = group.getAttribute("name").read<std::string>(); + + std::shared_ptr<const IConnectivity> i_connectivity = subitem_array_per_item.connectivity_ptr(); + + this->_throwIfNotComparable<DataType, ItemOfItem>(name, source_location, + std::vector<size_t>{subitem_array_per_item.numberOfItems(), + subitem_array_per_item.sizeOfArrays()}, + i_connectivity, group); + + constexpr ItemType item_type = ItemOfItem::item_type; + constexpr ItemType sub_item_type = ItemOfItem::sub_item_type; + using IndexType = typename ConnectivityMatrix::IndexType; + + Array<const int> reference_item_numbers = this->_readArray<int>(group, "numbers"); + Array<const IndexType> reference_subitem_rows_map = this->_readArray<IndexType>(group, "rows_map"); + + Table<const DataType> reference_subitem_array_per_item = this->_readTable<DataType>(group, reference_name); + + Array<const int> item_numbers = this->_getItemNumber<item_type>(i_connectivity); + Array<const IndexType> sub_item_row_map = this->_getSubItemRowsMap<item_type, sub_item_type>(i_connectivity); + + using ItemId = ItemIdT<item_type>; + + std::unordered_map<int, ItemId> item_number_to_item_id_map; + + for (ItemId item_id = 0; item_id < item_numbers.size(); ++item_id) { + const auto& [iterator, success] = + item_number_to_item_id_map.insert(std::make_pair(item_numbers[item_id], item_id)); + + if (not success) { + throw UnexpectedError("item numbers have duplicate values"); + } + } + + Assert(item_number_to_item_id_map.size() == item_numbers.size()); + + Array<int> item_index_in_reference(item_numbers.size()); + item_index_in_reference.fill(-1); + for (size_t i = 0; i < reference_item_numbers.size(); ++i) { + const auto& i_number_to_item_id = item_number_to_item_id_map.find(reference_item_numbers[i]); + if (i_number_to_item_id != item_number_to_item_id_map.end()) { + item_index_in_reference[i_number_to_item_id->second] = i; + } + } + + if (parallel::allReduceMin(min(item_index_in_reference)) < 0) { + throw NormalError("some item numbers are not defined in reference"); + } + + Array<const int> owner = this->_getItemOwner<item_type>(i_connectivity); + + bool has_own_differences = false; + bool is_same = true; + + for (ItemId item_id = 0; item_id < subitem_array_per_item.numberOfItems(); ++item_id) { + const size_t reference_item_index = item_index_in_reference[item_id]; + const size_t index_begin_in_reference = reference_subitem_rows_map[reference_item_index]; + const size_t index_end_in_reference = reference_subitem_rows_map[reference_item_index + 1]; + + bool item_is_same = true; + if ((index_end_in_reference - index_begin_in_reference) != subitem_array_per_item[item_id].numberOfRows()) { + item_is_same = false; + } else { + for (size_t i_sub_item = 0; i_sub_item < subitem_array_per_item[item_id].numberOfRows(); ++i_sub_item) { + for (size_t i = 0; i < subitem_array_per_item.sizeOfArrays(); ++i) { + if (reference_subitem_array_per_item[index_begin_in_reference + i_sub_item][i] != + subitem_array_per_item[item_id][i_sub_item][i]) { + item_is_same = false; + } + } + } + } + if (not item_is_same) { + is_same = false; + if (static_cast<size_t>(owner[item_id]) == parallel::rank()) { + has_own_differences = true; + } + } + } + + is_same = parallel::allReduceAnd(is_same); + has_own_differences = parallel::allReduceOr(has_own_differences); + + if (is_same) { + std::cout << rang::fg::cyan << " | compare: " << rang::fgB::green << "success" << rang::fg::reset << '\n'; + } else { + if (has_own_differences) { + std::cout << rang::fg::cyan << " | compare: " << rang::fgB::red << "failed!" << rang::fg::reset; + } else { + std::cout << rang::fg::cyan << " | compare: " << rang::fgB::yellow << "not synchronized" << rang::fg::reset; + } + std::cout << rang::fg::cyan << " [see \"" << rang::fgB::blue << "parallel_differences_" << m_tag << "_*" + << rang::fg::cyan << "\" files for details]" << rang::fg::reset << '\n'; + + { + std::ofstream fout(std::string{"parallel_differences_"} + stringify(m_tag) + std::string{"_"} + + stringify(parallel::rank())); + + fout.precision(15); + for (ItemId item_id = 0; item_id < subitem_array_per_item.numberOfItems(); ++item_id) { + const size_t reference_item_index = item_index_in_reference[item_id]; + const size_t index_begin_in_reference = reference_subitem_rows_map[reference_item_index]; + const size_t index_end_in_reference = reference_subitem_rows_map[reference_item_index + 1]; + if ((index_end_in_reference - index_begin_in_reference) != subitem_array_per_item[item_id].numberOfRows()) { + const bool is_own_difference = (parallel::rank() == static_cast<size_t>(owner[item_id])); + if (is_own_difference) { + fout << rang::fgB::red << "[ own ]" << rang::fg::reset; + } else { + fout << rang::fgB::yellow << "[ghost]" << rang::fg::reset; + } + fout << " rank=" << parallel::rank() << " owner=" << owner[item_id] << " item_id=" << item_id + << " number=" << item_numbers[item_id] + << " reference[subitems number]=" << index_end_in_reference - index_begin_in_reference + << " target[subitems number]=" << subitem_array_per_item[item_id].numberOfRows() << '\n'; + + } else { + for (size_t i_sub_item = 0; i_sub_item < subitem_array_per_item[item_id].numberOfRows(); ++i_sub_item) { + for (size_t i = 0; i < subitem_array_per_item.sizeOfArrays(); ++i) { + if (reference_subitem_array_per_item[index_begin_in_reference + i_sub_item][i] != + subitem_array_per_item[item_id][i_sub_item][i]) { + const bool is_own_difference = (parallel::rank() == static_cast<size_t>(owner[item_id])); + if (is_own_difference) { + fout << rang::fgB::red << "[ own ]" << rang::fg::reset; + } else { + fout << rang::fgB::yellow << "[ghost]" << rang::fg::reset; + } + fout << " rank=" << parallel::rank() << " owner=" << owner[item_id] << " item_id=" << item_id + << " number=" << item_numbers[item_id] << " i_subitem=" << i_sub_item << " i_value=" << i + << " reference=" << reference_subitem_array_per_item[index_begin_in_reference + i_sub_item] + << " target=" << subitem_array_per_item[item_id][i_sub_item] << " difference=" + << reference_subitem_array_per_item[index_begin_in_reference + i_sub_item][i] - + subitem_array_per_item[item_id][i_sub_item][i] + << '\n'; + } + } + } + } + } + } + + if (has_own_differences) { + throw NormalError("calculations differ!"); + } + } + + ++m_tag; + } + catch (HighFive::Exception& e) { + throw NormalError(e.what()); + } + } + #else // PUGS_HAS_HDF5 template <typename T> void write(const T&, const std::string&, const SourceLocation&) { - throw UnexpectedError("parallel checker cannot be used without HDF5 support"); + throw NormalError("parallel checker cannot be used without HDF5 support"); } template <typename T> void compare(const T&, const std::string&, const SourceLocation&) { - throw UnexpectedError("parallel checker cannot be used without HDF5 support"); + throw NormalError("parallel checker cannot be used without HDF5 support"); } #endif // PUGS_HAS_HDF5 @@ -767,6 +1295,19 @@ class ParallelChecker } }; +template <typename DataType, ItemType item_type, typename ConnectivityPtr> +void +parallel_check(const ItemValue<DataType, item_type, ConnectivityPtr>& item_value, + const std::string& name, + const SourceLocation& source_location) +{ + if (ParallelChecker::instance().isWriting()) { + ParallelChecker::instance().write(item_value, name, source_location); + } else { + ParallelChecker::instance().compare(item_value, name, source_location); + } +} + template <typename DataType, ItemType item_type, typename ConnectivityPtr> void parallel_check(const ItemArray<DataType, item_type, ConnectivityPtr>& item_array, @@ -780,16 +1321,29 @@ parallel_check(const ItemArray<DataType, item_type, ConnectivityPtr>& item_array } } -template <typename DataType, ItemType item_type, typename ConnectivityPtr> +template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> void -parallel_check(const ItemValue<DataType, item_type, ConnectivityPtr>& item_value, +parallel_check(const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_value_per_item, const std::string& name, const SourceLocation& source_location) { if (ParallelChecker::instance().isWriting()) { - ParallelChecker::instance().write(item_value, name, source_location); + ParallelChecker::instance().write(subitem_value_per_item, name, source_location); } else { - ParallelChecker::instance().compare(item_value, name, source_location); + ParallelChecker::instance().compare(subitem_value_per_item, name, source_location); + } +} + +template <typename DataType, typename ItemOfItem, typename ConnectivityPtr> +void +parallel_check(const SubItemArrayPerItem<DataType, ItemOfItem, ConnectivityPtr>& subitem_array_per_item, + const std::string& name, + const SourceLocation& source_location) +{ + if (ParallelChecker::instance().isWriting()) { + ParallelChecker::instance().write(subitem_array_per_item, name, source_location); + } else { + ParallelChecker::instance().compare(subitem_array_per_item, name, source_location); } } @@ -815,6 +1369,18 @@ void parallel_check(const ItemValueVariant& item_value_variant, const std::string& name, const SourceLocation& source_location = SourceLocation{}); +void parallel_check(const ItemArrayVariant& item_array_variant, + const std::string& name, + const SourceLocation& source_location = SourceLocation{}); + +void parallel_check(const SubItemValuePerItemVariant& subitem_value_per_item_variant, + const std::string& name, + const SourceLocation& source_location = SourceLocation{}); + +void parallel_check(const SubItemArrayPerItemVariant& subitem_array_per_item_variant, + const std::string& name, + const SourceLocation& source_location = SourceLocation{}); + void parallel_check(const DiscreteFunctionVariant& discrete_function_variant, const std::string& name, const SourceLocation& source_location = SourceLocation{}); diff --git a/src/dev/ParallelCheckerDiscreteFunctionVariant.cpp b/src/dev/ParallelCheckerDiscreteFunctionVariant.cpp new file mode 100644 index 000000000..626fa6842 --- /dev/null +++ b/src/dev/ParallelCheckerDiscreteFunctionVariant.cpp @@ -0,0 +1,10 @@ +#include <dev/ParallelChecker.hpp> + +void +parallel_check(const DiscreteFunctionVariant& discrete_function_variant, + const std::string& name, + const SourceLocation& source_location) +{ + std::visit([&](auto&& discrete_function) { parallel_check(discrete_function, name, source_location); }, + discrete_function_variant.discreteFunction()); +} diff --git a/src/dev/ParallelCheckerItemArrayVariant.cpp b/src/dev/ParallelCheckerItemArrayVariant.cpp new file mode 100644 index 000000000..2ca0f7e3f --- /dev/null +++ b/src/dev/ParallelCheckerItemArrayVariant.cpp @@ -0,0 +1,10 @@ +#include <dev/ParallelChecker.hpp> + +void +parallel_check(const ItemArrayVariant& item_array_variant, + const std::string& name, + const SourceLocation& source_location) +{ + std::visit([&](auto&& item_value) { parallel_check(item_value, name, source_location); }, + item_array_variant.itemArray()); +} diff --git a/src/dev/ParallelCheckerItemValueVariant.cpp b/src/dev/ParallelCheckerItemValueVariant.cpp new file mode 100644 index 000000000..4cbc9ed21 --- /dev/null +++ b/src/dev/ParallelCheckerItemValueVariant.cpp @@ -0,0 +1,10 @@ +#include <dev/ParallelChecker.hpp> + +void +parallel_check(const ItemValueVariant& item_value_variant, + const std::string& name, + const SourceLocation& source_location) +{ + std::visit([&](auto&& item_value) { parallel_check(item_value, name, source_location); }, + item_value_variant.itemValue()); +} diff --git a/src/dev/ParallelCheckerSubItemArrayPerItemVariant.cpp b/src/dev/ParallelCheckerSubItemArrayPerItemVariant.cpp new file mode 100644 index 000000000..d33898f50 --- /dev/null +++ b/src/dev/ParallelCheckerSubItemArrayPerItemVariant.cpp @@ -0,0 +1,10 @@ +#include <dev/ParallelChecker.hpp> + +void +parallel_check(const SubItemArrayPerItemVariant& subitem_array_per_item_variant, + const std::string& name, + const SourceLocation& source_location) +{ + std::visit([&](auto&& item_value) { parallel_check(item_value, name, source_location); }, + subitem_array_per_item_variant.subItemArrayPerItem()); +} diff --git a/src/dev/ParallelCheckerSubItemValuePerItemVariant.cpp b/src/dev/ParallelCheckerSubItemValuePerItemVariant.cpp new file mode 100644 index 000000000..4584202e2 --- /dev/null +++ b/src/dev/ParallelCheckerSubItemValuePerItemVariant.cpp @@ -0,0 +1,10 @@ +#include <dev/ParallelChecker.hpp> + +void +parallel_check(const SubItemValuePerItemVariant& subitem_value_per_item_variant, + const std::string& name, + const SourceLocation& source_location) +{ + std::visit([&](auto&& item_value) { parallel_check(item_value, name, source_location); }, + subitem_value_per_item_variant.subItemValuePerItem()); +} diff --git a/src/language/modules/DevUtilsModule.cpp b/src/language/modules/DevUtilsModule.cpp index 1de94fa63..7685443b0 100644 --- a/src/language/modules/DevUtilsModule.cpp +++ b/src/language/modules/DevUtilsModule.cpp @@ -24,6 +24,16 @@ template <> inline ASTNodeDataType ast_node_data_type_from<std::shared_ptr<const ItemArrayVariant>> = ASTNodeDataType::build<ASTNodeDataType::type_id_t>("item_array"); +class SubItemValuePerItemVariant; +template <> +inline ASTNodeDataType ast_node_data_type_from<std::shared_ptr<const SubItemValuePerItemVariant>> = + ASTNodeDataType::build<ASTNodeDataType::type_id_t>("sub_item_value"); + +class SubItemArrayPerItemVariant; +template <> +inline ASTNodeDataType ast_node_data_type_from<std::shared_ptr<const SubItemArrayPerItemVariant>> = + ASTNodeDataType::build<ASTNodeDataType::type_id_t>("sub_item_array"); + DevUtilsModule::DevUtilsModule() { this->_addBuiltinFunction("getAST", std::function( @@ -97,6 +107,37 @@ DevUtilsModule::DevUtilsModule() parallel_check(*item_value, name, ASTBacktrace::getInstance().sourceLocation()); } + )); + + this->_addBuiltinFunction("parallel_check", + std::function( + + [](const std::shared_ptr<const ItemArrayVariant>& item_array, const std::string& name) { + parallel_check(*item_array, name, ASTBacktrace::getInstance().sourceLocation()); + } + + )); + + this->_addBuiltinFunction("parallel_check", + std::function( + + [](const std::shared_ptr<const SubItemValuePerItemVariant>& subitem_value_per_item, + const std::string& name) { + parallel_check(*subitem_value_per_item, name, + ASTBacktrace::getInstance().sourceLocation()); + } + + )); + + this->_addBuiltinFunction("parallel_check", + std::function( + + [](const std::shared_ptr<const SubItemArrayPerItemVariant>& subitem_array_per_item, + const std::string& name) { + parallel_check(*subitem_array_per_item, name, + ASTBacktrace::getInstance().sourceLocation()); + } + )); } diff --git a/src/mesh/SubItemArrayPerItem.hpp b/src/mesh/SubItemArrayPerItem.hpp index a98c039e0..14e0caa63 100644 --- a/src/mesh/SubItemArrayPerItem.hpp +++ b/src/mesh/SubItemArrayPerItem.hpp @@ -49,6 +49,13 @@ class SubItemArrayPerItem friend SubItemArrayPerItem<std::remove_const_t<DataType>, ItemOfItem, ConnectivityWeakPtr>; public: + // This is not the correct way to look at SubItemArrayPerItem, use with care + Table<const DataType> + tableView() const + { + return m_values; + } + [[nodiscard]] friend PUGS_INLINE SubItemArrayPerItem<std::remove_const_t<DataType>, ItemOfItem, ConnectivityPtr> copy(const SubItemArrayPerItem<DataType, ItemOfItem, ConnectivityPtr>& source) { diff --git a/src/mesh/SubItemArrayPerItemVariant.hpp b/src/mesh/SubItemArrayPerItemVariant.hpp index ffb0ea673..2f917aa7c 100644 --- a/src/mesh/SubItemArrayPerItemVariant.hpp +++ b/src/mesh/SubItemArrayPerItemVariant.hpp @@ -146,7 +146,7 @@ class SubItemArrayPerItemVariant public: PUGS_INLINE const Variant& - itemArray() const + subItemArrayPerItem() const { return m_sub_item_array_per_item; } @@ -189,7 +189,7 @@ class SubItemArrayPerItemVariant "SubItemArrayPerItem with this DataType is not allowed in variant"); } - SubItemArrayPerItemVariant& operator=(SubItemArrayPerItemVariant&&) = default; + SubItemArrayPerItemVariant& operator=(SubItemArrayPerItemVariant&&) = default; SubItemArrayPerItemVariant& operator=(const SubItemArrayPerItemVariant&) = default; SubItemArrayPerItemVariant(const SubItemArrayPerItemVariant&) = default; diff --git a/src/mesh/SubItemValuePerItem.hpp b/src/mesh/SubItemValuePerItem.hpp index f5ce39d5f..d1b569919 100644 --- a/src/mesh/SubItemValuePerItem.hpp +++ b/src/mesh/SubItemValuePerItem.hpp @@ -48,6 +48,14 @@ class SubItemValuePerItem friend SubItemValuePerItem<std::remove_const_t<DataType>, ItemOfItem, ConnectivityWeakPtr>; public: + // This is not the correct way to look at SubItemValuePerItem, use + // with care + Array<const DataType> + arrayView() const + { + return m_values; + } + [[nodiscard]] friend PUGS_INLINE SubItemValuePerItem<std::remove_const_t<DataType>, ItemOfItem, ConnectivityPtr> copy(const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& source) { diff --git a/src/mesh/SubItemValuePerItemVariant.hpp b/src/mesh/SubItemValuePerItemVariant.hpp index 618fd82d9..ef2bf58e3 100644 --- a/src/mesh/SubItemValuePerItemVariant.hpp +++ b/src/mesh/SubItemValuePerItemVariant.hpp @@ -146,7 +146,7 @@ class SubItemValuePerItemVariant public: PUGS_INLINE const Variant& - itemValue() const + subItemValuePerItem() const { return m_sub_item_value_per_item; } @@ -189,7 +189,7 @@ class SubItemValuePerItemVariant "SubItemValuePerItem with this DataType is not allowed in variant"); } - SubItemValuePerItemVariant& operator=(SubItemValuePerItemVariant&&) = default; + SubItemValuePerItemVariant& operator=(SubItemValuePerItemVariant&&) = default; SubItemValuePerItemVariant& operator=(const SubItemValuePerItemVariant&) = default; SubItemValuePerItemVariant(const SubItemValuePerItemVariant&) = default; diff --git a/src/utils/PugsUtils.cpp b/src/utils/PugsUtils.cpp index 54bec14a0..104e70cdd 100644 --- a/src/utils/PugsUtils.cpp +++ b/src/utils/PugsUtils.cpp @@ -120,8 +120,8 @@ initialize(int& argc, char* argv[]) app.add_flag("--exec-stat,!--no-exec-stat", print_exec_stat, "Display memory and CPU usage after execution [default: true]"); - bool show_backtrace = true; - app.add_flag("-b,--backtrace,!--no-backtrace", show_backtrace, "Show backtrace on failure [default: true]"); + bool show_backtrace = false; + app.add_flag("-b,--backtrace,!--no-backtrace", show_backtrace, "Show backtrace on failure [default: false]"); app.add_flag("--signal,!--no-signal", enable_signals, "Catches signals [default: true]"); -- GitLab