diff --git a/packages/HighFive/.github/workflows/ci.yml b/packages/HighFive/.github/workflows/ci.yml index de0810d06dba11528658ba249599f5f9fd4098d1..e7f5fca10b9bf26ceb00276ec3dc749e11b0b94c 100644 --- a/packages/HighFive/.github/workflows/ci.yml +++ b/packages/HighFive/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -C $BUILD_TYPE + run: ctest -j2 --output-on-failure -C $BUILD_TYPE # Job testing several versions of hdf5 @@ -89,7 +89,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - hdf5_version : [ hdf5-1_8_23, hdf5-1_10_10, hdf5-1_12_2, hdf5-1_14_1 ] + hdf5_version : [ hdf5-1_8_23, hdf5-1_10_11, hdf5-1_12_2, hdf5-1_14_3 ] steps: - uses: actions/checkout@v3 @@ -105,7 +105,7 @@ jobs: wget https://github.com/HDFGroup/hdf5/archive/refs/tags/${{ matrix.hdf5_version }}.tar.gz --output-document hdf5.tar.gz tar xf hdf5.tar.gz mkdir -p hdf5-${{ matrix.hdf5_version }}/BUILD && cd hdf5-${{ matrix.hdf5_version }}/BUILD - cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja -DCMAKE_INSTALL_PREFIX=$HOME/${{ matrix.hdf5_version }} -DHDF5_ENABLE_Z_LIB_SUPPORT=ON -DUSE_LIBAEC=ON + cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja -DCMAKE_INSTALL_PREFIX=$HOME/${{ matrix.hdf5_version }} -DHDF5_ENABLE_Z_LIB_SUPPORT=ON -DUSE_LIBAEC=ON -DHDF5_BUILD_EXAMPLES=OFF -DBUILD_STATIC_LIBS=OFF -DBUILD_TESTING=OFF ninja && ninja install - name: Build @@ -118,7 +118,7 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -C $BUILD_TYPE + run: ctest -j2 --output-on-failure -C $BUILD_TYPE - name: Examples @@ -156,7 +156,7 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -C $BUILD_TYPE + run: ctest -j2 --output-on-failure -C $BUILD_TYPE - name: Examples working-directory: ${{github.workspace}}/build/src/examples @@ -199,7 +199,7 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -C $BUILD_TYPE + run: ctest -j2 --output-on-failure -C $BUILD_TYPE - name: Examples working-directory: ${{github.workspace}}/build/src/examples @@ -295,7 +295,7 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -C $BUILD_TYPE + run: ctest -j2 --output-on-failure -C $BUILD_TYPE - name: Examples working-directory: ${{github.workspace}}/build/src/examples @@ -348,4 +348,4 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build shell: bash -l {0} - run: ctest --output-on-failure -C $BUILD_TYPE + run: ctest -j2 --output-on-failure -C $BUILD_TYPE diff --git a/packages/HighFive/.github/workflows/version_file.yml b/packages/HighFive/.github/workflows/version_file.yml new file mode 100644 index 0000000000000000000000000000000000000000..816137e955fe920c432cf985cbd3a43fa2e924ff --- /dev/null +++ b/packages/HighFive/.github/workflows/version_file.yml @@ -0,0 +1,36 @@ +name: HighFive Check Version File + +on: + push: + branches: + - ci_test + - release/** + pull_request: + branches: + - master + - release/** + +jobs: + CheckVersion: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: "Install libraries" + run: | + sudo apt-get -qq update + sudo apt-get -qq install libhdf5-dev ninja-build + + - name: Build + run: | + # Will trigger `configure_file` for H5Version.hpp. + cmake -DHIGHFIVE_USE_BOOST=Off -B build . + + - name: Test + run: | + # Check that the file hasn't changed, i.e. was updated + # after changing the version number. + ! git status | grep include/highfive/H5Version.hpp diff --git a/packages/HighFive/.gitrepo b/packages/HighFive/.gitrepo index 8a0046eea29d1f2b4fac6534ac21ac2f845e2dea..14b8f5fba715beda2d02da0beb4c2e267ac16373 100644 --- a/packages/HighFive/.gitrepo +++ b/packages/HighFive/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = git@github.com:BlueBrain/HighFive.git branch = master - commit = 7340d1fcc7c455519c113216d8415e003ed37cc6 - parent = 95783fdd2e50f442c21e5a9e668d14d4985bf2a0 + commit = 88fcc898970f93a63be8e25760d6b9f33589690f + parent = 88cd46aaec511360705df0fc7e577098877862d0 method = merge cmdver = 0.4.6 diff --git a/packages/HighFive/AUTHORS.txt b/packages/HighFive/AUTHORS.txt index 0a3cdf580fa008d4fb622a6fdcd5128955cb58ee..78f573a9a6abfe5db14e7ea8921b89e80bb78d21 100644 --- a/packages/HighFive/AUTHORS.txt +++ b/packages/HighFive/AUTHORS.txt @@ -2,6 +2,7 @@ Adrien Devresse Alexandru Săvulescu Ali Can Demiralp Angelos Plastropoulos +@antonysigma Chris Byrohl Chris De Grendele @contre @@ -12,6 +13,8 @@ Fernando L. Pereira @guoxy Haoran Ni Henry Schreiner +@hn-sl +Hunter Belanger @JaWSnl Jia Li John W. Peterson @@ -49,6 +52,7 @@ Tino Wagner Tobias Klauser Tom de Geus Tom Vander Aa +Torsten Reuschel Tristan Carel Wolf Vollprecht Y. Yang diff --git a/packages/HighFive/CHANGELOG.md b/packages/HighFive/CHANGELOG.md index eeeb628c95f46f9680f4a79d9609b1257e89e4cd..9a8cd86139f80bc1523e6f0e33fc049316a0f830 100644 --- a/packages/HighFive/CHANGELOG.md +++ b/packages/HighFive/CHANGELOG.md @@ -1,14 +1,34 @@ # Changes -## Version 2.8.0 - 2023-MM-DD +## Version 2.8.0 - 2023-11-02 +### Important Change + - `Eigen::Matrix` is (by default) stored with column-major index ordering. Under + certain conditions `Eigen::Matrix` was written and read as row-major. + Due to code duplication H5Easy isn't affected by this bug. Starting + `2.8.0` HighFive will now throw an exception whenever prior versions would + have read with incorrect assumptions about the index ordering. (#731) + ### New Features + - Improve reading and writing `std::string` as fixed and variable length HDF5 strings (#744). - Implement creation of hard links (#765). Thanks to @Quark-X10. - Get the size of file and amound of tracked unused space (#764). Thanks to @Quark-X10. + - `class DataType` has a new ctor to open a commited `DataType` (#796). Thanks to @Quark-X10. + - Allow user-specified `mem_space` for hyperslabs. (#740) + - New properties: `AttributePhaseChange`. (#785) + - New options to link against HDF5 statically (#823). Thanks @HunterBelanger. + - Add support for `std::complex<integral_type>` valid with C++23 (#828). Thanks @unbtorsten. + - Add a top-level header to include all compononents (#818). ### Improvements + - Add concept checks to `Property` if C++20 for better errors (#811). Thanks @antonysigma. - Add parallel HDF5 test in CI (#760). - Simplify github workflow (#761). - Move inspectors in their own file to be able to better implements strings (#759). +### Bug Fix + - Fix vector constructor ambiguity in H5DataType.hpp (#775). Thanks to @hn-sl. + - `getElementCount()` fixed. (#787) + - Remove leak when calling dtor of `CompoundType`. (#798) + ## Version 2.7.1 - 2023-04-04 ### Bug Fix - Revert removing `#include "H5FileDriver.hpp"` from `H5File.hpp` (#711). diff --git a/packages/HighFive/CMake/HighFiveTargetDeps.cmake b/packages/HighFive/CMake/HighFiveTargetDeps.cmake index da28423bce60dad614c0cf9fb83673dff9af63ce..919b53544ea198e040d86d5620bb05f05ab5a7f2 100644 --- a/packages/HighFive/CMake/HighFiveTargetDeps.cmake +++ b/packages/HighFive/CMake/HighFiveTargetDeps.cmake @@ -26,6 +26,7 @@ if(NOT TARGET libdeps) # HDF5 if(NOT DEFINED HDF5_C_LIBRARIES) set(HDF5_PREFER_PARALLEL ${HIGHFIVE_PARALLEL_HDF5}) + set(HDF5_USE_STATIC_LIBRARIES ${HIGHFIVE_STATIC_HDF5}) find_package(HDF5 REQUIRED) endif() diff --git a/packages/HighFive/CMake/HighFiveTargetExport.cmake b/packages/HighFive/CMake/HighFiveTargetExport.cmake index 011f7f4832973f3816abc5293b4d1b2a5465e969..9906f39513c385e9dba2dca5194453fe9b2eec16 100644 --- a/packages/HighFive/CMake/HighFiveTargetExport.cmake +++ b/packages/HighFive/CMake/HighFiveTargetExport.cmake @@ -4,7 +4,6 @@ add_library(libheaders INTERFACE) target_include_directories(libheaders INTERFACE "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>" - "$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>" "$<INSTALL_INTERFACE:include>") # Combined HighFive diff --git a/packages/HighFive/CMakeLists.txt b/packages/HighFive/CMakeLists.txt index a9deba11342bb3bcd1263ccc4ff861885f867bf3..af274d9e25ea0ba40cfbd0407531db449be95f67 100644 --- a/packages/HighFive/CMakeLists.txt +++ b/packages/HighFive/CMakeLists.txt @@ -5,10 +5,10 @@ else() cmake_policy(VERSION 3.13) endif() -project(HighFive VERSION 2.7.1) +project(HighFive VERSION 2.8.0) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/highfive/H5Version.hpp.in - ${CMAKE_CURRENT_BINARY_DIR}/include/highfive/H5Version.hpp) + ${CMAKE_CURRENT_SOURCE_DIR}/include/highfive/H5Version.hpp) # INCLUDES list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/CMake @@ -32,6 +32,7 @@ option(HIGHFIVE_USE_OPENCV "Enable OpenCV testing" ${USE_OPENCV}) option(HIGHFIVE_USE_XTENSOR "Enable xtensor testing" ${USE_XTENSOR}) option(HIGHFIVE_EXAMPLES "Compile examples" ON) option(HIGHFIVE_PARALLEL_HDF5 "Enable Parallel HDF5 support" OFF) +option(HIGHFIVE_STATIC_HDF5 "Staticly link to HDF5 library" OFF) option(HIGHFIVE_BUILD_DOCS "Enable documentation building" ON) option(HIGHFIVE_VERBOSE "Set logging level to verbose." OFF) option(HIGHFIVE_GLIBCXX_ASSERTIONS "Enable bounds check for STL." OFF) @@ -103,11 +104,6 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION "include" PATTERN "*.in" EXCLUDE) -# Installation of configured headers -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/ - DESTINATION "include") - - # Preparing local building (tests, examples) # ------------------------------------------ diff --git a/packages/HighFive/README.md b/packages/HighFive/README.md index 2ef1de8d547294e619de2158c7e109d5767f0f8f..3ea0680157cdc72040a39c68c2f4ebe2d1569ca1 100644 --- a/packages/HighFive/README.md +++ b/packages/HighFive/README.md @@ -53,7 +53,7 @@ It integrates nicely with other CMake projects by defining (and exporting) a Hig #### Write a std::vector<int> to 1D HDF5 dataset and read it back ```c++ -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/doc/developer_guide.md b/packages/HighFive/doc/developer_guide.md index 9b33cdcf9c6702bae0c049d0fe4764eec5c71902..3017289b5172bbb443829da0daf6bcaa04413e54 100644 --- a/packages/HighFive/doc/developer_guide.md +++ b/packages/HighFive/doc/developer_guide.md @@ -42,7 +42,7 @@ the git repository. Conveniently, `clang-format` is available via `pip`: python -m venv venv source venv/bin/activate -pip install clang-format=12.0.1 +pip install clang-format==12.0.1 ``` The changed lines can be formatted with `git-clang-format`, e.g. to format all lines changed compared to master: @@ -57,6 +57,7 @@ your changes.) Before releasing a new version perform the following: * Update `CHANGELOG.md` and `AUTHORS.txt` as required. +* Update `CMakeLists.txt` and `include/highfive/H5Version.hpp`. * Follow semantic versioning when deciding the next version number. * Check that [HighFive-testing](https://github.com/BlueBrain/HighFive-testing/actions) ran diff --git a/packages/HighFive/doc/installation.md b/packages/HighFive/doc/installation.md index 36676c2c7fee35263c46337cc6366f6f2572e6b1..41521bba5e57212e7a7cd08b9423c5fcea655ddd 100644 --- a/packages/HighFive/doc/installation.md +++ b/packages/HighFive/doc/installation.md @@ -202,7 +202,7 @@ install. The detailed instructions would be git clone --recursive https://github.com/BlueBrain/HighFive.git cd HighFive - git checkout v2.7.1 + git checkout v2.8.0 If it complains that Catch is missing, you forgot the `--recursive`. To fix this you type @@ -218,7 +218,7 @@ Okay, on to configure, compile and install. The CMake commands are ### Confirming It Works We again need a dummy file called `dummy.cpp` with the following contents - #include <highfive/H5File.hpp> + #include <highfive/highfive.hpp> int main() { auto file = HighFive::File("foo.h5", HighFive::File::Create); diff --git a/packages/HighFive/doc/poster/example3.cpp b/packages/HighFive/doc/poster/example3.cpp index 64f2ed6be7630c7d47b20cafed007383fae8c622..e18fbbf83e4fb015647a70d91857f18902eb0c39 100644 --- a/packages/HighFive/doc/poster/example3.cpp +++ b/packages/HighFive/doc/poster/example3.cpp @@ -1,4 +1,4 @@ -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> typedef struct { diff --git a/packages/HighFive/doc/poster/example6.cpp b/packages/HighFive/doc/poster/example6.cpp index 8f7419f23fd1a7054491b14bc27db6d31c2976c6..41a050570598d4fb1a2dec0aa040eee6d72763d5 100644 --- a/packages/HighFive/doc/poster/example6.cpp +++ b/packages/HighFive/doc/poster/example6.cpp @@ -2,9 +2,7 @@ #include <mpi.h> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> int main(int argc, char** argv) { diff --git a/packages/HighFive/doc/poster/example_boost.cpp b/packages/HighFive/doc/poster/example_boost.cpp index aceaa20681b31b14d46a94bac2a0a0391e7ebde0..56b78d074e3b136c193dc4781b9812e9074db88c 100644 --- a/packages/HighFive/doc/poster/example_boost.cpp +++ b/packages/HighFive/doc/poster/example_boost.cpp @@ -1,11 +1,9 @@ #include <complex> #define H5_USE_BOOST 1 +#include <highfive/highfive.hpp> #include <boost/multi_array.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> using complex_t = std::complex<double>; diff --git a/packages/HighFive/doc/poster/example_boost_ublas.cpp b/packages/HighFive/doc/poster/example_boost_ublas.cpp index 3a2b4c73a583060aec3441ccdaa890daecd03fee..986a671de57f59d2f0da98c2d1e76efa529bef83 100644 --- a/packages/HighFive/doc/poster/example_boost_ublas.cpp +++ b/packages/HighFive/doc/poster/example_boost_ublas.cpp @@ -1,6 +1,7 @@ #include <iostream> #define H5_USE_BOOST 1 +#include <highfive/highfive.hpp> // In some versions of Boost (starting with 1.64), you have to // include the serialization header before ublas @@ -8,7 +9,6 @@ #include <boost/numeric/ublas/io.hpp> #include <boost/numeric/ublas/matrix.hpp> -#include <highfive/H5File.hpp> using namespace HighFive; diff --git a/packages/HighFive/doc/poster/example_easy_highfive.cpp b/packages/HighFive/doc/poster/example_easy_highfive.cpp index 07d37a22c5f3584aa34fbf544c4394679c5522da..700056cae978ecc043c60d558927f282ad14b127 100644 --- a/packages/HighFive/doc/poster/example_easy_highfive.cpp +++ b/packages/HighFive/doc/poster/example_easy_highfive.cpp @@ -1,4 +1,5 @@ #include <xtensor/xarray.hpp> + #include <highfive/H5Easy.hpp> int main() { diff --git a/packages/HighFive/doc/poster/example_props.cpp b/packages/HighFive/doc/poster/example_props.cpp index e46fe119ceebdee3f6a2cfa8f0c7a9ecf2edd981..0e5b14bde7ce990798d02035fd3c780bd25e0058 100644 --- a/packages/HighFive/doc/poster/example_props.cpp +++ b/packages/HighFive/doc/poster/example_props.cpp @@ -1,4 +1,4 @@ -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/include/highfive/H5DataType.hpp b/packages/HighFive/include/highfive/H5DataType.hpp index 8403618e2596cac6a9605d7a7c176d4f07d0e2c2..886107961b87e32911702f3aaf3496acc8ce8c26 100644 --- a/packages/HighFive/include/highfive/H5DataType.hpp +++ b/packages/HighFive/include/highfive/H5DataType.hpp @@ -11,6 +11,8 @@ #include <type_traits> #include <vector> +#include <H5Tpublic.h> + #include "H5Object.hpp" #include "bits/H5Utils.hpp" diff --git a/packages/HighFive/include/highfive/H5Version.hpp b/packages/HighFive/include/highfive/H5Version.hpp new file mode 100644 index 0000000000000000000000000000000000000000..dc238432cb01d651769c663276ea15a94008f3e9 --- /dev/null +++ b/packages/HighFive/include/highfive/H5Version.hpp @@ -0,0 +1,33 @@ +/* + * Copyright (c), 2020 + * + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * http://www.boost.org/LICENSE_1_0.txt) + * + */ +#pragma once + +#define HIGHFIVE_VERSION_MAJOR 2 +#define HIGHFIVE_VERSION_MINOR 8 +#define HIGHFIVE_VERSION_PATCH 0 + +/** \brief Concatenated representation of the HighFive version. + * + * \warning The macro `HIGHFIVE_VERSION` by itself isn't valid C/C++. + * + * However, it can be stringified with two layers of macros, e.g., + * \code{.cpp} + * #define STRINGIFY_VALUE(s) STRINGIFY_NAME(s) + * #define STRINGIFY_NAME(s) #s + * + * std::cout << STRINGIFY_VALUE(HIGHFIVE_VERSION) << "\n"; + * \endcode + */ +#define HIGHFIVE_VERSION 2.8.0 + +/** \brief String representation of the HighFive version. + * + * \warning This macro only exists from 2.7.1 onwards. + */ +#define HIGHFIVE_VERSION_STRING "2.8.0" diff --git a/packages/HighFive/include/highfive/bits/H5Attribute_misc.hpp b/packages/HighFive/include/highfive/bits/H5Attribute_misc.hpp index 5c52f7bbb1ce507228b1c54b953b1f8e843f3b9a..6516788297819c9f6e318bd588df3ac668ac63fd 100644 --- a/packages/HighFive/include/highfive/bits/H5Attribute_misc.hpp +++ b/packages/HighFive/include/highfive/bits/H5Attribute_misc.hpp @@ -87,8 +87,10 @@ inline void Attribute::read(T& array) const { read(r.getPointer(), buffer_info.data_type); // re-arrange results r.unserialize(array); - auto t = create_datatype<typename details::inspector<T>::base_type>(); + + auto t = buffer_info.data_type; auto c = t.getClass(); + if (c == DataTypeClass::VarLen || t.isVariableStr()) { #if H5_VERSION_GE(1, 12, 0) // This one have been created in 1.12.0 diff --git a/packages/HighFive/include/highfive/bits/H5Converter_misc.hpp b/packages/HighFive/include/highfive/bits/H5Converter_misc.hpp index 8f4411118a7a40f33ae41118b458f2e27709ff58..00749d1b6d6157bc745a86005a7a80d6b195fbb3 100644 --- a/packages/HighFive/include/highfive/bits/H5Converter_misc.hpp +++ b/packages/HighFive/include/highfive/bits/H5Converter_misc.hpp @@ -16,11 +16,23 @@ namespace HighFive { namespace details { +template <class T> +struct is_std_string { + static constexpr bool value = + std::is_same<typename inspector<T>::base_type, std::string>::value; +}; + template <class T, class V = void> -struct enable_shallow_copy: public std::enable_if<inspector<T>::is_trivially_copyable, V> {}; +struct enable_shallow_copy + : public std::enable_if<!is_std_string<T>::value && inspector<T>::is_trivially_copyable, V> {}; template <class T, class V = void> -struct enable_deep_copy: public std::enable_if<!inspector<T>::is_trivially_copyable, V> {}; +struct enable_deep_copy + : public std::enable_if<!is_std_string<T>::value && !inspector<T>::is_trivially_copyable, V> {}; + +template <class T, class V = void> +struct enable_string_copy: public std::enable_if<is_std_string<T>::value, V> {}; + template <typename T, bool IsReadOnly> struct ShallowCopyBuffer { @@ -85,6 +97,243 @@ struct DeepCopyBuffer { std::vector<size_t> dims; }; +enum class BufferMode { Read, Write }; + + +/// +/// \brief String length in bytes excluding the `\0`. +/// +inline size_t char_buffer_size(char const* const str, size_t max_string_length) { + for (size_t i = 0; i <= max_string_length; ++i) { + if (str[i] == '\0') { + return i; + } + } + + return max_string_length; +} + + +/// +/// \brief A buffer for reading/writing strings. +/// +/// A string in HDF5 can be represented as a fixed or variable length string. +/// The important difference for this buffer is that `H5D{read,write}` expects +/// different input depending on whether the strings are fixed or variable length. +/// For fixed length strings, it expects an array of chars, i.e. one string +/// packed after the other contiguously. While for variable length strings it +/// expects a list of pointers to the beginning of each string. Variable length +/// string must be null-terminated; because that's how their length is +/// determined. +/// +/// This buffer hides the difference between fixed and variable length strings +/// by having internal data structures available for both cases at compile time. +/// The choice which internal buffer to use is made at runtime. +/// +/// Consider an HDF5 dataset with N fixed-length strings, each of which is M +/// characters long. Then the in-memory strings are copied into an internal +/// buffer of size N*M. If null- or space-padded the buffer should be filled +/// with the appropriate character. This is important if the in-memory strings +/// are less than M characters long. +/// +/// An HDF5 dataset with N variable-length strings (all null-terminated) uses +/// the internal list of pointers to the beginning of each string. Those +/// pointers can either point to the in-memory strings themselves, if those +/// strings are known to be null-terminated. Otherwise the in-memory strings are +/// copied to an internal buffer of null-terminated strings; and the pointer +/// points to the start of the string in the internal buffer. +/// +/// This class is responsible for arranging the strings properly before passing +/// the buffers to HDF5. To keep this class generic, it provides a generic +/// read/write interface to the internal strings, i.e. a pointer with a size. +/// For reading from the buffer the proxy is called `StringConstView`. This +/// proxy object is to be used by the `inspector` to copy from the buffer into +/// the final destination, e.g. an `std::string`. Similarly, there's a proxy +/// object for serializing into the buffer, i.e. the `StringView`. Again the +/// `inspector` is responsible for obtaining the pointer, size and padding of +/// the string. +/// +/// Nomenclature: +/// - size of a string is the number of bytes required to store the string, +/// including the null character for null-terminated strings. +/// +/// - length of a string is the number of bytes without the null character. +/// +/// Note: both 'length' and 'size' are counted in number of bytes, not number +/// of symbols or characters. Even for UTF8 strings. +template <typename T, BufferMode buffer_mode> +struct StringBuffer { + using type = unqualified_t<T>; + using hdf5_type = typename inspector<type>::hdf5_type; + + class StringView { + public: + StringView(StringBuffer<T, buffer_mode>& _buffer, size_t _i) + : buffer(_buffer) + , i(_i) {} + + /// + /// \brief Assign the in-memory string to the buffer. + /// + /// This method copies the in-memory string to the appropriate + /// internal buffer as needed. + /// + /// The `length` is the length of the string in bytes. + void assign(char const* data, size_t length, StringPadding padding) { + if (buffer.isVariableLengthString()) { + if (padding == StringPadding::NullTerminated) { + buffer.variable_length_pointers[i] = data; + } else { + buffer.variable_length_buffer[i] = std::string(data, length); + buffer.variable_length_pointers[i] = buffer.variable_length_buffer[i].data(); + } + } else if (buffer.isFixedLengthString()) { + // If the buffer is fixed-length and null-terminated, then + // `buffer.string_length` doesn't include the null-character. + if (length > buffer.string_length) { + throw std::invalid_argument("String length too big."); + } + + memcpy(&buffer.fixed_length_buffer[i * buffer.string_size], data, length); + } + } + + private: + StringBuffer<T, buffer_mode>& buffer; + size_t i; + }; + + + class StringConstView { + public: + StringConstView(const StringBuffer<T, buffer_mode>& _buffer, size_t _i) + : buffer(_buffer) + , i(_i) {} + + /// \brief Pointer to the first byte of the string. + /// + /// The valid indices for this pointer are: 0, ..., length() - 1. + char const* data() const { + if (buffer.isVariableLengthString()) { + return buffer.variable_length_pointers[i]; + } else { + return &buffer.fixed_length_buffer[i * buffer.string_size]; + } + } + + /// \brief Length of the string in bytes. + /// + /// Note that for null-terminated strings the "length" doesn't include + /// the null character. Hence, if storing this string as a + /// null-terminated string, the destination buffer needs to be at least + /// `length() + 1` bytes long. + size_t length() const { + if (buffer.isNullTerminated()) { + return char_buffer_size(data(), buffer.string_length); + } else { + return buffer.string_length; + } + } + + private: + const StringBuffer<T, buffer_mode>& buffer; + size_t i; + }; + + + class Iterator { + public: + Iterator(StringBuffer<T, buffer_mode>& _buffer, size_t _pos) + : buffer(_buffer) + , pos(_pos) {} + + Iterator operator+(size_t n_strings) const { + return Iterator(buffer, pos + n_strings); + } + + void operator+=(size_t n_strings) { + pos += n_strings; + } + + StringView operator*() { + return StringView(buffer, pos); + } + + StringConstView operator*() const { + return StringConstView(buffer, pos); + } + + private: + StringBuffer<T, buffer_mode>& buffer; + size_t pos; + }; + + StringBuffer(std::vector<size_t> _dims, const DataType& _file_datatype) + : file_datatype(_file_datatype.asStringType()) + , padding(file_datatype.getPadding()) + , string_size(file_datatype.isVariableStr() ? size_t(-1) : file_datatype.getSize()) + , string_length(string_size - size_t(isNullTerminated())) + , dims(_dims) { + if (string_size == 0 && isNullTerminated()) { + throw DataTypeException( + "Fixed-length, null-terminated need at least one byte to store the " + "null-character."); + } + + auto n_strings = compute_total_size(dims); + if (isVariableLengthString()) { + variable_length_buffer.resize(n_strings); + variable_length_pointers.resize(n_strings); + } else { + char pad = padding == StringPadding::SpacePadded ? ' ' : '\0'; + fixed_length_buffer.assign(n_strings * string_size, pad); + } + } + + bool isVariableLengthString() const { + return file_datatype.isVariableStr(); + } + + bool isFixedLengthString() const { + return file_datatype.isFixedLenStr(); + } + + bool isNullTerminated() const { + return file_datatype.getPadding() == StringPadding::NullTerminated; + } + + + void* getPointer() { + if (file_datatype.isVariableStr()) { + return variable_length_pointers.data(); + } else { + return fixed_length_buffer.data(); + } + } + + Iterator begin() { + return Iterator(*this, 0ul); + } + + void unserialize(T& val) { + inspector<type>::unserialize(begin(), dims, val); + } + + private: + StringType file_datatype; + StringPadding padding; + size_t string_size; // Size of buffer required to store the string. + // Meaningful for fixed length strings only. + size_t string_length; // Semantic length of string. + std::vector<size_t> dims; + + std::vector<char> fixed_length_buffer; + std::vector<std::string> variable_length_buffer; + std::vector< + typename std::conditional<buffer_mode == BufferMode::Write, const char, char>::type*> + variable_length_pointers; +}; + template <typename T, typename Enable = void> struct Writer; @@ -107,6 +356,14 @@ struct Writer<T, typename enable_deep_copy<T>::type>: public DeepCopyBuffer<T> { } }; +template <typename T> +struct Writer<T, typename enable_string_copy<T>::type>: public StringBuffer<T, BufferMode::Write> { + explicit Writer(const T& val, const DataType& _file_datatype) + : StringBuffer<T, BufferMode::Write>(inspector<T>::getDimensions(val), _file_datatype) { + inspector<T>::serialize(val, this->begin()); + } +}; + template <typename T, typename Enable = void> struct Reader; @@ -133,6 +390,15 @@ struct Reader<T, typename enable_deep_copy<T>::type>: public DeepCopyBuffer<T> { }; +template <typename T> +struct Reader<T, typename enable_string_copy<T>::type>: public StringBuffer<T, BufferMode::Write> { + public: + explicit Reader(const std::vector<size_t>& _dims, + const T& /* val */, + const DataType& _file_datatype) + : StringBuffer<T, BufferMode::Write>(_dims, _file_datatype) {} +}; + struct data_converter { template <typename T> static Writer<T> serialize(const typename inspector<T>::type& val, diff --git a/packages/HighFive/include/highfive/bits/H5DataType_misc.hpp b/packages/HighFive/include/highfive/bits/H5DataType_misc.hpp index 934d5f5e5f64e40f1b13bb433f9a8b11ec416753..8535d617ab443405dbdd88353f52cf4850f8e7d1 100644 --- a/packages/HighFive/include/highfive/bits/H5DataType_misc.hpp +++ b/packages/HighFive/include/highfive/bits/H5DataType_misc.hpp @@ -22,7 +22,7 @@ #include <half.hpp> #endif -#include "H5Converter_misc.hpp" +#include "H5Inspector_misc.hpp" namespace HighFive { @@ -300,8 +300,8 @@ class AtomicType<std::complex<T>>: public DataType { : DataType( CompoundType({{"r", create_datatype<T>(), 0}, {"i", create_datatype<T>(), sizeof(T)}}, sizeof(std::complex<T>))) { - static_assert(std::is_floating_point<T>::value, - "std::complex accepts only floating point numbers."); + static_assert(std::is_arithmetic<T>::value, + "std::complex accepts only floating point and integral numbers."); } }; diff --git a/packages/HighFive/include/highfive/bits/H5Inspector_misc.hpp b/packages/HighFive/include/highfive/bits/H5Inspector_misc.hpp index c25bcfbd566649a33ea372ca4f202bf6ea29fe7c..05ed6bc3ec0250b18e0ce4311ff710e703a6fca9 100644 --- a/packages/HighFive/include/highfive/bits/H5Inspector_misc.hpp +++ b/packages/HighFive/include/highfive/bits/H5Inspector_misc.hpp @@ -12,6 +12,9 @@ #include <type_traits> #include <cstring> #include <cassert> +#include <vector> +#include <array> +#include <string> #include <numeric> #include "../H5Reference.hpp" @@ -28,7 +31,9 @@ #include <Eigen/Eigen> #endif + namespace HighFive { + namespace details { inline bool checkDimensions(const std::vector<size_t>& dims, size_t n_dim_requested) { @@ -260,14 +265,15 @@ struct inspector<std::string>: type_helper<std::string> { throw DataSpaceException("A std::string cannot be written directly."); } - static void serialize(const type& val, hdf5_type* m) { - *m = val.c_str(); + template <class It> + static void serialize(const type& val, It m) { + (*m).assign(val.data(), val.size(), StringPadding::NullTerminated); } - static void unserialize(const hdf5_type* vec, - const std::vector<size_t>& /* dims */, - type& val) { - val = vec[0]; + template <class It> + static void unserialize(const It& vec, const std::vector<size_t>& /* dims */, type& val) { + const auto& view = *vec; + val.assign(view.data(), view.length()); } }; @@ -488,6 +494,7 @@ struct inspector<std::array<T, N>> { static constexpr size_t ndim = 1; static constexpr size_t recursive_ndim = ndim + inspector<value_type>::recursive_ndim; static constexpr bool is_trivially_copyable = std::is_trivially_copyable<value_type>::value && + sizeof(type) == N * sizeof(T) && inspector<value_type>::is_trivially_copyable; static std::vector<size_t> getDimensions(const type& val) { diff --git a/packages/HighFive/include/highfive/bits/H5ReadWrite_misc.hpp b/packages/HighFive/include/highfive/bits/H5ReadWrite_misc.hpp index 9da473b05e9fa63730f8e6c1e486658d7ab51ae7..c8e73617400fa9cec3b727d84fb49f1ac8e74c11 100644 --- a/packages/HighFive/include/highfive/bits/H5ReadWrite_misc.hpp +++ b/packages/HighFive/include/highfive/bits/H5ReadWrite_misc.hpp @@ -19,9 +19,13 @@ template <typename T> using unqualified_t = typename std::remove_const<typename std::remove_reference<T>::type>::type; // Find the type of an eventual char array, otherwise void -template <typename> +template <typename T> struct type_char_array { - using type = void; + using type = typename std::conditional< + std::is_same<typename inspector<T>::base_type, std::string>::value, + std::string, + void>::type; + static constexpr bool is_char_array = false; }; template <typename T> @@ -29,6 +33,7 @@ struct type_char_array<T*> { using type = typename std::conditional<std::is_same<unqualified_t<T>, char>::value, char*, typename type_char_array<T>::type>::type; + static constexpr bool is_char_array = true; }; template <typename T, std::size_t N> @@ -36,6 +41,7 @@ struct type_char_array<T[N]> { using type = typename std::conditional<std::is_same<unqualified_t<T>, char>::value, char[N], typename type_char_array<T>::type>::type; + static constexpr bool is_char_array = true; }; template <typename T> @@ -43,7 +49,7 @@ struct BufferInfo { using type_no_const = typename std::remove_const<T>::type; using elem_type = typename details::inspector<type_no_const>::base_type; using char_array_t = typename details::type_char_array<type_no_const>::type; - static constexpr bool is_char_array = !std::is_same<char_array_t, void>::value; + static constexpr bool is_char_array = details::type_char_array<type_no_const>::is_char_array; enum Operation { read, write }; const Operation op; @@ -85,6 +91,16 @@ struct string_type_checker<void> { } }; +template <> +struct string_type_checker<std::string> { + inline static DataType getDataType(const DataType&, const DataType& file_datatype) { + // The StringBuffer ensures that the data is transformed such that it + // matches the datatype of the dataset, i.e. `file_datatype` and + // `mem_datatype` are the same. + return file_datatype; + } +}; + template <std::size_t FixedLen> struct string_type_checker<char[FixedLen]> { inline static DataType getDataType(const DataType& element_type, const DataType& dtype) { @@ -98,8 +114,9 @@ struct string_type_checker<char[FixedLen]> { template <> struct string_type_checker<char*> { inline static DataType getDataType(const DataType&, const DataType& dtype) { - if (dtype.isFixedLenStr()) + if (dtype.isFixedLenStr()) { throw DataSetException("Can't output variable-length to fixed-length strings"); + } DataType return_type = AtomicType<std::string>(); enforce_ascii_hack(return_type, dtype); return return_type; @@ -116,11 +133,6 @@ BufferInfo<T>::BufferInfo(const DataType& dtype, F getName, Operation _op) ((is_fixed_len_string && is_char_array) ? 1 : 0)) , data_type( string_type_checker<char_array_t>::getDataType(create_datatype<elem_type>(), dtype)) { - if (is_fixed_len_string && std::is_same<elem_type, std::string>::value) { - throw DataSetException( - "Can't output std::string as fixed-length. " - "Use raw arrays or FixedLenStringArray"); - } // We warn. In case they are really not convertible an exception will rise on read/write if (dtype.getClass() != data_type.getClass()) { HIGHFIVE_LOG_WARN(getName() + "\": data and hdf5 dataset have different types: " + diff --git a/packages/HighFive/include/highfive/bits/H5Slice_traits_misc.hpp b/packages/HighFive/include/highfive/bits/H5Slice_traits_misc.hpp index a5dc9ee378e5d494b1e46bc62804322836dae937..7b07c9abf9406b214e068aadd7c5a34011eb3d60 100644 --- a/packages/HighFive/include/highfive/bits/H5Slice_traits_misc.hpp +++ b/packages/HighFive/include/highfive/bits/H5Slice_traits_misc.hpp @@ -199,7 +199,8 @@ inline void SliceTraits<Derivate>::read(T& array, const DataTransferProps& xfer_ read(r.getPointer(), buffer_info.data_type, xfer_props); // re-arrange results r.unserialize(array); - auto t = create_datatype<typename details::inspector<T>::base_type>(); + + auto t = buffer_info.data_type; auto c = t.getClass(); if (c == DataTypeClass::VarLen || t.isVariableStr()) { #if H5_VERSION_GE(1, 12, 0) diff --git a/packages/HighFive/include/highfive/bits/string_padding.hpp b/packages/HighFive/include/highfive/bits/string_padding.hpp index 984549e240a9f402752ebf86aecb60d13aa1e3e1..e6e6908ddc5e320019bff48fb5cf494c76bb96b0 100644 --- a/packages/HighFive/include/highfive/bits/string_padding.hpp +++ b/packages/HighFive/include/highfive/bits/string_padding.hpp @@ -1,5 +1,7 @@ #pragma once +#include <H5Tpublic.h> + namespace HighFive { enum class StringPadding : std::underlying_type<H5T_str_t>::type { diff --git a/packages/HighFive/include/highfive/highfive.hpp b/packages/HighFive/include/highfive/highfive.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f5e20cae9131ae7e6440f6fa7f16bc8b3430b336 --- /dev/null +++ b/packages/HighFive/include/highfive/highfive.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include <highfive/H5Attribute.hpp> +#include <highfive/H5DataSet.hpp> +#include <highfive/H5DataSpace.hpp> +#include <highfive/H5DataType.hpp> +#include <highfive/H5File.hpp> +#include <highfive/H5FileDriver.hpp> +#include <highfive/H5Group.hpp> +#include <highfive/H5PropertyList.hpp> +#include <highfive/H5Reference.hpp> +#include <highfive/H5Selection.hpp> +#include <highfive/H5Utility.hpp> +#include <highfive/H5Version.hpp> diff --git a/packages/HighFive/src/benchmarks/highfive_bench.cpp b/packages/HighFive/src/benchmarks/highfive_bench.cpp index e4e04dee8e7a970eafd2010a8fcf13132359b511..bcc7be93ce742701d03e8daa6f21286f838efe9e 100644 --- a/packages/HighFive/src/benchmarks/highfive_bench.cpp +++ b/packages/HighFive/src/benchmarks/highfive_bench.cpp @@ -1,4 +1,4 @@ -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> #include <vector> const std::vector<std::vector<int>> data(1000000, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); diff --git a/packages/HighFive/src/examples/boost_multi_array_2D.cpp b/packages/HighFive/src/examples/boost_multi_array_2D.cpp index 582049efdde323ba89eaf207a490cce51a4adc37..4bec1ec12991391a58549e8fb044294523f4a74b 100644 --- a/packages/HighFive/src/examples/boost_multi_array_2D.cpp +++ b/packages/HighFive/src/examples/boost_multi_array_2D.cpp @@ -12,7 +12,7 @@ #define H5_USE_BOOST #include <boost/multi_array.hpp> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/boost_multiarray_complex.cpp b/packages/HighFive/src/examples/boost_multiarray_complex.cpp index 1b66b3ddf3cfe106fd7191bf13613781cbd59b4b..37481db625b5b21d7d13c0ae7da4223c9d7eb327 100644 --- a/packages/HighFive/src/examples/boost_multiarray_complex.cpp +++ b/packages/HighFive/src/examples/boost_multiarray_complex.cpp @@ -12,10 +12,9 @@ #undef H5_USE_BOOST #define H5_USE_BOOST +#include <highfive/highfive.hpp> + #include <boost/multi_array.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> typedef std::complex<double> complex_t; diff --git a/packages/HighFive/src/examples/boost_ublas_double.cpp b/packages/HighFive/src/examples/boost_ublas_double.cpp index ca3307ecfc53281b6672fabe38fd385ff86d2a6c..b025475b9584b2f27c6dbfcd51e3ee402898031a 100644 --- a/packages/HighFive/src/examples/boost_ublas_double.cpp +++ b/packages/HighFive/src/examples/boost_ublas_double.cpp @@ -11,13 +11,14 @@ #undef H5_USE_BOOST #define H5_USE_BOOST +#include <highfive/highfive.hpp> + // In some versions of Boost (starting with 1.64), you have to include the serialization header // before ublas #include <boost/serialization/vector.hpp> #include <boost/numeric/ublas/io.hpp> #include <boost/numeric/ublas/matrix.hpp> -#include <highfive/H5File.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/compound_types.cpp b/packages/HighFive/src/examples/compound_types.cpp index d456859a432e7b35a34c8203c2d47c96ed962943..158bc6dc6de6e53314470d1db57bc392bdb765aa 100644 --- a/packages/HighFive/src/examples/compound_types.cpp +++ b/packages/HighFive/src/examples/compound_types.cpp @@ -10,7 +10,7 @@ // Compound datatype test :: May 2021 // ////////////////////////////////// -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> typedef struct { diff --git a/packages/HighFive/src/examples/create_attribute_string_integer.cpp b/packages/HighFive/src/examples/create_attribute_string_integer.cpp index a716c827a4b3b14ecacb4e9225930eaa2a1a55ef..f658457adfe0e0a51534a7e4512ca982c13fae11 100644 --- a/packages/HighFive/src/examples/create_attribute_string_integer.cpp +++ b/packages/HighFive/src/examples/create_attribute_string_integer.cpp @@ -10,10 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5Attribute.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/create_dataset_double.cpp b/packages/HighFive/src/examples/create_dataset_double.cpp index e5b96fcf6bf042372eb33ec0ec767db17fbf41d4..d15fbdc54f196195880347cd56f2e7e6783e453d 100644 --- a/packages/HighFive/src/examples/create_dataset_double.cpp +++ b/packages/HighFive/src/examples/create_dataset_double.cpp @@ -10,9 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> // Create a dataset name "dset" of double 4x6 int main(void) { diff --git a/packages/HighFive/src/examples/create_dataset_half_float.cpp b/packages/HighFive/src/examples/create_dataset_half_float.cpp index 204e45374bc1d2134a21d0ed22b659ecacf74cd8..2b720cd187036c98392ed7fd2e9518934ae948ce 100644 --- a/packages/HighFive/src/examples/create_dataset_half_float.cpp +++ b/packages/HighFive/src/examples/create_dataset_half_float.cpp @@ -13,9 +13,7 @@ #include <string> #include <vector> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> const std::string FILE_NAME("create_dataset_half_float_example.h5"); const std::string DATASET_NAME("dset"); diff --git a/packages/HighFive/src/examples/create_datatype.cpp b/packages/HighFive/src/examples/create_datatype.cpp index d1e342bb4a5d911391b9f1d845e51bdde77dced8..0b1a0fb52044c915f185c52d5145f1738d279834 100644 --- a/packages/HighFive/src/examples/create_datatype.cpp +++ b/packages/HighFive/src/examples/create_datatype.cpp @@ -1,7 +1,6 @@ #include <iostream> -#include <highfive/H5File.hpp> -#include <highfive/H5DataType.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/create_extensible_dataset.cpp b/packages/HighFive/src/examples/create_extensible_dataset.cpp index 53a6b2b43ae0d846714101819a597649de35fcc2..17153cd5740bda666808a2d7f45f7fa19c262235 100644 --- a/packages/HighFive/src/examples/create_extensible_dataset.cpp +++ b/packages/HighFive/src/examples/create_extensible_dataset.cpp @@ -10,9 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> const std::string FILE_NAME("create_extensible_dataset_example.h5"); const std::string DATASET_NAME("dset"); diff --git a/packages/HighFive/src/examples/create_large_attribute.cpp b/packages/HighFive/src/examples/create_large_attribute.cpp index 004386e6eb6dc83a269a5af614a32150ee69e118..022d11d87bcfdd8e9ab9c6719ea37a500e837176 100644 --- a/packages/HighFive/src/examples/create_large_attribute.cpp +++ b/packages/HighFive/src/examples/create_large_attribute.cpp @@ -1,9 +1,8 @@ -#include <H5Fpublic.h> -#include <highfive/H5File.hpp> -#include <highfive/H5Group.hpp> #include <numeric> #include <vector> +#include <highfive/highfive.hpp> + int main() { std::vector<double> large_attr(16000, 0.0); diff --git a/packages/HighFive/src/examples/create_page_allocated_files.cpp b/packages/HighFive/src/examples/create_page_allocated_files.cpp index 73f15c5fb2cd92c34da5629fd3dc1d1484e9b340..0b4d29dffb3c1a21ca90114b122ef020fd394ae2 100644 --- a/packages/HighFive/src/examples/create_page_allocated_files.cpp +++ b/packages/HighFive/src/examples/create_page_allocated_files.cpp @@ -12,7 +12,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> // This example requires HDF5 version 1.10.1 or newer. #if H5_VERSION_GE(1, 10, 1) diff --git a/packages/HighFive/src/examples/hl_hdf5_inmemory_files.cpp b/packages/HighFive/src/examples/hl_hdf5_inmemory_files.cpp index 58bdbbec1c0909fd0465c227e4c86a930a46d2f5..088fd71cc8db6a165cecdde1307212b4f828e18d 100644 --- a/packages/HighFive/src/examples/hl_hdf5_inmemory_files.cpp +++ b/packages/HighFive/src/examples/hl_hdf5_inmemory_files.cpp @@ -9,8 +9,8 @@ #include <iostream> #include <vector> +#include <highfive/highfive.hpp> -#include <highfive/H5File.hpp> #include <hdf5_hl.h> using namespace HighFive; diff --git a/packages/HighFive/src/examples/parallel_hdf5_collective_io.cpp b/packages/HighFive/src/examples/parallel_hdf5_collective_io.cpp index c7d7f9920d08e9fd310f6b8558c794548d96d00e..7261b7cf1d8b8a92623ef166bea72156250d9b7b 100644 --- a/packages/HighFive/src/examples/parallel_hdf5_collective_io.cpp +++ b/packages/HighFive/src/examples/parallel_hdf5_collective_io.cpp @@ -13,10 +13,7 @@ #include <mpi.h> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5PropertyList.hpp> +#include <highfive/highfive.hpp> const std::string file_name("parallel_collective_example.h5"); const std::string dataset_name("dset"); diff --git a/packages/HighFive/src/examples/parallel_hdf5_independent_io.cpp b/packages/HighFive/src/examples/parallel_hdf5_independent_io.cpp index f56d87f05c5dd7cfbc80def6dbfd7c3867fc6e09..b43012890c209a05dfd0700333f608a6dd247211 100644 --- a/packages/HighFive/src/examples/parallel_hdf5_independent_io.cpp +++ b/packages/HighFive/src/examples/parallel_hdf5_independent_io.cpp @@ -13,10 +13,7 @@ #include <mpi.h> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5PropertyList.hpp> +#include <highfive/highfive.hpp> const std::string file_name("parallel_independent_example.h5"); diff --git a/packages/HighFive/src/examples/read_write_dataset_string.cpp b/packages/HighFive/src/examples/read_write_dataset_string.cpp index c5cf658a2d6bf8d41a2c34633383e4ea5062b61e..2a4c3c491105c8585ec3838fe609279e71b52ba9 100644 --- a/packages/HighFive/src/examples/read_write_dataset_string.cpp +++ b/packages/HighFive/src/examples/read_write_dataset_string.cpp @@ -10,9 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/read_write_fixedlen_string.cpp b/packages/HighFive/src/examples/read_write_fixedlen_string.cpp index 123e9443ff9a28c2b9c3a540e86723c9a29ea73f..60589637ea97e6c526ab8b8c6754026c7191331d 100644 --- a/packages/HighFive/src/examples/read_write_fixedlen_string.cpp +++ b/packages/HighFive/src/examples/read_write_fixedlen_string.cpp @@ -9,9 +9,7 @@ #include <iostream> #include <string> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/read_write_raw_ptr.cpp b/packages/HighFive/src/examples/read_write_raw_ptr.cpp index 5a95b03f53c10351e870d485b6d1e0e81aa683bd..b6cd9eda59e74e842a988f777ca5c95ffb26b702 100644 --- a/packages/HighFive/src/examples/read_write_raw_ptr.cpp +++ b/packages/HighFive/src/examples/read_write_raw_ptr.cpp @@ -11,9 +11,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> const std::string file_name("read_write_raw_ptr.h5"); const std::string dataset_name("array"); diff --git a/packages/HighFive/src/examples/read_write_single_scalar.cpp b/packages/HighFive/src/examples/read_write_single_scalar.cpp index b32d9b3b51c4791db0b116637379c82604c65121..4b4c6887c96e47bffba798dff3cebfd2ce3cbccd 100644 --- a/packages/HighFive/src/examples/read_write_single_scalar.cpp +++ b/packages/HighFive/src/examples/read_write_single_scalar.cpp @@ -10,9 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> const std::string file_name("read_write_scalar.h5"); const std::string dataset_name("single_scalar"); diff --git a/packages/HighFive/src/examples/read_write_std_strings.cpp b/packages/HighFive/src/examples/read_write_std_strings.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7699e0c0c4177da12ccc6d0372c9e42f10691246 --- /dev/null +++ b/packages/HighFive/src/examples/read_write_std_strings.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c), 2023, Blue Brain Project, EPFL + * + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * http://www.boost.org/LICENSE_1_0.txt) + * + */ +#include <iostream> +#include <string> +#include <vector> + +#include <highfive/H5File.hpp> +#include <highfive/H5DataSet.hpp> +#include <highfive/H5DataSpace.hpp> + +using namespace HighFive; + +// This example shows how to write (containers of) `std::string` +// to dataset either as fixed or variable length HDF5 strings. +// The feature is available from 2.8.0 onwards. +int main(void) { + auto file = File("read_write_std_string.h5", File::Truncate); + + // A string of length 3 in a buffer of size 4 bytes. We'll use "length" for + // the semantic length of the string, i.e. excluding the '\0' character and + // "size" to refer to the length of the buffer in which the string is stored. + // For null-terminated strings, the `size == length + 1`. + std::string ascii_string = "foo"; + auto scalar_dataspace = DataSpace(DataSpace::dataspace_scalar); + + // Just write the string: + file.createDataSet("single_automatic", ascii_string); + + // The above results in writing the string as an HDF5 variable length UTF8 + // string. In HDF5 a variable length string doesn't specify the length of + // the string. Variable length strings are always null-terminated. + auto variable_stringtype = VariableLengthStringType(); + file.createDataSet("single_variable", scalar_dataspace, variable_stringtype) + .write(ascii_string); + + // HDF5 also has the concept of fixed length string. In fixed length strings + // the size of the string, in bytes, is part of the datatype. The HDF5 API + // for fixed and variable length strings is distinct. Hence, when writing + // string that need to be read by other programs, it can matter if the string + // is stored as fixed or variable length. + // + // Important: The HDF5 string size is the size of the buffer required to + // store the string. + // + // We know that ascii_string requires 4 bytes to store, but want to store + // it in fixed length strings of length 8. Additionally, we promise that + // the strings are null-terminated. The character set defaults to ASCII. + auto fixed_stringtype = FixedLengthStringType(8, StringPadding::NullTerminated); + file.createDataSet("single_fixed_nullterm", scalar_dataspace, fixed_stringtype) + .write(ascii_string); + + // When reading into an `std::string` it doesn't matter if the HDF5 datatype + // is fixed or variable length. HighFive will internally read into a buffer + // and then write to the final destination. + auto from_variable = file.getDataSet("single_variable").read<std::string>(); + auto from_fixed = file.getDataSet("single_fixed_nullterm").read<std::string>(); + + // Note that because the fixed length string is null-terminated, + // `from_fixed.size() == ascii_string.size()` despite it being stored as a string of + // length 8. + std::cout << "from_variable = '" << from_variable << "' size = " << from_variable.size() + << "\n"; + std::cout << "from_fixed = '" << from_fixed << "' size = " << from_fixed.size() << "\n"; + + // Fixed-length string don't have to be null-terminated. Their length could + // be defined simply by the known size of the buffer required to store the + // string. To deal with the situation where the string is shorter than the + // buffer, one defines a padding character. This must be either the null or + // space character. We'll show null-padded, space-padded works the same way. + auto fixed_nullpad = FixedLengthStringType(8, StringPadding::NullPadded); + file.createDataSet("single_fixed_nullpad", scalar_dataspace, fixed_nullpad).write(ascii_string); + + // Note that because we only know that the string is padded with nulls, but we + // don't know if those nulls were part of the string to begin with. The full + // size of the buffer is read into the `std::string`. The length of the + // `std::string` is the size of the string type. + auto from_nullpad = file.getDataSet("single_fixed_nullpad").read<std::string>(); + std::cout << "from_nullpad = '" << from_nullpad << "' size = " << from_nullpad.size() << "\n"; + + // Let's look at UTF8 strings. In HDF5 the size of a string is the size in + // bytes of the buffer required to store the string. A UTF8 symbol/character + // requires 1 to 4 bytes. + // + // The 'a' is 1 byte, the 'α' 2 bytes, therefore a total of 3 bytes (same + // as `utf8_string.size()`). Which including the null character fits into + // 8 bytes. However, 8 bytes would, in general not be enough to store 2 + // UTF8 characters and the null character. Which would require 9 bytes. + std::string utf8_string = "aα"; + auto fixed_utf8_type = + FixedLengthStringType(8, StringPadding::NullTerminated, CharacterSet::Utf8); + file.createDataSet("single_fixed_utf8", scalar_dataspace, fixed_utf8_type).write(utf8_string); + + auto from_utf8 = file.getDataSet("single_fixed_utf8").read<std::string>(); + std::cout << "from_utf8 = '" << from_utf8 << "' size = " << from_utf8.size() << "\n"; + + // Finally, containers of `std::string`s work analogously: + auto ascii_strings = std::vector<std::string>{"123", "456"}; + file.createDataSet("multi_fixed_nullterm", DataSpace::From(ascii_strings), fixed_stringtype) + .write(ascii_strings); + + auto ascii_strings_from_fixed = + file.getDataSet("multi_fixed_nullterm").read<std::vector<std::string>>(); + + // In order to see details of how each is stored in the HDF5 file use: + // h5dump read_write_std_string.h5 + + return 0; +} diff --git a/packages/HighFive/src/examples/read_write_vector_dataset.cpp b/packages/HighFive/src/examples/read_write_vector_dataset.cpp index 605a231a534f96c48e45d94e16ccafa9fc8016ef..9718c1c2bc6d3cb915f6c504dc8d94aabfcccdf9 100644 --- a/packages/HighFive/src/examples/read_write_vector_dataset.cpp +++ b/packages/HighFive/src/examples/read_write_vector_dataset.cpp @@ -10,9 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/read_write_vector_dataset_references.cpp b/packages/HighFive/src/examples/read_write_vector_dataset_references.cpp index 2d9a8ac89703ee6ba137e4e3d5b48b0b5eb619e1..ca08467682184a6c5244dc63707f097012b05c8b 100644 --- a/packages/HighFive/src/examples/read_write_vector_dataset_references.cpp +++ b/packages/HighFive/src/examples/read_write_vector_dataset_references.cpp @@ -10,10 +10,7 @@ #include <string> #include <vector> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5Reference.hpp> +#include <highfive/highfive.hpp> // create a dataset 1D from a vector of int void write_dataset() { diff --git a/packages/HighFive/src/examples/readme_snippet.cpp b/packages/HighFive/src/examples/readme_snippet.cpp index 5ba18243a88f57a1c3cade7bea229699b135234f..160dabce569c7d30dd75d7aee74686f1f339d1c1 100644 --- a/packages/HighFive/src/examples/readme_snippet.cpp +++ b/packages/HighFive/src/examples/readme_snippet.cpp @@ -1,4 +1,4 @@ -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/renaming_objects.cpp b/packages/HighFive/src/examples/renaming_objects.cpp index 0f2a93ac70fcdb63b0ef3f7dd57934e26652053b..f0759e52a2374c9e72f63289e0fe2e0cded9c1bc 100644 --- a/packages/HighFive/src/examples/renaming_objects.cpp +++ b/packages/HighFive/src/examples/renaming_objects.cpp @@ -1,8 +1,4 @@ -#include <highfive/H5File.hpp> -#include <highfive/H5Group.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5Attribute.hpp> +#include <highfive/highfive.hpp> using namespace HighFive; diff --git a/packages/HighFive/src/examples/select_by_id_dataset_cpp11.cpp b/packages/HighFive/src/examples/select_by_id_dataset_cpp11.cpp index 2e0b6f149e824257b575f9193d7a9ea5cfaadc0c..973c57435fc26ff8a2f8e1c6232ed2c837691cb6 100644 --- a/packages/HighFive/src/examples/select_by_id_dataset_cpp11.cpp +++ b/packages/HighFive/src/examples/select_by_id_dataset_cpp11.cpp @@ -11,9 +11,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> const std::string file_name("select_partial_string.h5"); const std::string dataset_name("message"); diff --git a/packages/HighFive/src/examples/select_partial_dataset_cpp11.cpp b/packages/HighFive/src/examples/select_partial_dataset_cpp11.cpp index a21f67ed1c29d6d75a77edfebe21b866c9be2903..1e480c1603f460891d6b64b77f9e4d39500cab1c 100644 --- a/packages/HighFive/src/examples/select_partial_dataset_cpp11.cpp +++ b/packages/HighFive/src/examples/select_partial_dataset_cpp11.cpp @@ -11,9 +11,7 @@ #include <string> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> +#include <highfive/highfive.hpp> const std::string file_name("select_partial_example.h5"); const std::string dataset_name("dset"); diff --git a/packages/HighFive/tests/test_dependent_library/include/simpleton.hpp b/packages/HighFive/tests/test_dependent_library/include/simpleton.hpp index 279187a713e9bbe7b26b13eda7ac5966bf3d134c..b98a09fda04045dbcb72444feff720cbb9db84c4 100644 --- a/packages/HighFive/tests/test_dependent_library/include/simpleton.hpp +++ b/packages/HighFive/tests/test_dependent_library/include/simpleton.hpp @@ -3,21 +3,7 @@ // Include all headers here to catch any missing `inline` statements, since // they will be included by two different compilation units. -#include <highfive/H5Attribute.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5DataType.hpp> -#include <highfive/H5Easy.hpp> -#include <highfive/H5Exception.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5FileDriver.hpp> -#include <highfive/H5Group.hpp> -#include <highfive/H5Object.hpp> -#include <highfive/H5PropertyList.hpp> -#include <highfive/H5Reference.hpp> -#include <highfive/H5Selection.hpp> -#include <highfive/H5Utility.hpp> -#include <highfive/H5Version.hpp> +#include <highfive/highfive.hpp> // Boost should always be found in this setup #include <boost/numeric/ublas/matrix.hpp> diff --git a/packages/HighFive/tests/unit/test_all_types.cpp b/packages/HighFive/tests/unit/test_all_types.cpp index ed265e670424c01e6431cf57a131ec8b7f299539..d74579af6b23176cd99bd9436d690e623fdc5f9b 100644 --- a/packages/HighFive/tests/unit/test_all_types.cpp +++ b/packages/HighFive/tests/unit/test_all_types.cpp @@ -8,21 +8,15 @@ */ #include <string> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5Group.hpp> -#include <highfive/H5Reference.hpp> -#include <highfive/H5Utility.hpp> - #include <catch2/catch_template_test_macros.hpp> +#include <highfive/highfive.hpp> #include "tests_high_five.hpp" using namespace HighFive; TEMPLATE_TEST_CASE("Scalar in DataSet", "[Types]", bool, std::string) { - const std::string FILE_NAME("Test_type.h5"); + const std::string FILE_NAME("rw_dataset_" + typeNameHelper<TestType>() + ".h5"); const std::string DATASET_NAME("dset"); TestType t1{}; @@ -52,7 +46,7 @@ TEMPLATE_TEST_CASE("Scalar in DataSet", "[Types]", bool, std::string) { } TEMPLATE_PRODUCT_TEST_CASE("Scalar in std::vector", "[Types]", std::vector, (bool, std::string)) { - const std::string FILE_NAME("Test_vector.h5"); + const std::string FILE_NAME("rw_dataset_" + typeNameHelper<TestType>() + ".h5"); const std::string DATASET_NAME("dset"); TestType t1(5); @@ -84,7 +78,7 @@ TEMPLATE_PRODUCT_TEST_CASE("Scalar in std::vector<std::vector>", "[Types]", std::vector, (bool, std::string)) { - const std::string FILE_NAME("Test_vector_vector.h5"); + const std::string FILE_NAME("rw_dataset_vector_" + typeNameHelper<TestType>() + ".h5"); const std::string DATASET_NAME("dset"); std::vector<TestType> t1(5); for (auto&& e: t1) { @@ -118,7 +112,7 @@ TEMPLATE_PRODUCT_TEST_CASE("Scalar in std::vector<std::vector>", } TEMPLATE_TEST_CASE("Scalar in std::array", "[Types]", bool, std::string) { - const std::string FILE_NAME("Test_array.h5"); + const std::string FILE_NAME("rw_dataset_array_" + typeNameHelper<TestType>() + ".h5"); const std::string DATASET_NAME("dset"); std::array<TestType, 5> t1{}; @@ -149,7 +143,7 @@ TEMPLATE_TEST_CASE("Scalar in std::array", "[Types]", bool, std::string) { } TEMPLATE_TEST_CASE("Scalar in std::vector<std::array>", "[Types]", bool, std::string) { - const std::string FILE_NAME("Test_vector_array.h5"); + const std::string FILE_NAME("rw_dataset_vector_array_" + typeNameHelper<TestType>() + ".h5"); const std::string DATASET_NAME("dset"); std::vector<std::array<TestType, 6>> t1(5); @@ -182,7 +176,7 @@ TEMPLATE_TEST_CASE("Scalar in std::vector<std::array>", "[Types]", bool, std::st #if HIGHFIVE_CXX_STD >= 17 TEMPLATE_PRODUCT_TEST_CASE("Scalar in std::vector<std::byte>", "[Types]", std::vector, std::byte) { - const std::string FILE_NAME("Test_vector_byte.h5"); + const std::string FILE_NAME("rw_dataset_vector_" + typeNameHelper<TestType>() + ".h5"); const std::string DATASET_NAME("dset"); TestType t1(5, std::byte(0xCD)); diff --git a/packages/HighFive/tests/unit/tests_high_five.hpp b/packages/HighFive/tests/unit/tests_high_five.hpp index 2d23c0879c0828dcd9e2ed8f0e282c6d34869171..0ebd58c44890e65ddbf4ee149d6c13c6662a15e3 100644 --- a/packages/HighFive/tests/unit/tests_high_five.hpp +++ b/packages/HighFive/tests/unit/tests_high_five.hpp @@ -12,6 +12,9 @@ #include <string> #include <vector> #include <tuple> +#include <sstream> +#include <functional> +#include <iomanip> // We don't need windows specific functionality. However, to better detect defects caused by macros, // we include this header. @@ -162,16 +165,22 @@ struct ContentGenerate<std::string> { template <typename T> inline std::string typeNameHelper() { std::string name = typeid(T).name(); -#if defined(WIN32) - // Replace illegal windows file path characters std::replace(std::begin(name), std::end(name), ' ', '_'); std::replace(std::begin(name), std::end(name), '<', '_'); std::replace(std::begin(name), std::end(name), '>', '_'); std::replace(std::begin(name), std::end(name), ':', '_'); -#endif - return name; + + if (name.size() > 64) { + std::stringstream hash; + hash << std::hex << std::hash<std::string>{}(name); + + return hash.str(); + } else { + return name; + } } + template <typename ElemT, typename DataT> inline HighFive::DataSet readWriteDataset(const DataT& ndvec, DataT& result, @@ -193,4 +202,4 @@ inline HighFive::DataSet readWriteDataset(const DataT& ndvec, dataset.read(result); return dataset; -} \ No newline at end of file +} diff --git a/packages/HighFive/tests/unit/tests_high_five_base.cpp b/packages/HighFive/tests/unit/tests_high_five_base.cpp index bba8ded889b600101b0b2a6143cd4931aabd9612..899170d9328683b2832e592871ff7fac917ea58e 100644 --- a/packages/HighFive/tests/unit/tests_high_five_base.cpp +++ b/packages/HighFive/tests/unit/tests_high_five_base.cpp @@ -18,18 +18,11 @@ #include <typeinfo> #include <vector> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5File.hpp> -#include <highfive/H5Group.hpp> -#include <highfive/H5Reference.hpp> -#include <highfive/H5Utility.hpp> -#include <highfive/H5Version.hpp> - #include <catch2/catch_test_macros.hpp> #include <catch2/catch_template_test_macros.hpp> #include <catch2/matchers/catch_matchers_vector.hpp> +#include <highfive/highfive.hpp> #include "tests_high_five.hpp" using namespace HighFive; @@ -317,13 +310,21 @@ TEST_CASE("Test allocation time") { CHECK(alloc_size == data.size() * sizeof(decltype(data)::value_type)); } +/* + * Test to ensure legacy support: DataSet used to have a default constructor. + * However, it is not useful to have a DataSet object that does not actually + * refer to a dataset in a file. Hence, the the default constructor was + * deprecated. + * This test is to ensure that the constructor is not accidentally removed and + * thereby break users' code. + */ TEST_CASE("Test default constructors") { - const std::string file_name("h5_group_test.h5"); + const std::string file_name("h5_default_ctors.h5"); const std::string dataset_name("dset"); File file(file_name, File::Truncate); auto ds = file.createDataSet(dataset_name, std::vector<int>{1, 2, 3, 4, 5}); - DataSet d2; // deprecated as it constructs unsafe objects + DataSet d2; // expect deprecation warning, as it constructs unsafe object // d2.getFile(); // runtime error CHECK(!d2.isValid()); d2 = ds; // copy @@ -2439,7 +2440,7 @@ TEST_CASE("HighFiveHardLinks Group") { } TEST_CASE("HighFiveRename") { - File file("move.h5", File::ReadWrite | File::Create | File::Truncate); + File file("h5_rename.h5", File::ReadWrite | File::Create | File::Truncate); int number = 100; @@ -2464,7 +2465,7 @@ TEST_CASE("HighFiveRename") { } TEST_CASE("HighFiveRenameRelative") { - File file("move.h5", File::ReadWrite | File::Create | File::Truncate); + File file("h5_rename_relative.h5", File::ReadWrite | File::Create | File::Truncate); Group group = file.createGroup("group"); int number = 100; @@ -2928,6 +2929,229 @@ TEST_CASE("HighFiveReadType") { CHECK(t4 == t3); } +class ForwardToAttribute { + public: + ForwardToAttribute(const HighFive::File& file) + : _file(file) {} + + template <class T> + HighFive::Attribute create(const std::string& name, const T& value) { + return _file.createAttribute(name, value); + } + + HighFive::Attribute create(const std::string& name, + const HighFive::DataSpace filespace, + const HighFive::DataType& datatype) { + return _file.createAttribute(name, filespace, datatype); + } + + HighFive::Attribute get(const std::string& name) { + return _file.getAttribute(name); + } + + private: + HighFive::File _file; +}; + +class ForwardToDataSet { + public: + ForwardToDataSet(const HighFive::File& file) + : _file(file) {} + + template <class T> + HighFive::DataSet create(const std::string& name, const T& value) { + return _file.createDataSet(name, value); + } + + HighFive::DataSet create(const std::string& name, + const HighFive::DataSpace filespace, + const HighFive::DataType& datatype) { + return _file.createDataSet(name, filespace, datatype); + } + + HighFive::DataSet get(const std::string& name) { + return _file.getDataSet(name); + } + + private: + HighFive::File _file; +}; + +template <class Proxy> +void check_single_string(Proxy proxy, size_t string_length) { + auto value = std::string(string_length, 'o'); + auto dataspace = DataSpace::From(value); + + auto n_chars = value.size() + 1; + auto n_chars_overlength = n_chars + 10; + auto fixed_length = FixedLengthStringType(n_chars, StringPadding::NullTerminated); + auto overlength_nullterm = FixedLengthStringType(n_chars_overlength, + StringPadding::NullTerminated); + auto overlength_nullpad = FixedLengthStringType(n_chars_overlength, StringPadding::NullPadded); + auto overlength_spacepad = FixedLengthStringType(n_chars_overlength, + StringPadding::SpacePadded); + auto variable_length = VariableLengthStringType(); + + SECTION("automatic") { + proxy.create("auto", value); + REQUIRE(proxy.get("auto").template read<std::string>() == value); + } + + SECTION("fixed length") { + proxy.create("fixed", dataspace, fixed_length).write(value); + REQUIRE(proxy.get("fixed").template read<std::string>() == value); + } + + SECTION("overlength null-terminated") { + proxy.create("overlength_nullterm", dataspace, overlength_nullterm).write(value); + REQUIRE(proxy.get("overlength_nullterm").template read<std::string>() == value); + } + + SECTION("overlength null-padded") { + proxy.create("overlength_nullpad", dataspace, overlength_nullpad).write(value); + auto expected = std::string(n_chars_overlength, '\0'); + expected.replace(0, value.size(), value.data()); + REQUIRE(proxy.get("overlength_nullpad").template read<std::string>() == expected); + } + + SECTION("overlength space-padded") { + proxy.create("overlength_spacepad", dataspace, overlength_spacepad).write(value); + auto expected = std::string(n_chars_overlength, ' '); + expected.replace(0, value.size(), value.data()); + REQUIRE(proxy.get("overlength_spacepad").template read<std::string>() == expected); + } + + SECTION("variable length") { + proxy.create("variable", dataspace, variable_length).write(value); + REQUIRE(proxy.get("variable").template read<std::string>() == value); + } +} + +template <class Proxy> +void check_multiple_string(Proxy proxy, size_t string_length) { + using value_t = std::vector<std::string>; + auto value = value_t{std::string(string_length, 'o'), std::string(string_length, 'x')}; + + auto dataspace = DataSpace::From(value); + + auto string_overlength = string_length + 10; + auto onpoint_nullpad = FixedLengthStringType(string_length, StringPadding::NullPadded); + auto onpoint_spacepad = FixedLengthStringType(string_length, StringPadding::SpacePadded); + + auto overlength_nullterm = FixedLengthStringType(string_overlength, + StringPadding::NullTerminated); + auto overlength_nullpad = FixedLengthStringType(string_overlength, StringPadding::NullPadded); + auto overlength_spacepad = FixedLengthStringType(string_overlength, StringPadding::SpacePadded); + auto variable_length = VariableLengthStringType(); + + auto check = [](const value_t actual, const value_t& expected) { + REQUIRE(actual.size() == expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + REQUIRE(actual[i] == expected[i]); + } + }; + + SECTION("automatic") { + proxy.create("auto", value); + check(proxy.get("auto").template read<value_t>(), value); + } + + SECTION("variable length") { + proxy.create("variable", dataspace, variable_length).write(value); + check(proxy.get("variable").template read<value_t>(), value); + } + + auto make_padded_reference = [&](char pad, size_t n) { + auto expected = std::vector<std::string>(value.size(), std::string(n, pad)); + for (size_t i = 0; i < value.size(); ++i) { + expected[i].replace(0, value[i].size(), value[i].data()); + } + + return expected; + }; + + auto check_fixed_length = [&](const std::string& label, size_t length) { + SECTION(label + " null-terminated") { + auto datatype = FixedLengthStringType(length + 1, StringPadding::NullTerminated); + proxy.create(label + "_nullterm", dataspace, datatype).write(value); + check(proxy.get(label + "_nullterm").template read<value_t>(), value); + } + + SECTION(label + " null-padded") { + auto datatype = FixedLengthStringType(length, StringPadding::NullPadded); + proxy.create(label + "_nullpad", dataspace, datatype).write(value); + auto expected = make_padded_reference('\0', length); + check(proxy.get(label + "_nullpad").template read<value_t>(), expected); + } + + SECTION(label + " space-padded") { + auto datatype = FixedLengthStringType(length, StringPadding::SpacePadded); + proxy.create(label + "_spacepad", dataspace, datatype).write(value); + auto expected = make_padded_reference(' ', length); + check(proxy.get(label + "_spacepad").template read<value_t>(), expected); + } + }; + + check_fixed_length("onpoint", string_length); + check_fixed_length("overlength", string_length + 5); + + + SECTION("underlength null-terminated") { + auto datatype = FixedLengthStringType(string_length, StringPadding::NullTerminated); + REQUIRE_THROWS(proxy.create("underlength_nullterm", dataspace, datatype).write(value)); + } + + SECTION("underlength nullpad") { + auto datatype = FixedLengthStringType(string_length - 1, StringPadding::NullPadded); + REQUIRE_THROWS(proxy.create("underlength_nullpad", dataspace, datatype).write(value)); + } + + SECTION("underlength spacepad") { + auto datatype = FixedLengthStringType(string_length - 1, StringPadding::NullTerminated); + REQUIRE_THROWS(proxy.create("underlength_spacepad", dataspace, datatype).write(value)); + } +} + +TEST_CASE("HighFiveSTDString (dataset, single, short)") { + File file("std_string_dataset_single_short.h5", File::Truncate); + check_single_string(ForwardToDataSet(file), 3); +} + +TEST_CASE("HighFiveSTDString (attribute, single, short)") { + File file("std_string_attribute_single_short.h5", File::Truncate); + check_single_string(ForwardToAttribute(file), 3); +} + +TEST_CASE("HighFiveSTDString (dataset, single, long)") { + File file("std_string_dataset_single_long.h5", File::Truncate); + check_single_string(ForwardToDataSet(file), 256); +} + +TEST_CASE("HighFiveSTDString (attribute, single, long)") { + File file("std_string_attribute_single_long.h5", File::Truncate); + check_single_string(ForwardToAttribute(file), 256); +} + +TEST_CASE("HighFiveSTDString (dataset, multiple, short)") { + File file("std_string_dataset_multiple_short.h5", File::Truncate); + check_multiple_string(ForwardToDataSet(file), 3); +} + +TEST_CASE("HighFiveSTDString (attribute, multiple, short)") { + File file("std_string_attribute_multiple_short.h5", File::Truncate); + check_multiple_string(ForwardToAttribute(file), 3); +} + +TEST_CASE("HighFiveSTDString (dataset, multiple, long)") { + File file("std_string_dataset_multiple_short.h5", File::Truncate); + check_multiple_string(ForwardToDataSet(file), 256); +} + +TEST_CASE("HighFiveSTDString (attribute, multiple, long)") { + File file("std_string_attribute_multiple_short.h5", File::Truncate); + check_multiple_string(ForwardToAttribute(file), 256); +} + TEST_CASE("HighFiveFixedString") { const std::string file_name("array_atomic_types.h5"); const std::string group_1("group1"); @@ -2962,6 +3186,7 @@ TEST_CASE("HighFiveFixedString") { file.createDataSet<char[10]>("ds4", DataSpace(2)).write(strings_fixed); } + { // Cant convert flex-length to fixed-length const char* buffer[] = {"abcd", "1234"}; SilenceHDF5 silencer; diff --git a/packages/HighFive/tests/unit/tests_high_five_easy.cpp b/packages/HighFive/tests/unit/tests_high_five_easy.cpp index 64a7706740fe441990884284d81d0ed4700c6efd..e003c323401a9460332883abd618fef10a9a169d 100644 --- a/packages/HighFive/tests/unit/tests_high_five_easy.cpp +++ b/packages/HighFive/tests/unit/tests_high_five_easy.cpp @@ -56,40 +56,66 @@ TEST_CASE("H5Easy_Compression") { } TEST_CASE("H5Easy_scalar") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_scalar.h5", H5Easy::File::Overwrite); double a = 1.2345; int b = 12345; std::string c = "12345"; + std::complex<double> d = std::complex<double>(1.2345, -5.4321); + std::complex<int32_t> e = std::complex<int32_t>(12345, -54321); H5Easy::dump(file, "/path/to/a", a); H5Easy::dump(file, "/path/to/b", b); H5Easy::dump(file, "/path/to/c", c); H5Easy::dump(file, "/path/to/c", c, H5Easy::DumpMode::Overwrite); + H5Easy::dump(file, "/path/to/d", d); + H5Easy::dump(file, "/path/to/e", e); double a_r = H5Easy::load<double>(file, "/path/to/a"); int b_r = H5Easy::load<int>(file, "/path/to/b"); std::string c_r = H5Easy::load<std::string>(file, "/path/to/c"); + std::complex<double> d_r = H5Easy::load<std::complex<double>>(file, "/path/to/d"); + std::complex<int32_t> e_r = H5Easy::load<std::complex<int32_t>>(file, "/path/to/e"); CHECK(a == a_r); CHECK(b == b_r); CHECK(c == c_r); + CHECK(d == d_r); + CHECK(e == e_r); } TEST_CASE("H5Easy_vector1d") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_vector1d.h5", H5Easy::File::Overwrite); std::vector<size_t> a = {1, 2, 3, 4, 5}; + std::vector<std::complex<double>> b = {std::complex<double>(1, .1), + std::complex<double>(2, -.4), + std::complex<double>(3, .9), + std::complex<double>(4, -.16), + std::complex<double>(5, .25)}; + std::vector<std::complex<int32_t>> c = {std::complex<int32_t>(1, -5), + std::complex<int32_t>(2, -4), + std::complex<int32_t>(3, -3), + std::complex<int32_t>(4, -2), + std::complex<int32_t>(5, -1)}; H5Easy::dump(file, "/path/to/a", a); + H5Easy::dump(file, "/path/to/b", b); + H5Easy::dump(file, "/path/to/c", c); std::vector<size_t> a_r = H5Easy::load<std::vector<size_t>>(file, "/path/to/a"); + std::vector<std::complex<double>> b_r = + H5Easy::load<std::vector<std::complex<double>>>(file, "/path/to/b"); + std::vector<std::complex<int32_t>> c_r = + H5Easy::load<std::vector<std::complex<int32_t>>>(file, "/path/to/c"); CHECK(a == a_r); + CHECK(b == b_r); + CHECK(c == c_r); } TEST_CASE("H5Easy_vector2d") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_vector2d.h5", H5Easy::File::Overwrite); std::vector<std::vector<size_t>> a({{0, 1}, {2, 3}, {4, 5}}); @@ -101,7 +127,7 @@ TEST_CASE("H5Easy_vector2d") { } TEST_CASE("H5Easy_vector2d_compression") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_vector2d_compression.h5", H5Easy::File::Overwrite); std::vector<std::vector<size_t>> a({{0, 1}, {2, 3}, {4, 5}}); @@ -118,7 +144,7 @@ TEST_CASE("H5Easy_vector2d_compression") { } TEST_CASE("H5Easy_vector3d") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_vector3d.h5", H5Easy::File::Overwrite); using type = std::vector<std::vector<std::vector<size_t>>>; @@ -132,7 +158,7 @@ TEST_CASE("H5Easy_vector3d") { } TEST_CASE("H5Easy_Attribute_scalar") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_attribute_scalar.h5", H5Easy::File::Overwrite); double a = 1.2345; int b = 12345; @@ -155,7 +181,7 @@ TEST_CASE("H5Easy_Attribute_scalar") { #ifdef H5_USE_XTENSOR TEST_CASE("H5Easy_extend1d") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_extend1d.h5", H5Easy::File::Overwrite); for (size_t i = 0; i < 10; ++i) { H5Easy::dump(file, "/path/to/A", i, {i}); @@ -172,7 +198,7 @@ TEST_CASE("H5Easy_extend1d") { } TEST_CASE("H5Easy_extend2d") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_extend2d.h5", H5Easy::File::Overwrite); for (size_t i = 0; i < 10; ++i) { for (size_t j = 0; j < 5; ++j) { @@ -193,7 +219,7 @@ TEST_CASE("H5Easy_extend2d") { } TEST_CASE("H5Easy_xtensor") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_xtensor.h5", H5Easy::File::Overwrite); xt::xtensor<double, 2> A = 100. * xt::random::randn<double>({20, 5}); xt::xtensor<int, 2> B = A; @@ -209,7 +235,7 @@ TEST_CASE("H5Easy_xtensor") { } TEST_CASE("H5Easy_xarray") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_xarray.h5", H5Easy::File::Overwrite); xt::xarray<double> A = 100. * xt::random::randn<double>({20, 5}); xt::xarray<int> B = A; @@ -225,7 +251,7 @@ TEST_CASE("H5Easy_xarray") { } TEST_CASE("H5Easy_view") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_view.h5", H5Easy::File::Overwrite); xt::xtensor<double, 2> A = 100. * xt::random::randn<double>({20, 5}); auto a = xt::view(A, xt::range(0, 10), xt::range(0, 10)); @@ -238,7 +264,7 @@ TEST_CASE("H5Easy_view") { } TEST_CASE("H5Easy_xtensor_compress") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_xtensor_compress.h5", H5Easy::File::Overwrite); xt::xtensor<double, 2> A = 100. * xt::random::randn<double>({20, 5}); xt::xtensor<int, 2> B = A; @@ -260,7 +286,7 @@ TEST_CASE("H5Easy_xtensor_compress") { } TEST_CASE("H5Easy_Attribute_xtensor") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_attribute_xtensor.h5", H5Easy::File::Overwrite); xt::xtensor<double, 2> A = 100. * xt::random::randn<double>({20, 5}); xt::xtensor<int, 2> B = A; @@ -280,7 +306,7 @@ TEST_CASE("H5Easy_Attribute_xtensor") { #ifdef H5_USE_EIGEN TEST_CASE("H5Easy_Eigen_MatrixX") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_eigen_MatrixX.h5", H5Easy::File::Overwrite); Eigen::MatrixXd A = 100. * Eigen::MatrixXd::Random(20, 5); Eigen::MatrixXi B = A.cast<int>(); @@ -296,7 +322,7 @@ TEST_CASE("H5Easy_Eigen_MatrixX") { } TEST_CASE("H5Easy_Eigen_ArrayXX") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_eigen_ArrayXX.h5", H5Easy::File::Overwrite); Eigen::ArrayXXf A = 100. * Eigen::ArrayXXf::Random(20, 5); Eigen::ArrayXXi B = A.cast<int>(); @@ -312,7 +338,7 @@ TEST_CASE("H5Easy_Eigen_ArrayXX") { } TEST_CASE("H5Easy_Eigen_ArrayX") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_eigen_ArrayX.h5", H5Easy::File::Overwrite); Eigen::ArrayXf A = Eigen::ArrayXf::Random(50); Eigen::ArrayXi B = A.cast<int>(); @@ -329,7 +355,7 @@ TEST_CASE("H5Easy_Eigen_ArrayX") { TEST_CASE("H5Easy_Eigen_VectorX") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_eigen_VectorX.h5", H5Easy::File::Overwrite); Eigen::VectorXd A = 100. * Eigen::VectorXd::Random(20); Eigen::VectorXi B = A.cast<int>(); @@ -348,7 +374,7 @@ TEST_CASE("H5Easy_Eigen_MatrixXRowMajor") { typedef Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> MatrixXd; typedef Eigen::Matrix<int, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> MatrixXi; - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("H5Easy_Eigen_MatrixXRowMajor.h5", H5Easy::File::Overwrite); MatrixXd A = 100. * MatrixXd::Random(20, 5); MatrixXi B = A.cast<int>(); @@ -367,7 +393,7 @@ TEST_CASE("H5Easy_Eigen_VectorXRowMajor") { typedef Eigen::Matrix<double, 1, Eigen::Dynamic, Eigen::RowMajor> VectorXd; typedef Eigen::Matrix<int, 1, Eigen::Dynamic, Eigen::RowMajor> VectorXi; - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_eigen_VectorXRowMajor.h5", H5Easy::File::Overwrite); VectorXd A = 100. * VectorXd::Random(20); VectorXi B = A.cast<int>(); @@ -383,7 +409,7 @@ TEST_CASE("H5Easy_Eigen_VectorXRowMajor") { } TEST_CASE("H5Easy_Eigen_Map") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_eigen_Map.h5", H5Easy::File::Overwrite); std::vector<int> A = {1, 2, 3, 4, 5, 6, 7, 8, 9}; Eigen::Map<Eigen::VectorXi> mapped_vector(A.data(), static_cast<int>(A.size())); @@ -396,7 +422,7 @@ TEST_CASE("H5Easy_Eigen_Map") { } TEST_CASE("H5Easy_Attribute_Eigen_MatrixX") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_attribute_eigen_MatrixX.h5", H5Easy::File::Overwrite); Eigen::MatrixXd A = 100. * Eigen::MatrixXd::Random(20, 5); Eigen::MatrixXi B = A.cast<int>(); @@ -415,7 +441,7 @@ TEST_CASE("H5Easy_Attribute_Eigen_MatrixX") { #ifdef H5_USE_OPENCV TEST_CASE("H5Easy_OpenCV_Mat_") { - H5Easy::File file("test.h5", H5Easy::File::Overwrite); + H5Easy::File file("h5easy_opencv_Mat_.h5", H5Easy::File::Overwrite); using T = typename cv::Mat_<double>; @@ -436,6 +462,7 @@ TEST_CASE("H5Easy_OpenCV_Mat_") { H5Easy::dump(file, "/path/to/A", A); H5Easy::dumpAttribute(file, "/path/to/A", "attr", A); + T A_r = H5Easy::load<T>(file, "/path/to/A"); T B_r = H5Easy::loadAttribute<T>(file, "/path/to/A", "attr"); diff --git a/packages/HighFive/tests/unit/tests_high_five_multi_dims.cpp b/packages/HighFive/tests/unit/tests_high_five_multi_dims.cpp index 4a4b8231cef368fc977825c116b916ff43ecc6e3..442f1c9cc5d92ceb3a4ef4729d03a724ab900e8c 100644 --- a/packages/HighFive/tests/unit/tests_high_five_multi_dims.cpp +++ b/packages/HighFive/tests/unit/tests_high_five_multi_dims.cpp @@ -10,8 +10,7 @@ #include <string> #include <iostream> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5File.hpp> +#include <highfive/highfive.hpp> #ifdef H5_USE_BOOST diff --git a/packages/HighFive/tests/unit/tests_high_five_parallel.cpp b/packages/HighFive/tests/unit/tests_high_five_parallel.cpp index b5518f48c661628b38ee1c284e4529d0fac83b95..8b096205e8528df17c6ff03cb68c592c81e0c9bd 100644 --- a/packages/HighFive/tests/unit/tests_high_five_parallel.cpp +++ b/packages/HighFive/tests/unit/tests_high_five_parallel.cpp @@ -13,15 +13,11 @@ #include <typeinfo> #include <vector> -#include <highfive/H5File.hpp> -#include <highfive/H5DataSet.hpp> -#include <highfive/H5DataSpace.hpp> -#include <highfive/H5Group.hpp> - #include <catch2/catch_test_macros.hpp> #include <catch2/catch_template_test_macros.hpp> #include <catch2/catch_session.hpp> +#include <highfive/highfive.hpp> #include "tests_high_five.hpp" using namespace HighFive;