diff --git a/packages/Catch2/.gitrepo b/packages/Catch2/.gitrepo index f91066d5da01f7b0ba64a16516f4b3e3a151c168..eeda199924a84e824ecd08a9d7917712ed180085 100644 --- a/packages/Catch2/.gitrepo +++ b/packages/Catch2/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = git@github.com:catchorg/Catch2.git branch = master - commit = d399a308d04b885433396f4198c0cda999f28d77 - parent = 1bb31be07095e1b4f28fe9012b634f2a9e866180 + commit = 38f897c88772d6640196d1bd617ebbcaf44bc05a + parent = 2f7310d11077fd2ebdcdc77443bb9560060372c3 cmdver = 0.4.1 method = merge diff --git a/packages/Catch2/docs/command-line.md b/packages/Catch2/docs/command-line.md index 51eeb2f6d7f7a0d6fff5951905f4de768bb4ae48..2b506f167753b4d825953de78e24093482192740 100644 --- a/packages/Catch2/docs/command-line.md +++ b/packages/Catch2/docs/command-line.md @@ -243,15 +243,23 @@ This option lists all available tests in a non-indented form, one on each line. Test cases are ordered one of three ways: - ### decl -Declaration order (this is the default order if no --order argument is provided). The order the tests were originally declared in. Note that ordering between files is not guaranteed and is implementation dependent. +Declaration order (this is the default order if no --order argument is provided). +Tests in the same TU are sorted using their declaration orders, different +TUs are in an implementation (linking) dependent order. + ### lex -Lexicographically sorted. Tests are sorted, alpha-numerically, by name. +Lexicographic order. Tests are sorted by their name, their tags are ignored. + ### rand -Randomly sorted. Test names are sorted using ```std::random_shuffle()```. By default the random number generator is seeded with 0 - and so the order is repeatable. To control the random seed see <a href="#rng-seed">rng-seed</a>. + +Randomly sorted. The order is dependent on Catch2's random seed (see +[`--rng-seed`](#rng-seed)), and is subset invariant. What this means +is that as long as the random seed is fixed, running only some tests +(e.g. via tag) does not change their relative order. + <a id="rng-seed"></a> ## Specify a seed for the Random Number Generator diff --git a/packages/Catch2/include/internal/catch_compiler_capabilities.h b/packages/Catch2/include/internal/catch_compiler_capabilities.h index 1529f582e2f56c47d5bc6892228ee7e717d3e755..26351c7acbfdcc7a664668080a656aa6c393fe14 100644 --- a/packages/Catch2/include/internal/catch_compiler_capabilities.h +++ b/packages/Catch2/include/internal/catch_compiler_capabilities.h @@ -39,7 +39,7 @@ #endif -#if defined(CATCH_CPP17_OR_GREATER) +#if defined(__cpp_lib_uncaught_exceptions) # define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS #endif @@ -58,7 +58,20 @@ # define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic push" ) # define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic pop" ) +// As of this writing, IBM XL's implementation of __builtin_constant_p has a bug +// which results in calls to destructors being emitted for each temporary, +// without a matching initialization. In practice, this can result in something +// like `std::string::~string` being called on an uninitialized value. +// +// For example, this code will likely segfault under IBM XL: +// ``` +// REQUIRE(std::string("12") + "34" == "1234") +// ``` +// +// Therefore, `CATCH_INTERNAL_IGNORE_BUT_WARN` is not implemented. +# if !defined(__ibmxl__) # define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) /* NOLINT(cppcoreguidelines-pro-type-vararg) */ +# endif # define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ diff --git a/packages/Catch2/include/internal/catch_matchers_vector.h b/packages/Catch2/include/internal/catch_matchers_vector.h index 2dc8a7150c7d2cf51bdc413719f76cb4485bdcaf..084130f5227d768f49417790c0db44e353927d99 100644 --- a/packages/Catch2/include/internal/catch_matchers_vector.h +++ b/packages/Catch2/include/internal/catch_matchers_vector.h @@ -17,12 +17,12 @@ namespace Catch { namespace Matchers { namespace Vector { - template<typename T> - struct ContainsElementMatcher : MatcherBase<std::vector<T>> { + template<typename T, typename Alloc> + struct ContainsElementMatcher : MatcherBase<std::vector<T, Alloc>> { ContainsElementMatcher(T const &comparator) : m_comparator( comparator) {} - bool match(std::vector<T> const &v) const override { + bool match(std::vector<T, Alloc> const &v) const override { for (auto const& el : v) { if (el == m_comparator) { return true; @@ -38,12 +38,12 @@ namespace Matchers { T const& m_comparator; }; - template<typename T> - struct ContainsMatcher : MatcherBase<std::vector<T>> { + template<typename T, typename AllocComp, typename AllocMatch> + struct ContainsMatcher : MatcherBase<std::vector<T, AllocMatch>> { - ContainsMatcher(std::vector<T> const &comparator) : m_comparator( comparator ) {} + ContainsMatcher(std::vector<T, AllocComp> const &comparator) : m_comparator( comparator ) {} - bool match(std::vector<T> const &v) const override { + bool match(std::vector<T, AllocMatch> const &v) const override { // !TBD: see note in EqualsMatcher if (m_comparator.size() > v.size()) return false; @@ -65,18 +65,18 @@ namespace Matchers { return "Contains: " + ::Catch::Detail::stringify( m_comparator ); } - std::vector<T> const& m_comparator; + std::vector<T, AllocComp> const& m_comparator; }; - template<typename T> - struct EqualsMatcher : MatcherBase<std::vector<T>> { + template<typename T, typename AllocComp, typename AllocMatch> + struct EqualsMatcher : MatcherBase<std::vector<T, AllocMatch>> { - EqualsMatcher(std::vector<T> const &comparator) : m_comparator( comparator ) {} + EqualsMatcher(std::vector<T, AllocComp> const &comparator) : m_comparator( comparator ) {} - bool match(std::vector<T> const &v) const override { + bool match(std::vector<T, AllocMatch> const &v) const override { // !TBD: This currently works if all elements can be compared using != // - a more general approach would be via a compare template that defaults - // to using !=. but could be specialised for, e.g. std::vector<T> etc + // to using !=. but could be specialised for, e.g. std::vector<T, Alloc> etc // - then just call that directly if (m_comparator.size() != v.size()) return false; @@ -88,15 +88,15 @@ namespace Matchers { std::string describe() const override { return "Equals: " + ::Catch::Detail::stringify( m_comparator ); } - std::vector<T> const& m_comparator; + std::vector<T, AllocComp> const& m_comparator; }; - template<typename T> - struct ApproxMatcher : MatcherBase<std::vector<T>> { + template<typename T, typename AllocComp, typename AllocMatch> + struct ApproxMatcher : MatcherBase<std::vector<T, AllocMatch>> { - ApproxMatcher(std::vector<T> const& comparator) : m_comparator( comparator ) {} + ApproxMatcher(std::vector<T, AllocComp> const& comparator) : m_comparator( comparator ) {} - bool match(std::vector<T> const &v) const override { + bool match(std::vector<T, AllocMatch> const &v) const override { if (m_comparator.size() != v.size()) return false; for (std::size_t i = 0; i < v.size(); ++i) @@ -123,14 +123,14 @@ namespace Matchers { return *this; } - std::vector<T> const& m_comparator; + std::vector<T, AllocComp> const& m_comparator; mutable Catch::Detail::Approx approx = Catch::Detail::Approx::custom(); }; - template<typename T> - struct UnorderedEqualsMatcher : MatcherBase<std::vector<T>> { - UnorderedEqualsMatcher(std::vector<T> const& target) : m_target(target) {} - bool match(std::vector<T> const& vec) const override { + template<typename T, typename AllocComp, typename AllocMatch> + struct UnorderedEqualsMatcher : MatcherBase<std::vector<T, AllocMatch>> { + UnorderedEqualsMatcher(std::vector<T, AllocComp> const& target) : m_target(target) {} + bool match(std::vector<T, AllocMatch> const& vec) const override { // Note: This is a reimplementation of std::is_permutation, // because I don't want to include <algorithm> inside the common path if (m_target.size() != vec.size()) { @@ -143,7 +143,7 @@ namespace Matchers { return "UnorderedEquals: " + ::Catch::Detail::stringify(m_target); } private: - std::vector<T> const& m_target; + std::vector<T, AllocComp> const& m_target; }; } // namespace Vector @@ -151,29 +151,29 @@ namespace Matchers { // The following functions create the actual matcher objects. // This allows the types to be inferred - template<typename T> - Vector::ContainsMatcher<T> Contains( std::vector<T> const& comparator ) { - return Vector::ContainsMatcher<T>( comparator ); + template<typename T, typename AllocComp, typename AllocMatch = AllocComp> + Vector::ContainsMatcher<T, AllocComp, AllocMatch> Contains( std::vector<T, AllocComp> const& comparator ) { + return Vector::ContainsMatcher<T, AllocComp, AllocMatch>( comparator ); } - template<typename T> - Vector::ContainsElementMatcher<T> VectorContains( T const& comparator ) { - return Vector::ContainsElementMatcher<T>( comparator ); + template<typename T, typename Alloc = std::allocator<T>> + Vector::ContainsElementMatcher<T, Alloc> VectorContains( T const& comparator ) { + return Vector::ContainsElementMatcher<T, Alloc>( comparator ); } - template<typename T> - Vector::EqualsMatcher<T> Equals( std::vector<T> const& comparator ) { - return Vector::EqualsMatcher<T>( comparator ); + template<typename T, typename AllocComp, typename AllocMatch = AllocComp> + Vector::EqualsMatcher<T, AllocComp, AllocMatch> Equals( std::vector<T, AllocComp> const& comparator ) { + return Vector::EqualsMatcher<T, AllocComp, AllocMatch>( comparator ); } - template<typename T> - Vector::ApproxMatcher<T> Approx( std::vector<T> const& comparator ) { - return Vector::ApproxMatcher<T>( comparator ); + template<typename T, typename AllocComp, typename AllocMatch = AllocComp> + Vector::ApproxMatcher<T, AllocComp, AllocMatch> Approx( std::vector<T, AllocComp> const& comparator ) { + return Vector::ApproxMatcher<T, AllocComp, AllocMatch>( comparator ); } - template<typename T> - Vector::UnorderedEqualsMatcher<T> UnorderedEquals(std::vector<T> const& target) { - return Vector::UnorderedEqualsMatcher<T>(target); + template<typename T, typename AllocComp, typename AllocMatch = AllocComp> + Vector::UnorderedEqualsMatcher<T, AllocComp, AllocMatch> UnorderedEquals(std::vector<T, AllocComp> const& target) { + return Vector::UnorderedEqualsMatcher<T, AllocComp, AllocMatch>( target ); } } // namespace Matchers diff --git a/packages/Catch2/include/internal/catch_test_case_registry_impl.cpp b/packages/Catch2/include/internal/catch_test_case_registry_impl.cpp index b254ca08e6910f411b46cbc8ddf7e57d0586f5c0..065f70bda60c4936869e8e0543fe561f7660c24d 100644 --- a/packages/Catch2/include/internal/catch_test_case_registry_impl.cpp +++ b/packages/Catch2/include/internal/catch_test_case_registry_impl.cpp @@ -15,27 +15,78 @@ #include "catch_string_manip.h" #include "catch_test_case_info.h" +#include <algorithm> #include <sstream> namespace Catch { - std::vector<TestCase> sortTests( IConfig const& config, std::vector<TestCase> const& unsortedTestCases ) { + namespace { + struct TestHasher { + explicit TestHasher(Catch::SimplePcg32& rng) { + basis = rng(); + basis <<= 32; + basis |= rng(); + } + + uint64_t basis; + + uint64_t operator()(TestCase const& t) const { + // Modified FNV-1a hash + static constexpr uint64_t prime = 1099511628211; + uint64_t hash = basis; + for (const char c : t.name) { + hash ^= c; + hash *= prime; + } + return hash; + } + }; + } // end unnamed namespace - std::vector<TestCase> sorted = unsortedTestCases; + std::vector<TestCase> sortTests( IConfig const& config, std::vector<TestCase> const& unsortedTestCases ) { switch( config.runOrder() ) { - case RunTests::InLexicographicalOrder: - std::sort( sorted.begin(), sorted.end() ); - break; - case RunTests::InRandomOrder: - seedRng( config ); - std::shuffle( sorted.begin(), sorted.end(), rng() ); - break; case RunTests::InDeclarationOrder: // already in declaration order break; + + case RunTests::InLexicographicalOrder: { + std::vector<TestCase> sorted = unsortedTestCases; + std::sort( sorted.begin(), sorted.end() ); + return sorted; + } + + case RunTests::InRandomOrder: { + seedRng( config ); + TestHasher h( rng() ); + + using hashedTest = std::pair<uint64_t, TestCase const*>; + std::vector<hashedTest> indexed_tests; + indexed_tests.reserve( unsortedTestCases.size() ); + + for (auto const& testCase : unsortedTestCases) { + indexed_tests.emplace_back(h(testCase), &testCase); + } + + std::sort(indexed_tests.begin(), indexed_tests.end(), + [](hashedTest const& lhs, hashedTest const& rhs) { + if (lhs.first == rhs.first) { + return lhs.second->name < rhs.second->name; + } + return lhs.first < rhs.first; + }); + + std::vector<TestCase> sorted; + sorted.reserve( indexed_tests.size() ); + + for (auto const& hashed : indexed_tests) { + sorted.emplace_back(*hashed.second); + } + + return sorted; + } } - return sorted; + return unsortedTestCases; } bool isThrowSafe( TestCase const& testCase, IConfig const& config ) { diff --git a/packages/Catch2/include/internal/catch_test_spec_parser.cpp b/packages/Catch2/include/internal/catch_test_spec_parser.cpp index dad15c01c9a9d32eb3b076af3d6ff01040748fea..40fcdcc39502f680083adfe44c2b33e17ab324db 100644 --- a/packages/Catch2/include/internal/catch_test_spec_parser.cpp +++ b/packages/Catch2/include/internal/catch_test_spec_parser.cpp @@ -168,6 +168,7 @@ namespace Catch { m_pos = m_arg.size(); m_substring.clear(); m_patternName.clear(); + m_realPatternPos = 0; return false; } endMode(); @@ -186,6 +187,7 @@ namespace Catch { } m_patternName.clear(); + m_realPatternPos = 0; return token; } diff --git a/packages/Catch2/projects/CMakeLists.txt b/packages/Catch2/projects/CMakeLists.txt index 0aafc1780d708cb75d5e77cbef5392d5ceb7aad6..08badb809a995f317648e91c9f0b8a5aedcb5ac7 100644 --- a/packages/Catch2/projects/CMakeLists.txt +++ b/packages/Catch2/projects/CMakeLists.txt @@ -451,6 +451,8 @@ set_tests_properties(TestsInFile::InvalidTestNames-1 PROPERTIES PASS_REGULAR_EXP add_test(NAME TestsInFile::InvalidTestNames-2 COMMAND $<TARGET_FILE:SelfTest> "-f ${CATCH_DIR}/projects/SelfTest/Misc/invalid-test-names.input") set_tests_properties(TestsInFile::InvalidTestNames-2 PROPERTIES PASS_REGULAR_EXPRESSION "No tests ran") +add_test(NAME RandomTestOrdering COMMAND ${PYTHON_EXECUTABLE} + ${CATCH_DIR}/projects/TestScripts/testRandomOrder.py $<TARGET_FILE:SelfTest>) if (CATCH_USE_VALGRIND) add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>) diff --git a/packages/Catch2/projects/SelfTest/Baselines/compact.sw.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/compact.sw.approved.txt index eef23b5a972c01e953d1a08b77b016dbd973ee23..280f697adbaaed137e65cd260bb2cb7d1a07f8b5 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -23,6 +23,9 @@ This would not be caught previously Nor would this Tricky.tests.cpp:<line number>: failed: explicitly with 1 message: '1514' Compilation.tests.cpp:<line number>: passed: std::is_same<TypeList<int>, TypeList<int>>::value for: true +CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase("spec . char")) for: true +CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase("spec , char")) for: true +CmdLine.tests.cpp:<line number>: passed: !(spec.matches(fakeTestCase(R"(spec \, char)"))) for: !false Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception' Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception' Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42' @@ -1528,18 +1531,26 @@ Matchers.tests.cpp:<line number>: failed: empty, Approx(t1) for: { } is approx: Matchers.tests.cpp:<line number>: failed: v1, Approx(v2) for: { 2.0, 4.0, 6.0 } is approx: { 1.0, 3.0, 5.0 } Matchers.tests.cpp:<line number>: passed: v, VectorContains(1) for: { 1, 2, 3 } Contains: 1 Matchers.tests.cpp:<line number>: passed: v, VectorContains(2) for: { 1, 2, 3 } Contains: 2 +Matchers.tests.cpp:<line number>: passed: v5, (VectorContains<int, CustomAllocator<int>>(2)) for: { 1, 2, 3 } Contains: 2 Matchers.tests.cpp:<line number>: passed: v, Contains(v2) for: { 1, 2, 3 } Contains: { 1, 2 } +Matchers.tests.cpp:<line number>: passed: v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2)) for: { 1, 2, 3 } Contains: { 1, 2 } Matchers.tests.cpp:<line number>: passed: v, Contains(v2) for: { 1, 2, 3 } Contains: { 1, 2, 3 } Matchers.tests.cpp:<line number>: passed: v, Contains(empty) for: { 1, 2, 3 } Contains: { } Matchers.tests.cpp:<line number>: passed: empty, Contains(empty) for: { } Contains: { } +Matchers.tests.cpp:<line number>: passed: v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2)) for: { 1, 2, 3 } Contains: { 1, 2, 3 } +Matchers.tests.cpp:<line number>: passed: v5, Contains(v6) for: { 1, 2, 3 } Contains: { 1, 2 } Matchers.tests.cpp:<line number>: passed: v, VectorContains(1) && VectorContains(2) for: { 1, 2, 3 } ( Contains: 1 and Contains: 2 ) Matchers.tests.cpp:<line number>: passed: v, Equals(v) for: { 1, 2, 3 } Equals: { 1, 2, 3 } Matchers.tests.cpp:<line number>: passed: empty, Equals(empty) for: { } Equals: { } Matchers.tests.cpp:<line number>: passed: v, Equals(v2) for: { 1, 2, 3 } Equals: { 1, 2, 3 } +Matchers.tests.cpp:<line number>: passed: v5, (Equals<int, std::allocator<int>, CustomAllocator<int>>(v2)) for: { 1, 2, 3 } Equals: { 1, 2, 3 } +Matchers.tests.cpp:<line number>: passed: v5, Equals(v6) for: { 1, 2, 3 } Equals: { 1, 2, 3 } Matchers.tests.cpp:<line number>: passed: v, UnorderedEquals(v) for: { 1, 2, 3 } UnorderedEquals: { 1, 2, 3 } Matchers.tests.cpp:<line number>: passed: empty, UnorderedEquals(empty) for: { } UnorderedEquals: { } Matchers.tests.cpp:<line number>: passed: permuted, UnorderedEquals(v) for: { 1, 3, 2 } UnorderedEquals: { 1, 2, 3 } Matchers.tests.cpp:<line number>: passed: permuted, UnorderedEquals(v) for: { 2, 3, 1 } UnorderedEquals: { 1, 2, 3 } +Matchers.tests.cpp:<line number>: passed: v5, (UnorderedEquals<int, std::allocator<int>, CustomAllocator<int>>(permuted)) for: { 1, 2, 3 } UnorderedEquals: { 2, 3, 1 } +Matchers.tests.cpp:<line number>: passed: v5_permuted, UnorderedEquals(v5) for: { 1, 3, 2 } UnorderedEquals: { 1, 2, 3 } Matchers.tests.cpp:<line number>: failed: v, VectorContains(-1) for: { 1, 2, 3 } Contains: -1 Matchers.tests.cpp:<line number>: failed: empty, VectorContains(1) for: { } Contains: 1 Matchers.tests.cpp:<line number>: failed: empty, Contains(v) for: { } Contains: { 1, 2, 3 } diff --git a/packages/Catch2/projects/SelfTest/Baselines/console.std.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/console.std.approved.txt index c0357adb4d8fea84705e014dd596cec5df7f4dbe..6b29a98ca99fb8b15bc81bd0f2524d99fb0c8c8b 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/console.std.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/console.std.approved.txt @@ -1380,6 +1380,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 307 | 233 passed | 70 failed | 4 failed as expected -assertions: 1677 | 1525 passed | 131 failed | 21 failed as expected +test cases: 308 | 234 passed | 70 failed | 4 failed as expected +assertions: 1688 | 1536 passed | 131 failed | 21 failed as expected diff --git a/packages/Catch2/projects/SelfTest/Baselines/console.sw.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/console.sw.approved.txt index 1b0a8ce2125d4a258bee2c626dfa248ba14ed257..fcb7a802dc7be5a9754550ae4d9b9701fde5f55a 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/console.sw.approved.txt @@ -188,6 +188,27 @@ Compilation.tests.cpp:<line number>: PASSED: with expansion: true +------------------------------------------------------------------------------- +#1905 -- test spec parser properly clears internal state between compound tests +------------------------------------------------------------------------------- +CmdLine.tests.cpp:<line number> +............................................................................... + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE( spec.matches(fakeTestCase("spec . char")) ) +with expansion: + true + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE( spec.matches(fakeTestCase("spec , char")) ) +with expansion: + true + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( spec.matches(fakeTestCase(R"(spec \, char)")) ) +with expansion: + !false + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -11183,6 +11204,11 @@ Matchers.tests.cpp:<line number>: PASSED: with expansion: { 1, 2, 3 } Contains: 2 +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, (VectorContains<int, CustomAllocator<int>>(2)) ) +with expansion: + { 1, 2, 3 } Contains: 2 + ------------------------------------------------------------------------------- Vector matchers Contains (vector) @@ -11195,6 +11221,11 @@ Matchers.tests.cpp:<line number>: PASSED: with expansion: { 1, 2, 3 } Contains: { 1, 2 } +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2)) ) +with expansion: + { 1, 2, 3 } Contains: { 1, 2 } + Matchers.tests.cpp:<line number>: PASSED: CHECK_THAT( v, Contains(v2) ) with expansion: @@ -11210,6 +11241,16 @@ Matchers.tests.cpp:<line number>: PASSED: with expansion: { } Contains: { } +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2)) ) +with expansion: + { 1, 2, 3 } Contains: { 1, 2, 3 } + +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, Contains(v6) ) +with expansion: + { 1, 2, 3 } Contains: { 1, 2 } + ------------------------------------------------------------------------------- Vector matchers Contains (element), composed @@ -11244,6 +11285,16 @@ Matchers.tests.cpp:<line number>: PASSED: with expansion: { 1, 2, 3 } Equals: { 1, 2, 3 } +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, (Equals<int, std::allocator<int>, CustomAllocator<int>>(v2)) ) +with expansion: + { 1, 2, 3 } Equals: { 1, 2, 3 } + +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, Equals(v6) ) +with expansion: + { 1, 2, 3 } Equals: { 1, 2, 3 } + ------------------------------------------------------------------------------- Vector matchers UnorderedEquals @@ -11271,6 +11322,16 @@ Matchers.tests.cpp:<line number>: PASSED: with expansion: { 2, 3, 1 } UnorderedEquals: { 1, 2, 3 } +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5, (UnorderedEquals<int, std::allocator<int>, CustomAllocator<int>>(permuted)) ) +with expansion: + { 1, 2, 3 } UnorderedEquals: { 2, 3, 1 } + +Matchers.tests.cpp:<line number>: PASSED: + CHECK_THAT( v5_permuted, UnorderedEquals(v5) ) +with expansion: + { 1, 3, 2 } UnorderedEquals: { 1, 2, 3 } + ------------------------------------------------------------------------------- Vector matchers that fail Contains (element) @@ -13427,6 +13488,6 @@ Misc.tests.cpp:<line number> Misc.tests.cpp:<line number>: PASSED: =============================================================================== -test cases: 307 | 217 passed | 86 failed | 4 failed as expected -assertions: 1694 | 1525 passed | 148 failed | 21 failed as expected +test cases: 308 | 218 passed | 86 failed | 4 failed as expected +assertions: 1705 | 1536 passed | 148 failed | 21 failed as expected diff --git a/packages/Catch2/projects/SelfTest/Baselines/console.swa4.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/console.swa4.approved.txt index 322b815864edddca2a52a477c7038de176f52385..e32fd88f425033e017773cc22799420740249562 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/console.swa4.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/console.swa4.approved.txt @@ -188,6 +188,27 @@ Compilation.tests.cpp:<line number>: PASSED: with expansion: true +------------------------------------------------------------------------------- +#1905 -- test spec parser properly clears internal state between compound tests +------------------------------------------------------------------------------- +CmdLine.tests.cpp:<line number> +............................................................................... + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE( spec.matches(fakeTestCase("spec . char")) ) +with expansion: + true + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE( spec.matches(fakeTestCase("spec , char")) ) +with expansion: + true + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( spec.matches(fakeTestCase(R"(spec \, char)")) ) +with expansion: + !false + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -368,6 +389,6 @@ Condition.tests.cpp:<line number>: FAILED: CHECK( true != true ) =============================================================================== -test cases: 19 | 14 passed | 3 failed | 2 failed as expected -assertions: 42 | 35 passed | 4 failed | 3 failed as expected +test cases: 20 | 15 passed | 3 failed | 2 failed as expected +assertions: 45 | 38 passed | 4 failed | 3 failed as expected diff --git a/packages/Catch2/projects/SelfTest/Baselines/junit.sw.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/junit.sw.approved.txt index e8f7c09ff42a5f251996a008f0fd176df6d49271..76284a2fba97b31bfed2bc87e48ecc4371b2f2cc 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesloose text artifact > - <testsuite name="<exe-name>" errors="17" failures="132" tests="1695" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="17" failures="132" tests="1706" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <properties> <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/> <property name="random-seed" value="1"/> @@ -30,6 +30,7 @@ Nor would this </system-err> </testcase> <testcase classname="<exe-name>.global" name="#1548" time="{duration}"/> + <testcase classname="<exe-name>.global" name="#1905 -- test spec parser properly clears internal state between compound tests" time="{duration}"/> <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}"> <error type="TEST_CASE"> FAILED: diff --git a/packages/Catch2/projects/SelfTest/Baselines/sonarqube.sw.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/sonarqube.sw.approved.txt index 2f11cf5bba511f0101718d09c582859a7be0b01e..72fdd780190ff3caa016fa9bc5cf172ea1566e03 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -2,6 +2,7 @@ <testExecutions version="1"loose text artifact > <file path="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp"> + <testCase name="#1905 -- test spec parser properly clears internal state between compound tests" duration="{duration}"/> <testCase name="Parse test names and tags/Empty test spec should have no filters" duration="{duration}"/> <testCase name="Parse test names and tags/Test spec from empty string should have no filters" duration="{duration}"/> <testCase name="Parse test names and tags/Test spec from just a comma should have no filters" duration="{duration}"/> diff --git a/packages/Catch2/projects/SelfTest/Baselines/xml.sw.approved.txt b/packages/Catch2/projects/SelfTest/Baselines/xml.sw.approved.txt index cc90e0cb29482f681c5bcde1d48a5299769f8143..807a946e68e1da07a5710f2bf2580145cbb82426 100644 --- a/packages/Catch2/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/packages/Catch2/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -196,6 +196,33 @@ Nor would this </Expression> <OverallResult success="true"/> </TestCase> + <TestCase name="#1905 -- test spec parser properly clears internal state between compound tests" tags="[command-line][test-spec]" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Original> + spec.matches(fakeTestCase("spec . char")) + </Original> + <Expanded> + true + </Expanded> + </Expression> + <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Original> + spec.matches(fakeTestCase("spec , char")) + </Original> + <Expanded> + true + </Expanded> + </Expression> + <Expression success="true" type="REQUIRE_FALSE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Original> + !(spec.matches(fakeTestCase(R"(spec \, char)"))) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <OverallResult success="true"/> + </TestCase> <TestCase name="#748 - captures with unexpected exceptions" tags="[!hide][!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" > <Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" > <Info> @@ -13630,7 +13657,15 @@ There is no extra whitespace here { 1, 2, 3 } Contains: 2 </Expanded> </Expression> - <OverallResults successes="2" failures="0" expectedFailures="0"/> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, (VectorContains<int, CustomAllocator<int>>(2)) + </Original> + <Expanded> + { 1, 2, 3 } Contains: 2 + </Expanded> + </Expression> + <OverallResults successes="3" failures="0" expectedFailures="0"/> </Section> <Section name="Contains (vector)" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > @@ -13641,6 +13676,14 @@ There is no extra whitespace here { 1, 2, 3 } Contains: { 1, 2 } </Expanded> </Expression> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2)) + </Original> + <Expanded> + { 1, 2, 3 } Contains: { 1, 2 } + </Expanded> + </Expression> <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > <Original> v, Contains(v2) @@ -13665,7 +13708,23 @@ There is no extra whitespace here { } Contains: { } </Expanded> </Expression> - <OverallResults successes="4" failures="0" expectedFailures="0"/> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2)) + </Original> + <Expanded> + { 1, 2, 3 } Contains: { 1, 2, 3 } + </Expanded> + </Expression> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, Contains(v6) + </Original> + <Expanded> + { 1, 2, 3 } Contains: { 1, 2 } + </Expanded> + </Expression> + <OverallResults successes="7" failures="0" expectedFailures="0"/> </Section> <Section name="Contains (element), composed" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > @@ -13703,7 +13762,23 @@ There is no extra whitespace here { 1, 2, 3 } Equals: { 1, 2, 3 } </Expanded> </Expression> - <OverallResults successes="3" failures="0" expectedFailures="0"/> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, (Equals<int, std::allocator<int>, CustomAllocator<int>>(v2)) + </Original> + <Expanded> + { 1, 2, 3 } Equals: { 1, 2, 3 } + </Expanded> + </Expression> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, Equals(v6) + </Original> + <Expanded> + { 1, 2, 3 } Equals: { 1, 2, 3 } + </Expanded> + </Expression> + <OverallResults successes="5" failures="0" expectedFailures="0"/> </Section> <Section name="UnorderedEquals" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > @@ -13738,7 +13813,23 @@ There is no extra whitespace here { 2, 3, 1 } UnorderedEquals: { 1, 2, 3 } </Expanded> </Expression> - <OverallResults successes="4" failures="0" expectedFailures="0"/> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5, (UnorderedEquals<int, std::allocator<int>, CustomAllocator<int>>(permuted)) + </Original> + <Expanded> + { 1, 2, 3 } UnorderedEquals: { 2, 3, 1 } + </Expanded> + </Expression> + <Expression success="true" type="CHECK_THAT" filename="projects/<exe-name>/UsageTests/Matchers.tests.cpp" > + <Original> + v5_permuted, UnorderedEquals(v5) + </Original> + <Expanded> + { 1, 3, 2 } UnorderedEquals: { 1, 2, 3 } + </Expanded> + </Expression> + <OverallResults successes="6" failures="0" expectedFailures="0"/> </Section> <OverallResult success="true"/> </TestCase> @@ -16048,7 +16139,7 @@ loose text artifact </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="1525" failures="149" expectedFailures="21"/> + <OverallResults successes="1536" failures="149" expectedFailures="21"/> </Group> - <OverallResults successes="1525" failures="148" expectedFailures="21"/> + <OverallResults successes="1536" failures="148" expectedFailures="21"/> </Catch> diff --git a/packages/Catch2/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/packages/Catch2/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index 6e590326fea00164e913ee310dbda020b05a40c0..0d8232e0a11c507a349d6f005e9e46bfca16ec3e 100644 --- a/packages/Catch2/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/packages/Catch2/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -292,6 +292,17 @@ TEST_CASE( "Parse test names and tags", "[command-line][test-spec]" ) { } } +TEST_CASE("#1905 -- test spec parser properly clears internal state between compound tests", "[command-line][test-spec]") { + using Catch::parseTestSpec; + using Catch::TestSpec; + // We ask for one of 2 different tests and the latter one of them has a , in name that needs escaping + TestSpec spec = parseTestSpec(R"("spec . char","spec \, char")"); + + REQUIRE(spec.matches(fakeTestCase("spec . char"))); + REQUIRE(spec.matches(fakeTestCase("spec , char"))); + REQUIRE_FALSE(spec.matches(fakeTestCase(R"(spec \, char)"))); +} + TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) { #ifndef CATCH_CONFIG_DISABLE_MATCHERS diff --git a/packages/Catch2/projects/SelfTest/UsageTests/Matchers.tests.cpp b/packages/Catch2/projects/SelfTest/UsageTests/Matchers.tests.cpp index 52f2912d7461e403d11e2040142b8f4350f727ed..e7a1d5424f3956bd84604ff8470fae610ab445fe 100644 --- a/packages/Catch2/projects/SelfTest/UsageTests/Matchers.tests.cpp +++ b/packages/Catch2/projects/SelfTest/UsageTests/Matchers.tests.cpp @@ -214,6 +214,42 @@ namespace { namespace MatchersTests { CHECK_THAT(testStringForMatching(), !Contains("substring")); } + template<typename T> + struct CustomAllocator : private std::allocator<T> + { + using size_type = size_t; + using difference_type = ptrdiff_t; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using value_type = T; + + template<typename U> + struct rebind + { using other = CustomAllocator<U>; }; + + using propagate_on_container_move_assignment = std::true_type; + using is_always_equal = std::true_type; + + CustomAllocator() = default; + + CustomAllocator(const CustomAllocator& other) + : std::allocator<T>(other) { } + + template<typename U> + CustomAllocator(const CustomAllocator<U>&) { } + + ~CustomAllocator() = default; + + using std::allocator<T>::address; + using std::allocator<T>::allocate; + using std::allocator<T>::construct; + using std::allocator<T>::deallocate; + using std::allocator<T>::max_size; + using std::allocator<T>::destroy; + }; + TEST_CASE("Vector matchers", "[matchers][vector]") { std::vector<int> v; v.push_back(1); @@ -234,19 +270,34 @@ namespace { namespace MatchersTests { v4.push_back(2 + 1e-8); v4.push_back(3 + 1e-8); + std::vector<int, CustomAllocator<int>> v5; + v5.push_back(1); + v5.push_back(2); + v5.push_back(3); + + std::vector<int, CustomAllocator<int>> v6; + v6.push_back(1); + v6.push_back(2); + std::vector<int> empty; SECTION("Contains (element)") { CHECK_THAT(v, VectorContains(1)); CHECK_THAT(v, VectorContains(2)); + CHECK_THAT(v5, (VectorContains<int, CustomAllocator<int>>(2))); } SECTION("Contains (vector)") { CHECK_THAT(v, Contains(v2)); + CHECK_THAT(v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2))); + v2.push_back(3); // now exactly matches CHECK_THAT(v, Contains(v2)); CHECK_THAT(v, Contains(empty)); CHECK_THAT(empty, Contains(empty)); + + CHECK_THAT(v5, (Contains<int, std::allocator<int>, CustomAllocator<int>>(v2))); + CHECK_THAT(v5, Contains(v6)); } SECTION("Contains (element), composed") { CHECK_THAT(v, VectorContains(1) && VectorContains(2)); @@ -262,6 +313,11 @@ namespace { namespace MatchersTests { // Different vector with same elements v2.push_back(3); CHECK_THAT(v, Equals(v2)); + + CHECK_THAT(v5, (Equals<int, std::allocator<int>, CustomAllocator<int>>(v2))); + + v6.push_back(3); + CHECK_THAT(v5, Equals(v6)); } SECTION("UnorderedEquals") { CHECK_THAT(v, UnorderedEquals(v)); @@ -273,6 +329,12 @@ namespace { namespace MatchersTests { std::reverse(begin(permuted), end(permuted)); REQUIRE_THAT(permuted, UnorderedEquals(v)); + + CHECK_THAT(v5, (UnorderedEquals<int, std::allocator<int>, CustomAllocator<int>>(permuted))); + + auto v5_permuted = v5; + std::next_permutation(begin(v5_permuted), end(v5_permuted)); + CHECK_THAT(v5_permuted, UnorderedEquals(v5)); } } diff --git a/packages/Catch2/projects/TestScripts/testRandomOrder.py b/packages/Catch2/projects/TestScripts/testRandomOrder.py new file mode 100755 index 0000000000000000000000000000000000000000..a6ffb996d778be666ad3e8a1dfec3a1b0d459e00 --- /dev/null +++ b/packages/Catch2/projects/TestScripts/testRandomOrder.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +""" +This test script verifies that the random ordering of tests inside +Catch2 is invariant in regards to subsetting. This is done by running +the binary 3 times, once with all tests selected, and twice with smaller +subsets of tests selected, and verifying that the selected tests are in +the same relative order. +""" + +import subprocess +import sys +import random + +def list_tests(self_test_exe, tags, rng_seed): + cmd = [self_test_exe, '--list-test-names-only', '--order', 'rand', + '--rng-seed', str(rng_seed)] + tags_arg = ','.join('[{}]'.format(t) for t in tags) + if tags_arg: + cmd.append(tags_arg + '~[.]') + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if stderr: + raise RuntimeError("Unexpected error output:\n" + process.stderr) + result = stdout.split(b'\n') + result = [s for s in result if s] + if len(result) < 2: + raise RuntimeError("Unexpectedly few tests listed (got {})".format( + len(result))) + return result + +def check_is_sublist_of(shorter, longer): + assert len(shorter) < len(longer) + assert len(set(longer)) == len(longer) + + indexes_in_longer = {s: i for i, s in enumerate(longer)} + for s1, s2 in zip(shorter, shorter[1:]): + assert indexes_in_longer[s1] < indexes_in_longer[s2], ( + '{} comes before {} in longer list.\n' + 'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter)) + +def main(): + self_test_exe, = sys.argv[1:] + + # We want a random seed for the test, but want to avoid 0, + # because it has special meaning + seed = random.randint(1, 2 ** 32 - 1) + + list_one_tag = list_tests(self_test_exe, ['generators'], seed) + list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed) + list_all = list_tests(self_test_exe, [], seed) + + # First, verify that restricting to a subset yields the same order + check_is_sublist_of(list_two_tags, list_all) + check_is_sublist_of(list_one_tag, list_two_tags) + +if __name__ == '__main__': + sys.exit(main())