From d041ada92ff866fac67b8756b962bb1137166694 Mon Sep 17 00:00:00 2001 From: Ethan Slattery Date: Sat, 27 Apr 2024 19:42:09 -0700 Subject: [PATCH 01/20] add section event for reporters Catch2 has a section enter and exit event. This allows for finer resolution display of test failures in IDEs and CI dashboards without giving up the benefits of using sections. --- include/snitch/snitch_section.hpp | 11 +++++- include/snitch/snitch_test_data.hpp | 23 ++++++++++++ src/snitch_reporter_catch2_xml.cpp | 54 ++++++++++++++++++++--------- src/snitch_reporter_console.cpp | 2 ++ src/snitch_reporter_teamcity.cpp | 2 ++ src/snitch_section.cpp | 29 ++++++++++++++++ tests/runtime_tests/check.cpp | 2 +- tests/testing_event.cpp | 21 +++++++++++ tests/testing_event.hpp | 19 ++++++++++ 9 files changed, 144 insertions(+), 19 deletions(-) diff --git a/include/snitch/snitch_section.hpp b/include/snitch/snitch_section.hpp index 18934168..2a0365a9 100644 --- a/include/snitch/snitch_section.hpp +++ b/include/snitch/snitch_section.hpp @@ -3,12 +3,21 @@ #include "snitch/snitch_config.hpp" #include "snitch/snitch_test_data.hpp" +#if SNITCH_WITH_TIMINGS +# include +#endif namespace snitch::impl { struct section_entry_checker { section data = {}; test_state& state; - bool entered = false; + bool entered = false; + std::size_t asserts = 0; + std::size_t failures = 0; + std::size_t allowed_failures = 0; +#if SNITCH_WITH_TIMINGS + std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now(); +#endif SNITCH_EXPORT ~section_entry_checker(); diff --git a/include/snitch/snitch_test_data.hpp b/include/snitch/snitch_test_data.hpp index 7d1d81a7..3b3be62d 100644 --- a/include/snitch/snitch_test_data.hpp +++ b/include/snitch/snitch_test_data.hpp @@ -177,6 +177,27 @@ struct test_case_ended { bool failure_allowed = false; }; +struct section_started { + /// Identifiers (name, description) + section_id id = {}; + /// Location (file, line) + source_location location = {}; +}; + +struct section_ended { + /// Identifiers (name, description) + section_id id = {}; + /// Location (file, line) + source_location location = {}; + bool skipped = false; + std::size_t assertion_count = 0; + std::size_t assertion_failure_count = 0; + std::size_t allowed_assertion_failure_count = 0; +#if SNITCH_WITH_TIMINGS + float duration = 0.0f; +#endif +}; + struct assertion_failed { const test_id& id; section_info sections = {}; @@ -232,6 +253,8 @@ using data = std::variant< test_run_ended, test_case_started, test_case_ended, + section_started, + section_ended, assertion_failed, assertion_succeeded, test_case_skipped, diff --git a/src/snitch_reporter_catch2_xml.cpp b/src/snitch_reporter_catch2_xml.cpp index f7127527..b0ec32a3 100644 --- a/src/snitch_reporter_catch2_xml.cpp +++ b/src/snitch_reporter_catch2_xml.cpp @@ -119,14 +119,6 @@ void open_close( template void report_assertion(reporter& rep, const registry& r, const T& e, bool success) noexcept { - for (const auto& s : e.sections) { - open( - rep, r, "Section", - {{"name", make_escaped(s.id.name)}, - {"filename", make_escaped(s.location.file)}, - {"line", make_string(s.location.line)}}); - } - for (const auto& c : e.captures) { open(rep, r, "Info"); print(rep, r, make_escaped(c)); @@ -166,10 +158,6 @@ void report_assertion(reporter& rep, const registry& r, const T& e, bool success close(rep, r, "Expression"); }}, e.data); - - for (const auto& s [[maybe_unused]] : e.sections) { - close(rep, r, "Section"); - } } } // namespace @@ -205,13 +193,15 @@ void reporter::report(const registry& r, const snitch::event::data& event) noexc e.assertion_count - e.assertion_failure_count - e.allowed_assertion_failure_count)}, {"failures", make_string(e.assertion_failure_count)}, - {"expectedFailures", make_string(e.allowed_assertion_failure_count)}}); + {"expectedFailures", make_string(e.allowed_assertion_failure_count)}, + {"skips", make_string(e.skip_count)}}); node( *this, r, "OverallResultsCases", {{"successes", make_string(e.run_count - e.fail_count - e.allowed_fail_count)}, {"failures", make_string(e.fail_count)}, - {"expectedFailures", make_string(e.allowed_fail_count)}}); + {"expectedFailures", make_string(e.allowed_fail_count)}, + {"skips", make_string(e.skip_count)}}); close(*this, r, "Catch2TestRun"); }, @@ -228,16 +218,46 @@ void reporter::report(const registry& r, const snitch::event::data& event) noexc node( *this, r, "OverallResult", {{"success", e.state == test_case_state::failed ? "false" : "true"}, + {"skips", e.state == test_case_state::skipped ? "1" : "0"}, {"durationInSeconds", make_string(e.duration)}}); # else node( *this, r, "OverallResult", - {{"success", e.state == test_case_state::failed ? "false" : "true"}}); + {{"success", e.state == test_case_state::failed ? "false" : "true"}, + {"skips", e.state == test_case_state::skipped ? "1" : "0"}}); # endif close(*this, r, "TestCase"); }, - [&](const snitch::event::test_case_skipped&) { - // Nothing to do; this gets reported as "success". + [&](const snitch::event::section_started& e) { + open( + *this, r, "Section", + {{"name", make_escaped(e.id.name)}, + {"filename", make_escaped(e.location.file)}, + {"line", make_string(e.location.line)}}); + }, + [&](const snitch::event::section_ended& e) { + node( + *this, r, "OverallResults", + {{"successes", make_string( + e.assertion_count - e.assertion_failure_count - + e.allowed_assertion_failure_count)}, + {"failures", make_string(e.assertion_failure_count)}, + {"expectedFailures", make_string(e.allowed_assertion_failure_count)}, + {"skipped", e.skipped?"true":"false"} +# if SNITCH_WITH_TIMINGS + , + {"durationInSeconds", make_string(e.duration)} +# endif + }); + close(*this, r, "Section"); + }, + [&](const snitch::event::test_case_skipped& e) { + open( + *this, r, "Skip", + {{"filename", make_escaped(e.location.file)}, + {"line", make_string(e.location.line)}}); + print(*this, r, e.message); + close(*this, r, "Skip"); }, [&](const snitch::event::assertion_failed& e) { report_assertion(*this, r, e, false); }, [&](const snitch::event::assertion_succeeded& e) { diff --git a/src/snitch_reporter_console.cpp b/src/snitch_reporter_console.cpp index b4ea916e..4ef2d8f6 100644 --- a/src/snitch_reporter_console.cpp +++ b/src/snitch_reporter_console.cpp @@ -162,6 +162,8 @@ void reporter::report(const registry& r, const event::data& event) noexcept { make_colored(full_name, r.with_color, color::highlight1), "\n"); #endif }, + [&](const snitch::event::section_started&) {}, + [&](const snitch::event::section_ended&) {}, [&](const snitch::event::test_case_skipped& e) { r.print(make_colored("skipped: ", r.with_color, color::skipped)); print_location(r, e.id, e.sections, e.captures, e.location); diff --git a/src/snitch_reporter_teamcity.cpp b/src/snitch_reporter_teamcity.cpp index e49e8b70..2e28b784 100644 --- a/src/snitch_reporter_teamcity.cpp +++ b/src/snitch_reporter_teamcity.cpp @@ -148,6 +148,8 @@ void report(const registry& r, const snitch::event::data& event) noexcept { send_message(r, "testFinished", {{"name", make_full_name(e.id)}}); # endif }, + [&](const snitch::event::section_started&) {}, + [&](const snitch::event::section_ended&) {}, [&](const snitch::event::test_case_skipped& e) { send_message( r, "testIgnored", diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index bc56461d..bce8452b 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -2,6 +2,7 @@ #include "snitch/snitch_console.hpp" #include "snitch/snitch_registry.hpp" +#include "snitch/snitch_test_data.hpp" #if SNITCH_WITH_EXCEPTIONS # include @@ -23,6 +24,10 @@ section_entry_checker::~section_entry_checker() { pop_location(state); + asserts = state.asserts - asserts; + failures = state.failures - failures; + allowed_failures = state.allowed_failures - allowed_failures; + if (sections.depth == sections.levels.size()) { // We just entered this section, and there was no child section in it. // This is a leaf; flag that a leaf has been executed so that no other leaf @@ -31,6 +36,18 @@ section_entry_checker::~section_entry_checker() { // that we don't know about yet. Popping will be done when we exit from the parent, // since then we will know if there is any sibling. sections.leaf_executed = true; +#if SNITCH_WITH_TIMINGS + const auto end_time = std::chrono::steady_clock::now(); + const float duration = std::chrono::duration(end_time - start_time).count(); + state.reg.report_callback( + state.reg, + event::section_ended{ + data.id, data.location, false, asserts, failures, allowed_failures, duration}); +#else + state.reg.report_callback( + state.reg, + event::section_ended{data.id, data.location, asserts, failures, allowed_failures}); +#endif } else { // Check if there is any child section left to execute, at any depth below this one. bool no_child_section_left = true; @@ -44,6 +61,10 @@ section_entry_checker::~section_entry_checker() { if (no_child_section_left) { // No more children, we can pop this level and never go back. + state.reg.report_callback( + state.reg, event::section_ended{ + sections.current_section.back().id, + sections.current_section.back().location}); sections.levels.pop_back(); } } @@ -76,6 +97,9 @@ section_entry_checker::operator bool() { } ++sections.depth; + asserts = state.asserts; + failures = state.failures; + allowed_failures = state.allowed_failures; auto& level = sections.levels[sections.depth - 1]; @@ -97,6 +121,11 @@ section_entry_checker::operator bool() { (level.current_section_id == level.previous_section_id && sections.depth < sections.levels.size())) { + // First time entering this section, emit the section start event. + if (level.current_section_id == level.previous_section_id + 1) { + state.reg.report_callback(state.reg, event::section_started{data.id, data.location}); + } + level.previous_section_id = level.current_section_id; sections.current_section.push_back(data); push_location( diff --git a/tests/runtime_tests/check.cpp b/tests/runtime_tests/check.cpp index 49504afb..aeb00612 100644 --- a/tests/runtime_tests/check.cpp +++ b/tests/runtime_tests/check.cpp @@ -1765,7 +1765,7 @@ SNITCH_WARNING_PUSH SNITCH_WARNING_DISABLE_UNREACHABLE TEST_CASE("unhandled exceptions", "[test macros]") { - event_catcher<3> catcher; + event_catcher<7> catcher; test_check_line = 0u; test_section_line = 0u; diff --git a/tests/testing_event.cpp b/tests/testing_event.cpp index 9e8523e9..55498e48 100644 --- a/tests/testing_event.cpp +++ b/tests/testing_event.cpp @@ -141,6 +141,27 @@ owning_event::data deep_copy(snitch::small_string_span pool, const snitch::event c.failure_allowed = s.failure_allowed; return c; }, + [&](const snitch::event::section_started& s) -> owning_event::data { + owning_event::section_started c; + copy_location(pool, c, s); + c.id.name = append_to_pool(pool, s.id.name); + c.id.description = append_to_pool(pool, s.id.description); + return c; + }, + [&](const snitch::event::section_ended& s) -> owning_event::data { + owning_event::section_ended c; + copy_location(pool, c, s); + c.id.name = append_to_pool(pool, s.id.name); + c.id.description = append_to_pool(pool, s.id.description); + c.skipped = s.skipped; + c.assertion_count = s.assertion_count; + c.assertion_failure_count = s.assertion_failure_count; + c.allowed_assertion_failure_count = s.allowed_assertion_failure_count; +#if SNITCH_WITH_TIMINGS + c.duration = s.duration; +#endif + return c; + }, [&](const snitch::event::test_run_started& s) -> owning_event::data { owning_event::test_run_started c; copy_test_run_id(pool, c, s); diff --git a/tests/testing_event.hpp b/tests/testing_event.hpp index 18a605e5..4012803f 100644 --- a/tests/testing_event.hpp +++ b/tests/testing_event.hpp @@ -53,6 +53,23 @@ struct test_case_ended { bool failure_allowed = false; }; +struct section_started { + snitch::section_id id = {}; + snitch::source_location location = {}; +}; + +struct section_ended { + snitch::section_id id = {}; + snitch::source_location location = {}; + bool skipped = false; + std::size_t assertion_count = 0; + std::size_t assertion_failure_count = 0; + std::size_t allowed_assertion_failure_count = 0; +#if SNITCH_WITH_TIMINGS + float duration = 0.0f; +#endif +}; + struct assertion_failed { snitch::test_id id = {}; section_info sections = {}; @@ -99,6 +116,8 @@ using data = std::variant< owning_event::test_run_ended, owning_event::test_case_started, owning_event::test_case_ended, + owning_event::section_started, + owning_event::section_ended, owning_event::assertion_failed, owning_event::assertion_succeeded, owning_event::test_case_skipped, From 2687215b6d2b324c4bcb200266217757950ac449 Mon Sep 17 00:00:00 2001 From: Ethan Slattery Date: Sun, 12 May 2024 13:52:45 -0700 Subject: [PATCH 02/20] update approval tests --- .../data/expected/reporter_catch2_xml_allfail | 57 ++++---- .../data/expected/reporter_catch2_xml_allpass | 38 ++--- .../data/expected/reporter_catch2_xml_default | 138 +++++++++--------- 3 files changed, 118 insertions(+), 115 deletions(-) diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail index da54d4a0..e1ef6d96 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail @@ -10,13 +10,13 @@ 1 != 2 - + expected test to fail - + @@ -27,7 +27,7 @@ 1 != 2 - + @@ -38,7 +38,7 @@ 1 != 2 - + @@ -49,7 +49,7 @@ 1 != 2 - + @@ -60,7 +60,7 @@ 1 != 2 - + @@ -71,7 +71,7 @@ 1 != 2 - + @@ -82,13 +82,13 @@ 1 != 2 - + something bad - + @@ -99,7 +99,7 @@ 1 != 2 - + @@ -110,7 +110,7 @@ 1 == 1 - + @@ -121,19 +121,19 @@ super_long_string != super_long_string - + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... - + expected throw_something(true) not to throw but it threw a std::exception; message: I threw - + @@ -145,38 +145,41 @@ could not match caught std::runtime_error with expected content: could not find 'I throws' in 'I threw' - + unexpected std::exception caught; message: unexpected error - +
- - unexpected std::exception caught; message: unexpected error - +
+
- + + unexpected std::exception caught; message: unexpected error + + unexpected std::exception caught; message: unexpected error - +
- - unexpected std::exception caught; message: unexpected error - +
- + + unexpected std::exception caught; message: unexpected error + + - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_allpass b/tests/approval_tests/data/expected/reporter_catch2_xml_allpass index 381edbb7..8c9c19fe 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_allpass +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_allpass @@ -2,10 +2,10 @@ - + - + @@ -16,7 +16,7 @@ 1 != 2 - + @@ -27,50 +27,50 @@ 1 != 2 - + - + - + - + - + - + - + - + - + - + - + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... - + - + - + - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index bbd0395b..b56d1b7f 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -2,7 +2,7 @@ - + @@ -13,10 +13,10 @@ 1 != 2 - + - + @@ -27,13 +27,13 @@ 1 != 2 - + expected test to fail - + @@ -44,10 +44,10 @@ 1 != 2 - + - + @@ -58,13 +58,13 @@ 1 != 2 - + - + - + @@ -75,7 +75,7 @@ 1 != 2 - + @@ -86,13 +86,13 @@ 1 != 2 - + - + - + @@ -103,7 +103,7 @@ 1 != 2 - + @@ -114,10 +114,10 @@ 1 != 2 - + - + @@ -128,19 +128,19 @@ 1 != 2 - + - + something bad - + - + @@ -151,10 +151,10 @@ 1 != 2 - + - + @@ -165,10 +165,10 @@ 1 == 1 - + - + @@ -179,31 +179,31 @@ super_long_string != super_long_string - + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... - + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... - + - + expected throw_something(true) not to throw but it threw a std::exception; message: I threw - + - + @@ -215,40 +215,46 @@ could not match caught std::runtime_error with expected content: could not find 'I throws' in 'I threw' - + unexpected std::exception caught; message: unexpected error - +
- - unexpected std::exception caught; message: unexpected error - +
+
- + + unexpected std::exception caught; message: unexpected error + + unexpected std::exception caught; message: unexpected error - +
- - unexpected std::exception caught; message: unexpected error - +
- + + unexpected std::exception caught; message: unexpected error + + - + + not interesting + + @@ -257,7 +263,7 @@ failure - + @@ -284,21 +290,23 @@ failure 4 - +
failure +
- +
failure 1 +
failure 7 @@ -307,49 +315,39 @@ failure 2 - -
failure 3 +
-
- - failure 7 - -
+ + failure 7 + failure 2 -
-
failure 4 -
-
-
-
failure 5 +
-
-
-
-
failure 6 +
+
failure 7 - +
@@ -362,6 +360,7 @@ failure 1 +
info 1 @@ -379,6 +378,7 @@ failure 2 + info 1 @@ -386,20 +386,20 @@ failure 3 - + escape <>&"' in messages - + &&&&&&&&&&&&&&&&&&&&&&&&&... - + - - + + From 080558c3ddbc14dc2e52905a6aade434c97a8efa Mon Sep 17 00:00:00 2001 From: Ethan Slattery Date: Tue, 21 May 2024 19:46:33 -0700 Subject: [PATCH 03/20] catch2 XML is at version 3 --- src/snitch_reporter_catch2_xml.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snitch_reporter_catch2_xml.cpp b/src/snitch_reporter_catch2_xml.cpp index b0ec32a3..ba680cb7 100644 --- a/src/snitch_reporter_catch2_xml.cpp +++ b/src/snitch_reporter_catch2_xml.cpp @@ -182,7 +182,7 @@ void reporter::report(const registry& r, const snitch::event::data& event) noexc *this, r, "Catch2TestRun", {{"name", make_escaped(e.name)}, {"rng-seed", "0"}, - {"xml-format-version", "2"}, + {"xml-format-version", "3"}, {"catch2-version", SNITCH_FULL_VERSION ".snitch"}, {"filters", make_filters(e.filters)}}); }, From 5f80878884665f7489bb1d8a14006981addbb94b Mon Sep 17 00:00:00 2001 From: Ethan Slattery Date: Tue, 2 Jul 2024 11:10:42 -0700 Subject: [PATCH 04/20] Don't use chrono in snitch headers As suggested in code review, don't include chrono in the snitch headers since this increases compile times. Use a basic integral representation type and then convert back and forth to chrono types in the source files when actually reading clocks or calculating times. --- include/snitch/snitch_section.hpp | 7 +++---- src/snitch_section.cpp | 22 +++++++++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/include/snitch/snitch_section.hpp b/include/snitch/snitch_section.hpp index 2a0365a9..2d570797 100644 --- a/include/snitch/snitch_section.hpp +++ b/include/snitch/snitch_section.hpp @@ -3,9 +3,8 @@ #include "snitch/snitch_config.hpp" #include "snitch/snitch_test_data.hpp" -#if SNITCH_WITH_TIMINGS -# include -#endif + +#include namespace snitch::impl { struct section_entry_checker { @@ -16,7 +15,7 @@ struct section_entry_checker { std::size_t failures = 0; std::size_t allowed_failures = 0; #if SNITCH_WITH_TIMINGS - std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now(); + std::make_signed_t start_time = 0; #endif SNITCH_EXPORT ~section_entry_checker(); diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index bce8452b..e0f42fe3 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -7,8 +7,16 @@ #if SNITCH_WITH_EXCEPTIONS # include #endif +#if SNITCH_WITH_TIMINGS +# include +#endif namespace snitch::impl { +#if SNITCH_WITH_TIMINGS +using fsec = std::chrono::duration; +using snitch_clock = std::chrono::steady_clock; +#endif + section_entry_checker::~section_entry_checker() { auto& sections = state.info.sections; @@ -37,12 +45,13 @@ section_entry_checker::~section_entry_checker() { // since then we will know if there is any sibling. sections.leaf_executed = true; #if SNITCH_WITH_TIMINGS - const auto end_time = std::chrono::steady_clock::now(); - const float duration = std::chrono::duration(end_time - start_time).count(); + const auto end_time = snitch_clock::now().time_since_epoch(); + const auto duration = + std::chrono::duration_cast(end_time - snitch_clock::duration{start_time}); state.reg.report_callback( - state.reg, - event::section_ended{ - data.id, data.location, false, asserts, failures, allowed_failures, duration}); + state.reg, event::section_ended{ + data.id, data.location, false, asserts, failures, allowed_failures, + duration.count()}); #else state.reg.report_callback( state.reg, @@ -96,6 +105,9 @@ section_entry_checker::operator bool() { sections.levels.push_back({}); } +#if SNITCH_WITH_TIMINGS + start_time = snitch_clock::now().time_since_epoch().count(); +#endif ++sections.depth; asserts = state.asserts; failures = state.failures; From 6156453b8afb2ac4fa4fbb9d5fb6facfdf15a69a Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 16:06:17 +0100 Subject: [PATCH 05/20] Fix duplicated test case name for xml approval tests --- tests/approval_tests/reporter_catch2_xml.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/approval_tests/reporter_catch2_xml.cpp b/tests/approval_tests/reporter_catch2_xml.cpp index 3f6e063b..be504b77 100644 --- a/tests/approval_tests/reporter_catch2_xml.cpp +++ b/tests/approval_tests/reporter_catch2_xml.cpp @@ -8,7 +8,7 @@ using namespace std::literals; using snitch::matchers::contains_substring; -TEST_CASE("teamcity reporter", "[reporters]") { +TEST_CASE("xml reporter", "[reporters]") { mock_framework framework; register_tests_for_reporters(framework.registry); framework.registry.add({"test escape <>&\"'"}, SNITCH_CURRENT_LOCATION, [] { From 8421d8f1590b7caf71cd1d6a2ea99c8c81c8d97b Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 16:09:35 +0100 Subject: [PATCH 06/20] Bump XML format --- .../data/expected/reporter_catch2_xml_allfail | 2 +- .../data/expected/reporter_catch2_xml_allpass | 2 +- .../data/expected/reporter_catch2_xml_default | 2 +- tests/approval_tests/data/expected/reporter_catch2_xml_full | 2 +- .../approval_tests/data/expected/reporter_catch2_xml_notest | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail index e1ef6d96..b3e53874 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail @@ -1,6 +1,6 @@ - + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_allpass b/tests/approval_tests/data/expected/reporter_catch2_xml_allpass index 8c9c19fe..4cc139b2 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_allpass +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_allpass @@ -1,6 +1,6 @@ - + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index b56d1b7f..5a30d939 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -1,6 +1,6 @@ - + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index 68e61936..d0c8ffb5 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -1,6 +1,6 @@ - + no exception caught diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_notest b/tests/approval_tests/data/expected/reporter_catch2_xml_notest index cf99cca3..2d7e91d9 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_notest +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_notest @@ -1,6 +1,6 @@ - - - + + + From 9e26365dc41bc65fa1740109ae19c233bbc20315 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 16:44:08 +0100 Subject: [PATCH 07/20] Re-order section entering logic for early exit --- src/snitch_section.cpp | 52 +++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index e0f42fe3..aae31b53 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -105,13 +105,7 @@ section_entry_checker::operator bool() { sections.levels.push_back({}); } -#if SNITCH_WITH_TIMINGS - start_time = snitch_clock::now().time_since_epoch().count(); -#endif ++sections.depth; - asserts = state.asserts; - failures = state.failures; - allowed_failures = state.allowed_failures; auto& level = sections.levels[sections.depth - 1]; @@ -126,26 +120,38 @@ section_entry_checker::operator bool() { return false; } - // Only enter this section if: - // - The section entered in the previous run was its immediate previous sibling, or - // - This section was already entered in the previous run, and child sections exist in it. - if (level.current_section_id == level.previous_section_id + 1 || - (level.current_section_id == level.previous_section_id && - sections.depth < sections.levels.size())) { + const bool previous_was_preceeding_sibling = + level.current_section_id == level.previous_section_id + 1; + const bool children_remaining_in_self = level.current_section_id == level.previous_section_id && + sections.depth < sections.levels.size(); - // First time entering this section, emit the section start event. - if (level.current_section_id == level.previous_section_id + 1) { - state.reg.report_callback(state.reg, event::section_started{data.id, data.location}); - } + if (!previous_was_preceeding_sibling && !children_remaining_in_self) { + // Skip this section if: + // - The section entered in the previous run was not its immediate previous sibling, and + // - This section was not already entered in the previous run with remaining children. + return false; + } - level.previous_section_id = level.current_section_id; - sections.current_section.push_back(data); - push_location( - state, {data.location.file, data.location.line, location_type::section_scope}); - entered = true; - return true; + // Entering this section. + + // Emit the section start event (only on first entry). + if (level.current_section_id == level.previous_section_id + 1) { + state.reg.report_callback(state.reg, event::section_started{data.id, data.location}); } - return false; + // Keep records of current state. +#if SNITCH_WITH_TIMINGS + start_time = snitch_clock::now().time_since_epoch().count(); +#endif + asserts = state.asserts; + failures = state.failures; + allowed_failures = state.allowed_failures; + + // Push new section on the stack. + level.previous_section_id = level.current_section_id; + sections.current_section.push_back(data); + push_location(state, {data.location.file, data.location.line, location_type::section_scope}); + entered = true; + return true; } } // namespace snitch::impl From e6ab5b1bec0baa537514f41685b66f77d02afb98 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 17:21:38 +0100 Subject: [PATCH 08/20] Cumulative assert count in section, managed by registry --- include/snitch/snitch_section.hpp | 6 +- include/snitch/snitch_test_data.hpp | 25 +++- src/snitch_registry.cpp | 21 ++- src/snitch_section.cpp | 51 ++++--- .../data/expected/reporter_catch2_xml_allfail | 6 +- .../data/expected/reporter_catch2_xml_default | 10 +- .../data/expected/reporter_catch2_xml_full | 138 +++++++++--------- 7 files changed, 143 insertions(+), 114 deletions(-) diff --git a/include/snitch/snitch_section.hpp b/include/snitch/snitch_section.hpp index 2d570797..73239f51 100644 --- a/include/snitch/snitch_section.hpp +++ b/include/snitch/snitch_section.hpp @@ -10,10 +10,8 @@ namespace snitch::impl { struct section_entry_checker { section data = {}; test_state& state; - bool entered = false; - std::size_t asserts = 0; - std::size_t failures = 0; - std::size_t allowed_failures = 0; + bool entered = false; + #if SNITCH_WITH_TIMINGS std::make_signed_t start_time = 0; #endif diff --git a/include/snitch/snitch_test_data.hpp b/include/snitch/snitch_test_data.hpp index 3b3be62d..e9b58abf 100644 --- a/include/snitch/snitch_test_data.hpp +++ b/include/snitch/snitch_test_data.hpp @@ -46,6 +46,13 @@ struct section { section_id id = {}; /// Location (file, line) source_location location = {}; + + /// Counts all assertions; passed, failed, or allowed failed + std::size_t assertion_count = 0; + /// Counts failed assertions + std::size_t assertion_failure_count = 0; + /// Counts allowed failed assertions (e.g., [!shouldfail] and [!mayfail]) + std::size_t allowed_assertion_failure_count = 0; }; /// List of test case filters @@ -188,12 +195,20 @@ struct section_ended { /// Identifiers (name, description) section_id id = {}; /// Location (file, line) - source_location location = {}; - bool skipped = false; - std::size_t assertion_count = 0; - std::size_t assertion_failure_count = 0; - std::size_t allowed_assertion_failure_count = 0; + source_location location = {}; + + /// Whether the section has been skipped. + bool skipped = false; + + /// Counts all assertions; passed, failed, or allowed failed + std::size_t assertion_count = 0; + /// Counts failed assertions + std::size_t assertion_failure_count = 0; + /// Counts allowed failed assertions (e.g., [!shouldfail] and [!mayfail]) + std::size_t allowed_assertion_failure_count = 0; + #if SNITCH_WITH_TIMINGS + /// Section duration, in seconds float duration = 0.0f; #endif }; diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index 698c25a6..bde0cf22 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -391,16 +391,33 @@ const char* registry::add_fixture( namespace { void register_assertion(bool success, impl::test_state& state) { - ++state.asserts; - if (!success) { if (state.may_fail || state.should_fail) { + ++state.asserts; ++state.allowed_failures; + + for (auto& section : state.info.sections.current_section) { + ++section.assertion_count; + ++section.allowed_assertion_failure_count; + } + impl::set_state(state.test, impl::test_case_state::allowed_fail); } else { + ++state.asserts; ++state.failures; + + for (auto& section : state.info.sections.current_section) { + ++section.assertion_count; + ++section.assertion_failure_count; + } + impl::set_state(state.test, impl::test_case_state::failed); } + } else { + ++state.asserts; + for (auto& section : state.info.sections.current_section) { + ++section.assertion_count; + } } } diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index aae31b53..86e6db5f 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -32,10 +32,7 @@ section_entry_checker::~section_entry_checker() { pop_location(state); - asserts = state.asserts - asserts; - failures = state.failures - failures; - allowed_failures = state.allowed_failures - allowed_failures; - + bool last_entry = false; if (sections.depth == sections.levels.size()) { // We just entered this section, and there was no child section in it. // This is a leaf; flag that a leaf has been executed so that no other leaf @@ -44,19 +41,7 @@ section_entry_checker::~section_entry_checker() { // that we don't know about yet. Popping will be done when we exit from the parent, // since then we will know if there is any sibling. sections.leaf_executed = true; -#if SNITCH_WITH_TIMINGS - const auto end_time = snitch_clock::now().time_since_epoch(); - const auto duration = - std::chrono::duration_cast(end_time - snitch_clock::duration{start_time}); - state.reg.report_callback( - state.reg, event::section_ended{ - data.id, data.location, false, asserts, failures, allowed_failures, - duration.count()}); -#else - state.reg.report_callback( - state.reg, - event::section_ended{data.id, data.location, asserts, failures, allowed_failures}); -#endif + last_entry = true; } else { // Check if there is any child section left to execute, at any depth below this one. bool no_child_section_left = true; @@ -70,14 +55,31 @@ section_entry_checker::~section_entry_checker() { if (no_child_section_left) { // No more children, we can pop this level and never go back. - state.reg.report_callback( - state.reg, event::section_ended{ - sections.current_section.back().id, - sections.current_section.back().location}); sections.levels.pop_back(); + last_entry = true; } } + // Emit the section end event (only on last entry). + if (last_entry) { + const auto& section = sections.current_section.back(); + +#if SNITCH_WITH_TIMINGS + const auto end_time = snitch_clock::now().time_since_epoch(); + const auto duration = + std::chrono::duration_cast(end_time - snitch_clock::duration{start_time}); +#endif + + state.reg.report_callback( + state.reg, event::section_ended { + data.id, data.location, false, section.assertion_count, + section.assertion_failure_count, section.allowed_assertion_failure_count, +#if SNITCH_WITH_TIMINGS + duration.count() +#endif + }); + } + sections.current_section.pop_back(); } @@ -135,17 +137,14 @@ section_entry_checker::operator bool() { // Entering this section. // Emit the section start event (only on first entry). - if (level.current_section_id == level.previous_section_id + 1) { + if (previous_was_preceeding_sibling) { state.reg.report_callback(state.reg, event::section_started{data.id, data.location}); } - // Keep records of current state. #if SNITCH_WITH_TIMINGS + // Start the clock. start_time = snitch_clock::now().time_since_epoch().count(); #endif - asserts = state.asserts; - failures = state.failures; - allowed_failures = state.allowed_failures; // Push new section on the stack. level.previous_section_id = level.current_section_id; diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail index b3e53874..f45ede15 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail @@ -156,9 +156,9 @@
- +
- +
unexpected std::exception caught; message: unexpected error @@ -173,7 +173,7 @@
- +
unexpected std::exception caught; message: unexpected error diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index 5a30d939..fd15a026 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -226,9 +226,9 @@
- +
- +
unexpected std::exception caught; message: unexpected error @@ -243,7 +243,7 @@
- +
unexpected std::exception caught; message: unexpected error @@ -340,9 +340,9 @@ failure 6 - + - + failure 7 diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index d0c8ffb5..4220a079 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -5,7 +5,7 @@ no exception caught - + @@ -19,13 +19,13 @@ no exception caught - + no exception caught - + @@ -39,7 +39,7 @@ no exception caught - + @@ -48,7 +48,7 @@ expected test to fail - + @@ -65,13 +65,13 @@ expected test to fail - + no exception caught - + @@ -85,19 +85,19 @@ no exception caught - + no exception caught - + no exception caught - + @@ -111,7 +111,7 @@ no exception caught - + @@ -125,19 +125,19 @@ no exception caught - + no exception caught - + no exception caught - + @@ -151,7 +151,7 @@ no exception caught - + @@ -165,13 +165,13 @@ no exception caught - + no exception caught - + @@ -185,7 +185,7 @@ no exception caught - + @@ -194,7 +194,7 @@ no exception caught - + @@ -203,7 +203,7 @@ no exception caught - + @@ -217,7 +217,7 @@ no exception caught - + @@ -231,7 +231,7 @@ no exception caught - + @@ -245,7 +245,7 @@ no exception caught - + @@ -259,7 +259,7 @@ no exception caught - + @@ -273,7 +273,7 @@ no exception caught - + @@ -287,19 +287,19 @@ no exception caught - + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... - + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... - + @@ -308,7 +308,7 @@ no exception caught - + @@ -317,7 +317,7 @@ no exception caught - + @@ -326,7 +326,7 @@ no exception caught - + @@ -341,40 +341,46 @@ no exception caught - + unexpected std::exception caught; message: unexpected error - +
- - unexpected std::exception caught; message: unexpected error - +
+
- + + unexpected std::exception caught; message: unexpected error + + unexpected std::exception caught; message: unexpected error - +
- - unexpected std::exception caught; message: unexpected error - +
- + + unexpected std::exception caught; message: unexpected error + + - + + not interesting + + @@ -386,7 +392,7 @@ no exception caught - + @@ -416,24 +422,26 @@ no exception caught - +
failure +
no exception caught - +
failure 1 +
failure 7 @@ -442,44 +450,34 @@ failure 2 - -
failure 3 +
-
- - failure 7 - -
+ + failure 7 + failure 2 -
-
failure 4 -
-
-
-
failure 5 +
-
-
-
-
failure 6 +
+
failure 7 @@ -487,7 +485,7 @@ no exception caught - +
@@ -500,6 +498,7 @@ failure 1 +
info 1 @@ -517,6 +516,7 @@ failure 2 + info 1 @@ -527,20 +527,20 @@ no exception caught - + escape <>&"' in messages - + &&&&&&&&&&&&&&&&&&&&&&&&&... - + - - + + From 5882457fb811e705e31920d754dddd29ef9b0f3d Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 20:45:07 +0100 Subject: [PATCH 09/20] Add snitch_time.cpp --- CMakeLists.txt | 4 +++- include/snitch/snitch_time.hpp | 17 +++++++++++++++++ src/snitch.cpp | 1 + src/snitch_registry.cpp | 18 ++++++------------ src/snitch_time.cpp | 24 ++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 include/snitch/snitch_time.hpp create mode 100644 src/snitch_time.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bc8eef6c..36033218 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,6 +91,7 @@ set(SNITCH_INCLUDES ${PROJECT_SOURCE_DIR}/include/snitch/snitch_string.hpp ${PROJECT_SOURCE_DIR}/include/snitch/snitch_string_utility.hpp ${PROJECT_SOURCE_DIR}/include/snitch/snitch_test_data.hpp + ${PROJECT_SOURCE_DIR}/include/snitch/snitch_time.hpp ${PROJECT_SOURCE_DIR}/include/snitch/snitch_type_id.hpp ${PROJECT_SOURCE_DIR}/include/snitch/snitch_type_name.hpp ${PROJECT_SOURCE_DIR}/include/snitch/snitch_vector.hpp @@ -111,7 +112,8 @@ set(SNITCH_SOURCES_INDIVIDUAL ${PROJECT_SOURCE_DIR}/src/snitch_reporter_teamcity.cpp ${PROJECT_SOURCE_DIR}/src/snitch_section.cpp ${PROJECT_SOURCE_DIR}/src/snitch_string_utility.cpp - ${PROJECT_SOURCE_DIR}/src/snitch_test_data.cpp) + ${PROJECT_SOURCE_DIR}/src/snitch_test_data.cpp + ${PROJECT_SOURCE_DIR}/src/snitch_time.cpp) if (SNITCH_UNITY_BUILD) set(SNITCH_SOURCES ${PROJECT_SOURCE_DIR}/src/snitch.cpp) diff --git a/include/snitch/snitch_time.hpp b/include/snitch/snitch_time.hpp new file mode 100644 index 00000000..482a0ca1 --- /dev/null +++ b/include/snitch/snitch_time.hpp @@ -0,0 +1,17 @@ +#ifndef SNITCH_TIME_HPP +#define SNITCH_TIME_HPP + +#include "snitch/snitch_config.hpp" + +#if SNITCH_WITH_TIMINGS + +namespace snitch { +using time_point_t = std::size_t; + +SNITCH_EXPORT time_point_t get_current_time() noexcept; + +SNITCH_EXPORT float get_duration_in_seconds(time_point_t start, time_point_t end) noexcept; +} // namespace snitch + +#endif +#endif diff --git a/src/snitch.cpp b/src/snitch.cpp index 08d45ace..35d2803c 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -13,3 +13,4 @@ #include "snitch_section.cpp" #include "snitch_string_utility.cpp" #include "snitch_test_data.cpp" +#include "snitch_time.cpp" diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index bde0cf22..d4be8893 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -1,12 +1,10 @@ #include "snitch/snitch_registry.hpp" +#include "snitch/snitch_time.hpp" + #include // for std::sort #include // for std::optional -#if SNITCH_WITH_TIMINGS -# include // for measuring test time -#endif - // Testing framework implementation. // --------------------------------- @@ -543,8 +541,7 @@ impl::test_state registry::run(impl::test_case& test) noexcept { impl::set_current_test(&state); #if SNITCH_WITH_TIMINGS - using clock = std::chrono::steady_clock; - auto time_start = clock::now(); + const auto time_start = get_current_time(); #endif #if SNITCH_WITH_EXCEPTIONS @@ -598,8 +595,7 @@ impl::test_state registry::run(impl::test_case& test) noexcept { } #if SNITCH_WITH_TIMINGS - auto time_end = clock::now(); - state.duration = std::chrono::duration(time_end - time_start).count(); + state.duration = get_duration_in_seconds(time_start, get_current_time()); #endif if (verbose >= registry::verbosity::high) { @@ -650,8 +646,7 @@ bool registry::run_selected_tests( std::size_t allowed_assertion_failure_count = 0; #if SNITCH_WITH_TIMINGS - using clock = std::chrono::steady_clock; - auto time_start = clock::now(); + const auto time_start = get_current_time(); #endif for (impl::test_case& t : this->test_cases()) { @@ -692,8 +687,7 @@ bool registry::run_selected_tests( } #if SNITCH_WITH_TIMINGS - auto time_end = clock::now(); - float duration = std::chrono::duration(time_end - time_start).count(); + const float duration = get_duration_in_seconds(time_start, get_current_time()); #endif if (verbose >= registry::verbosity::normal) { diff --git a/src/snitch_time.cpp b/src/snitch_time.cpp new file mode 100644 index 00000000..d5744a25 --- /dev/null +++ b/src/snitch_time.cpp @@ -0,0 +1,24 @@ +#include "snitch/snitch_time.hpp" + +#if SNITCH_WITH_TIMINGS +# include + +namespace snitch { +namespace impl { +using clock = std::chrono::steady_clock; +using tick_resolution = std::chrono::microseconds; +} // namespace impl + +time_point_t get_current_time() noexcept { + static auto start_time = impl::clock::now(); + return static_cast( + std::chrono::duration_cast(impl::clock::now() - start_time).count()); +} + +float get_duration_in_seconds(time_point_t start, time_point_t end) noexcept { + return std::chrono::duration_cast>( + impl::tick_resolution(end - start)) + .count(); +} +} // namespace snitch +#endif From 455994edccbd2330889e98a3a40f9a58813d333b Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 20:48:10 +0100 Subject: [PATCH 10/20] Move section reporting to registry --- include/snitch/snitch_macros_misc.hpp | 2 +- include/snitch/snitch_registry.hpp | 6 +++ include/snitch/snitch_section.hpp | 13 ++---- include/snitch/snitch_test_data.hpp | 6 +++ src/snitch_registry.cpp | 34 ++++++++++++++++ src/snitch_section.cpp | 58 ++++++++++----------------- 6 files changed, 73 insertions(+), 46 deletions(-) diff --git a/include/snitch/snitch_macros_misc.hpp b/include/snitch/snitch_macros_misc.hpp index 66187ca7..74a5fd4d 100644 --- a/include/snitch/snitch_macros_misc.hpp +++ b/include/snitch/snitch_macros_misc.hpp @@ -9,7 +9,7 @@ #define SNITCH_SECTION(...) \ if (snitch::impl::section_entry_checker SNITCH_MACRO_CONCAT(section_id_, __COUNTER__){ \ - {{__VA_ARGS__}, SNITCH_CURRENT_LOCATION}, snitch::impl::get_current_test()}) + {__VA_ARGS__}, SNITCH_CURRENT_LOCATION, snitch::impl::get_current_test()}) #define SNITCH_CAPTURE(...) \ auto SNITCH_MACRO_CONCAT(capture_id_, __COUNTER__) = \ diff --git a/include/snitch/snitch_registry.hpp b/include/snitch/snitch_registry.hpp index 6fe63d29..051f9da0 100644 --- a/include/snitch/snitch_registry.hpp +++ b/include/snitch/snitch_registry.hpp @@ -291,6 +291,12 @@ class registry { // Internal API; do not use. SNITCH_EXPORT static void report_skipped(std::string_view message) noexcept; + // Internal API; do not use. + SNITCH_EXPORT static void report_section_started(const section& sec) noexcept; + + // Internal API; do not use. + SNITCH_EXPORT static void report_section_ended(const section& sec) noexcept; + // Internal API; do not use. SNITCH_EXPORT impl::test_state run(impl::test_case& test) noexcept; diff --git a/include/snitch/snitch_section.hpp b/include/snitch/snitch_section.hpp index 73239f51..8f83d3d0 100644 --- a/include/snitch/snitch_section.hpp +++ b/include/snitch/snitch_section.hpp @@ -4,17 +4,12 @@ #include "snitch/snitch_config.hpp" #include "snitch/snitch_test_data.hpp" -#include - namespace snitch::impl { struct section_entry_checker { - section data = {}; - test_state& state; - bool entered = false; - -#if SNITCH_WITH_TIMINGS - std::make_signed_t start_time = 0; -#endif + section_id id = {}; + source_location location = {}; + test_state& state; + bool entered = false; SNITCH_EXPORT ~section_entry_checker(); diff --git a/include/snitch/snitch_test_data.hpp b/include/snitch/snitch_test_data.hpp index e9b58abf..733216be 100644 --- a/include/snitch/snitch_test_data.hpp +++ b/include/snitch/snitch_test_data.hpp @@ -3,6 +3,7 @@ #include "snitch/snitch_config.hpp" #include "snitch/snitch_string.hpp" +#include "snitch/snitch_time.hpp" #include "snitch/snitch_vector.hpp" #include @@ -53,6 +54,11 @@ struct section { std::size_t assertion_failure_count = 0; /// Counts allowed failed assertions (e.g., [!shouldfail] and [!mayfail]) std::size_t allowed_assertion_failure_count = 0; + +#if SNITCH_WITH_TIMINGS + /// Time index of the instant when the section was first entered. + time_point_t start_time = 0; +#endif }; /// List of test case filters diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index d4be8893..1858ab87 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -511,6 +511,40 @@ void registry::report_skipped(std::string_view message) noexcept { message}); } +void registry::report_section_started(const section& sec) noexcept { + const impl::test_state& state = impl::get_current_test(); + + state.reg.report_callback(state.reg, event::section_started{sec.id, sec.location}); +} + +void registry::report_section_ended(const section& sec) noexcept { + const impl::test_state& state = impl::get_current_test(); + + const bool skipped = state.test.state == impl::test_case_state::skipped; + +#if SNITCH_WITH_TIMINGS + const auto duration = get_duration_in_seconds(sec.start_time, get_current_time()); + state.reg.report_callback( + state.reg, event::section_ended{ + .id = sec.id, + .location = sec.location, + .skipped = skipped, + .assertion_count = sec.assertion_count, + .assertion_failure_count = sec.assertion_failure_count, + .allowed_assertion_failure_count = sec.allowed_assertion_failure_count, + .duration = duration}); +#else + state.reg.report_callback( + state.reg, event::section_ended{ + .id = sec.id, + .location = sec.location, + .skipped = skipped, + .assertion_count = sec.assertion_count, + .assertion_failure_count = sec.assertion_failure_count, + .allowed_assertion_failure_count = sec.allowed_assertion_failure_count}); +#endif +} + impl::test_state registry::run(impl::test_case& test) noexcept { if (verbose >= registry::verbosity::high) { report_callback(*this, event::test_case_started{test.id, test.location}); diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index 86e6db5f..0f4ccd8b 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -3,20 +3,13 @@ #include "snitch/snitch_console.hpp" #include "snitch/snitch_registry.hpp" #include "snitch/snitch_test_data.hpp" +#include "snitch/snitch_time.hpp" #if SNITCH_WITH_EXCEPTIONS # include #endif -#if SNITCH_WITH_TIMINGS -# include -#endif namespace snitch::impl { -#if SNITCH_WITH_TIMINGS -using fsec = std::chrono::duration; -using snitch_clock = std::chrono::steady_clock; -#endif - section_entry_checker::~section_entry_checker() { auto& sections = state.info.sections; @@ -60,24 +53,14 @@ section_entry_checker::~section_entry_checker() { } } - // Emit the section end event (only on last entry). - if (last_entry) { - const auto& section = sections.current_section.back(); - -#if SNITCH_WITH_TIMINGS - const auto end_time = snitch_clock::now().time_since_epoch(); - const auto duration = - std::chrono::duration_cast(end_time - snitch_clock::duration{start_time}); -#endif - - state.reg.report_callback( - state.reg, event::section_ended { - data.id, data.location, false, section.assertion_count, - section.assertion_failure_count, section.allowed_assertion_failure_count, -#if SNITCH_WITH_TIMINGS - duration.count() + // Emit the section end event (only on last entry, and only if no exception in flight). +#if SNITCH_WITH_EXCEPTIONS + if (last_entry && std::uncaught_exceptions() == 0) +#else + if (last_entry) #endif - }); + { + registry::report_section_ended(sections.current_section.back()); } sections.current_section.pop_back(); @@ -136,21 +119,24 @@ section_entry_checker::operator bool() { // Entering this section. - // Emit the section start event (only on first entry). - if (previous_was_preceeding_sibling) { - state.reg.report_callback(state.reg, event::section_started{data.id, data.location}); - } - + // Push new section on the stack. + level.previous_section_id = level.current_section_id; + sections.current_section.push_back( #if SNITCH_WITH_TIMINGS - // Start the clock. - start_time = snitch_clock::now().time_since_epoch().count(); + section{.id = id, .location = location, .start_time = get_current_time()} +#else + section{.id = id, .location = location} #endif + ); - // Push new section on the stack. - level.previous_section_id = level.current_section_id; - sections.current_section.push_back(data); - push_location(state, {data.location.file, data.location.line, location_type::section_scope}); + push_location(state, {location.file, location.line, location_type::section_scope}); entered = true; + + // Emit the section start event (only on first entry). + if (previous_was_preceeding_sibling) { + registry::report_section_started(sections.current_section.back()); + } + return true; } } // namespace snitch::impl From 52ecd2643be94d6266243e255f3e8f99aca53f69 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 20:57:27 +0100 Subject: [PATCH 11/20] Increase time resolution --- src/snitch_time.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snitch_time.cpp b/src/snitch_time.cpp index d5744a25..8ce5b7b2 100644 --- a/src/snitch_time.cpp +++ b/src/snitch_time.cpp @@ -6,7 +6,7 @@ namespace snitch { namespace impl { using clock = std::chrono::steady_clock; -using tick_resolution = std::chrono::microseconds; +using tick_resolution = std::chrono::nanoseconds; } // namespace impl time_point_t get_current_time() noexcept { From 686d2c7eaf812f49605c2003920be23acc64f0d9 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:08:45 +0100 Subject: [PATCH 12/20] Close all open sections when asserting on unhandled exception --- src/snitch_registry.cpp | 41 ++++++++++++++++++- .../data/expected/reporter_catch2_xml_allfail | 18 ++++---- .../data/expected/reporter_catch2_xml_default | 18 ++++---- .../data/expected/reporter_catch2_xml_full | 18 ++++---- 4 files changed, 66 insertions(+), 29 deletions(-) diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index 1858ab87..a38759ed 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -399,6 +399,15 @@ void register_assertion(bool success, impl::test_state& state) { ++section.allowed_assertion_failure_count; } +#if SNITCH_WITH_EXCEPTIONS + if (state.held_info.has_value()) { + for (auto& section : state.held_info.value().sections.current_section) { + ++section.assertion_count; + ++section.allowed_assertion_failure_count; + } + } +#endif + impl::set_state(state.test, impl::test_case_state::allowed_fail); } else { ++state.asserts; @@ -409,13 +418,31 @@ void register_assertion(bool success, impl::test_state& state) { ++section.assertion_failure_count; } +#if SNITCH_WITH_EXCEPTIONS + if (state.held_info.has_value()) { + for (auto& section : state.held_info.value().sections.current_section) { + ++section.assertion_count; + ++section.assertion_failure_count; + } + } +#endif + impl::set_state(state.test, impl::test_case_state::failed); } } else { ++state.asserts; + for (auto& section : state.info.sections.current_section) { ++section.assertion_count; } + +#if SNITCH_WITH_EXCEPTIONS + if (state.held_info.has_value()) { + for (auto& section : state.held_info.value().sections.current_section) { + ++section.assertion_count; + } + } +#endif } } @@ -434,8 +461,8 @@ void report_assertion_impl( const auto captures_buffer = impl::make_capture_buffer( use_held_info ? state.held_info.value().captures : state.info.captures); - const auto& current_section = use_held_info ? state.held_info.value().sections.current_section - : state.info.sections.current_section; + auto& current_section = use_held_info ? state.held_info.value().sections.current_section + : state.info.sections.current_section; const auto& last_location = use_held_info ? state.held_info.value().locations.back() : state.info.locations.back(); @@ -464,6 +491,16 @@ void report_assertion_impl( state.test.id, current_section, captures_buffer.span(), location, data, state.should_fail, state.may_fail}); } + +#if SNITCH_WITH_EXCEPTIONS + if (state.unhandled_exception) { + // Close all sections that were left open by the exception. + while (!current_section.empty()) { + registry::report_section_ended(current_section.back()); + current_section.pop_back(); + } + } +#endif } } // namespace diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail index f45ede15..b8957f11 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_allfail +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_allfail @@ -156,13 +156,13 @@
- + + unexpected std::exception caught; message: unexpected error + +
- +
- - unexpected std::exception caught; message: unexpected error - @@ -173,11 +173,11 @@
- + + unexpected std::exception caught; message: unexpected error + +
- - unexpected std::exception caught; message: unexpected error - diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index fd15a026..0e5dc445 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -226,13 +226,13 @@
- + + unexpected std::exception caught; message: unexpected error + +
- +
- - unexpected std::exception caught; message: unexpected error - @@ -243,11 +243,11 @@
- + + unexpected std::exception caught; message: unexpected error + +
- - unexpected std::exception caught; message: unexpected error - diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index 4220a079..44799724 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -352,13 +352,13 @@
- + + unexpected std::exception caught; message: unexpected error + +
- +
- - unexpected std::exception caught; message: unexpected error - @@ -369,11 +369,11 @@
- + + unexpected std::exception caught; message: unexpected error + +
- - unexpected std::exception caught; message: unexpected error - From b0336d741bb1b64075d9bd2aa709c25a03c2bc84 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:16:12 +0100 Subject: [PATCH 13/20] Testing SKIP in SECTION --- src/snitch_registry.cpp | 29 +++++++++++-------- .../data/expected/reporter_catch2_xml_default | 16 ++++++++-- .../data/expected/reporter_catch2_xml_full | 16 ++++++++-- .../expected/reporter_catch2_xml_list_tests | 9 ++++++ .../data/expected/reporter_console_default | 7 ++++- .../data/expected/reporter_console_full | 9 +++++- .../data/expected/reporter_console_list_tests | 3 +- .../data/expected/reporter_console_withcolor | 7 ++++- .../data/expected/reporter_teamcity_default | 3 ++ .../data/expected/reporter_teamcity_full | 3 ++ .../expected/reporter_teamcity_list_tests | 1 + tests/testing_reporters.cpp | 14 +++++++++ 12 files changed, 97 insertions(+), 20 deletions(-) diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index a38759ed..65f196e9 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -461,8 +461,8 @@ void report_assertion_impl( const auto captures_buffer = impl::make_capture_buffer( use_held_info ? state.held_info.value().captures : state.info.captures); - auto& current_section = use_held_info ? state.held_info.value().sections.current_section - : state.info.sections.current_section; + const auto& current_section = use_held_info ? state.held_info.value().sections.current_section + : state.info.sections.current_section; const auto& last_location = use_held_info ? state.held_info.value().locations.back() : state.info.locations.back(); @@ -491,16 +491,6 @@ void report_assertion_impl( state.test.id, current_section, captures_buffer.span(), location, data, state.should_fail, state.may_fail}); } - -#if SNITCH_WITH_EXCEPTIONS - if (state.unhandled_exception) { - // Close all sections that were left open by the exception. - while (!current_section.empty()) { - registry::report_section_ended(current_section.back()); - current_section.pop_back(); - } - } -#endif } } // namespace @@ -647,6 +637,7 @@ impl::test_state registry::run(impl::test_case& test) noexcept { state.in_check = false; } catch (const impl::abort_exception&) { // Test aborted, assume its state was already set accordingly. + state.unhandled_exception = true; } catch (const std::exception& e) { state.unhandled_exception = true; report_assertion(false, "unexpected std::exception caught; message: ", e.what()); @@ -654,6 +645,20 @@ impl::test_state registry::run(impl::test_case& test) noexcept { state.unhandled_exception = true; report_assertion(false, "unexpected unknown exception caught"); } + + if (state.unhandled_exception) { + auto& current_section = state.held_info.has_value() + ? state.held_info.value().sections.current_section + : state.info.sections.current_section; + + // Close all sections that were left open by the exception. + while (!current_section.empty()) { + registry::report_section_ended(current_section.back()); + current_section.pop_back(); + } + } + + state.unhandled_exception = false; #endif if (state.should_fail) { diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index 0e5dc445..0df66f75 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -388,6 +388,18 @@ + +
+
+ + stopping here + + +
+ +
+ + escape <>&"' in messages @@ -400,6 +412,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index 44799724..64746b6d 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -529,6 +529,18 @@ + +
+
+ + stopping here + + +
+ +
+ + escape <>&"' in messages @@ -541,6 +553,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests index efa8af76..a2eb2f40 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests @@ -378,6 +378,15 @@ * + + test SKIP in SECTION + + + + *testing_reporters.cpp + * + + test escape <>&"' diff --git a/tests/approval_tests/data/expected/reporter_console_default b/tests/approval_tests/data/expected/reporter_console_default index 9fe5dccb..457e9995 100644 --- a/tests/approval_tests/data/expected/reporter_console_default +++ b/tests/approval_tests/data/expected/reporter_console_default @@ -169,5 +169,10 @@ failed: running test case "test SECTION & INFO" at *testing_reporters.cpp:* with info 1 failure 3 +skipped: running test case "test SKIP in SECTION" + in section "section 1" + in section "section 2" + at *testing_reporters.cpp:* + stopping here ========================================== -error: some tests failed (25 out of 42 test cases, 86 assertions, 1 test cases skipped, * seconds) +error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_full b/tests/approval_tests/data/expected/reporter_console_full index 5a586051..915665ac 100644 --- a/tests/approval_tests/data/expected/reporter_console_full +++ b/tests/approval_tests/data/expected/reporter_console_full @@ -387,5 +387,12 @@ passed: running test case "test SECTION & INFO" at *testing_reporters.cpp:* no exception caught finished: test SECTION & INFO (*) +starting: test SKIP in SECTION at *testing_reporters.cpp:* +skipped: running test case "test SKIP in SECTION" + in section "section 1" + in section "section 2" + at *testing_reporters.cpp:* + stopping here +finished: test SKIP in SECTION (*) ========================================== -error: some tests failed (25 out of 42 test cases, 86 assertions, 1 test cases skipped, * seconds) +error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_list_tests b/tests/approval_tests/data/expected/reporter_console_list_tests index 881843a7..207fd3fd 100644 --- a/tests/approval_tests/data/expected/reporter_console_list_tests +++ b/tests/approval_tests/data/expected/reporter_console_list_tests @@ -53,4 +53,5 @@ Matching test cases: test SECTION test multiple SECTION test SECTION & INFO -42 matching test cases + test SKIP in SECTION +43 matching test cases diff --git a/tests/approval_tests/data/expected/reporter_console_withcolor b/tests/approval_tests/data/expected/reporter_console_withcolor index 0ff12361..a6efab59 100644 --- a/tests/approval_tests/data/expected/reporter_console_withcolor +++ b/tests/approval_tests/data/expected/reporter_console_withcolor @@ -169,5 +169,10 @@ at *testing_reporters.cpp:* with info 1 failure 3 +skipped: running test case "test SKIP in SECTION" + in section "section 1" + in section "section 2" + at *testing_reporters.cpp:* + stopping here ========================================== -error: some tests failed (25 out of 42 test cases, 86 assertions, 1 test cases skipped, * seconds) +error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_teamcity_default b/tests/approval_tests/data/expected/reporter_teamcity_default index 729e29ae..32ba30c4 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_default +++ b/tests/approval_tests/data/expected/reporter_teamcity_default @@ -128,6 +128,9 @@ ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 2|ninfo 1|ninfo 3|n failure 2'] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] +##teamCity[testStarted name='test SKIP in SECTION'] +##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n stopping here'] +##teamCity[testFinished name='test SKIP in SECTION' duration='*'] ##teamCity[testStarted name='test escape |||'|n|r|[|]'] ##teamCity[testFailed name='test escape |||'|n|r|[|]' message='*reporter_teamcity.cpp:*|n escape || message |||| || |'|n|r|[|]'] ##teamCity[testFinished name='test escape |||'|n|r|[|]' duration='*'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_full b/tests/approval_tests/data/expected/reporter_teamcity_full index 6134b574..423beeb2 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_full +++ b/tests/approval_tests/data/expected/reporter_teamcity_full @@ -170,6 +170,9 @@ ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] ##teamCity[testStdOut name='test SECTION & INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] +##teamCity[testStarted name='test SKIP in SECTION'] +##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n stopping here'] +##teamCity[testFinished name='test SKIP in SECTION' duration='*'] ##teamCity[testStarted name='test escape |||'|n|r|[|]'] ##teamCity[testFailed name='test escape |||'|n|r|[|]' message='*reporter_teamcity.cpp:*|n escape || message |||| || |'|n|r|[|]'] ##teamCity[testFinished name='test escape |||'|n|r|[|]' duration='*'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_list_tests b/tests/approval_tests/data/expected/reporter_teamcity_list_tests index e97ac6f8..15bfd7b6 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_list_tests +++ b/tests/approval_tests/data/expected/reporter_teamcity_list_tests @@ -40,5 +40,6 @@ test multiple INFO test SECTION test multiple SECTION test SECTION & INFO +test SKIP in SECTION test escape |||'|n|r|[|] test escape very long diff --git a/tests/testing_reporters.cpp b/tests/testing_reporters.cpp index 280d046f..060cb2d9 100644 --- a/tests/testing_reporters.cpp +++ b/tests/testing_reporters.cpp @@ -194,6 +194,20 @@ void register_tests_for_reporters(snitch::registry& r) { } SNITCH_FAIL_CHECK("failure 3"); }); + + r.add({"test SKIP in SECTION"}, SNITCH_CURRENT_LOCATION, []() { + SNITCH_SECTION("section 1") { + SNITCH_SECTION("section 2") { + SNITCH_SKIP("stopping here"); + SNITCH_SECTION("section 3") { + SNITCH_FAIL_CHECK("failure 1"); + } + } + } + SNITCH_SECTION("section 2") { + SNITCH_FAIL_CHECK("failure 2"); + } + }); } SNITCH_WARNING_POP From a6625265032741d7e6964b37bd7381380530a691 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:24:14 +0100 Subject: [PATCH 14/20] Implement section reporting with TeamCity --- src/snitch_reporter_teamcity.cpp | 9 +++++-- .../data/expected/reporter_teamcity_allfail | 6 +++++ .../data/expected/reporter_teamcity_default | 26 +++++++++++++++++++ .../data/expected/reporter_teamcity_full | 26 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/snitch_reporter_teamcity.cpp b/src/snitch_reporter_teamcity.cpp index 2e28b784..573a8b40 100644 --- a/src/snitch_reporter_teamcity.cpp +++ b/src/snitch_reporter_teamcity.cpp @@ -148,8 +148,13 @@ void report(const registry& r, const snitch::event::data& event) noexcept { send_message(r, "testFinished", {{"name", make_full_name(e.id)}}); # endif }, - [&](const snitch::event::section_started&) {}, - [&](const snitch::event::section_ended&) {}, + [&](const snitch::event::section_started& e) { + send_message( + r, "blockOpened", {{"name", e.id.name}, {"description", e.id.description}}); + }, + [&](const snitch::event::section_ended& e) { + send_message(r, "blockClosed", {{"name", e.id.name}}); + }, [&](const snitch::event::test_case_skipped& e) { send_message( r, "testIgnored", diff --git a/tests/approval_tests/data/expected/reporter_teamcity_allfail b/tests/approval_tests/data/expected/reporter_teamcity_allfail index ab042213..2e0020d0 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_allfail +++ b/tests/approval_tests/data/expected/reporter_teamcity_allfail @@ -50,12 +50,18 @@ ##teamCity[testFailed name='test unexpected throw fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[testFinished name='test unexpected throw fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in section fail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 2'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in section fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check fail'] ##teamCity[testFailed name='test unexpected throw in check fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[testFinished name='test unexpected throw in check fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check & section fail'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|nsection 1|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] ##teamCity[testSuiteFinished name='test "* fail*"'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_default b/tests/approval_tests/data/expected/reporter_teamcity_default index 32ba30c4..0020ba29 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_default +++ b/tests/approval_tests/data/expected/reporter_teamcity_default @@ -87,13 +87,19 @@ ##teamCity[testFailed name='test unexpected throw fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[testFinished name='test unexpected throw fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in section fail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 2'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in section fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check fail'] ##teamCity[testFailed name='test unexpected throw in check fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[testFinished name='test unexpected throw in check fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check & section fail'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|nsection 1|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] @@ -108,28 +114,48 @@ ##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 4'] ##teamCity[testFinished name='test multiple INFO' duration='*'] ##teamCity[testStarted name='test SECTION'] +##teamCity[blockOpened name='section' description=''] ##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|nsection|n failure'] +##teamCity[blockClosed name='section'] ##teamCity[testFinished name='test SECTION' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 1|n failure 1'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[blockOpened name='section 2.1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.1|n failure 3'] +##teamCity[blockClosed name='section 2.1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[blockOpened name='section 2.2' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 4'] +##teamCity[blockOpened name='section 2.2.1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|nsection 2.2.1|n failure 5'] +##teamCity[blockClosed name='section 2.2.1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 6'] +##teamCity[blockClosed name='section 2.2'] +##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[testFinished name='test multiple SECTION' duration='*'] ##teamCity[testStarted name='test SECTION & INFO'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 1|ninfo 1|ninfo 2|n failure 1'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 2|ninfo 1|ninfo 3|n failure 2'] +##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] ##teamCity[testStarted name='test SKIP in SECTION'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n stopping here'] +##teamCity[blockClosed name='section 2'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test SKIP in SECTION' duration='*'] ##teamCity[testStarted name='test escape |||'|n|r|[|]'] ##teamCity[testFailed name='test escape |||'|n|r|[|]' message='*reporter_teamcity.cpp:*|n escape || message |||| || |'|n|r|[|]'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_full b/tests/approval_tests/data/expected/reporter_teamcity_full index 423beeb2..9b529a14 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_full +++ b/tests/approval_tests/data/expected/reporter_teamcity_full @@ -124,13 +124,19 @@ ##teamCity[testFailed name='test unexpected throw fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[testFinished name='test unexpected throw fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in section fail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 2'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in section fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check fail'] ##teamCity[testFailed name='test unexpected throw in check fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[testFinished name='test unexpected throw in check fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check & section fail'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|nsection 1|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] @@ -147,31 +153,51 @@ ##teamCity[testStdOut name='test multiple INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test multiple INFO' duration='*'] ##teamCity[testStarted name='test SECTION'] +##teamCity[blockOpened name='section' description=''] ##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|nsection|n failure'] +##teamCity[blockClosed name='section'] ##teamCity[testStdOut name='test SECTION' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 1|n failure 1'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[blockOpened name='section 2.1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.1|n failure 3'] +##teamCity[blockClosed name='section 2.1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[blockOpened name='section 2.2' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 4'] +##teamCity[blockOpened name='section 2.2.1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|nsection 2.2.1|n failure 5'] +##teamCity[blockClosed name='section 2.2.1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 6'] +##teamCity[blockClosed name='section 2.2'] +##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[testStdOut name='test multiple SECTION' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test multiple SECTION' duration='*'] ##teamCity[testStarted name='test SECTION & INFO'] +##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 1|ninfo 1|ninfo 2|n failure 1'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 2|ninfo 1|ninfo 3|n failure 2'] +##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] ##teamCity[testStdOut name='test SECTION & INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] ##teamCity[testStarted name='test SKIP in SECTION'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[blockOpened name='section 2' description=''] ##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n stopping here'] +##teamCity[blockClosed name='section 2'] +##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test SKIP in SECTION' duration='*'] ##teamCity[testStarted name='test escape |||'|n|r|[|]'] ##teamCity[testFailed name='test escape |||'|n|r|[|]' message='*reporter_teamcity.cpp:*|n escape || message |||| || |'|n|r|[|]'] From 6f1087d0f63bca505ad9f62ef1982febbc0d8d00 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:24:56 +0100 Subject: [PATCH 15/20] Limit section events to verbosity high --- src/snitch_registry.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index 65f196e9..63ac7cac 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -541,12 +541,20 @@ void registry::report_skipped(std::string_view message) noexcept { void registry::report_section_started(const section& sec) noexcept { const impl::test_state& state = impl::get_current_test(); + if (state.reg.verbose < registry::verbosity::high) { + return; + } + state.reg.report_callback(state.reg, event::section_started{sec.id, sec.location}); } void registry::report_section_ended(const section& sec) noexcept { const impl::test_state& state = impl::get_current_test(); + if (state.reg.verbose < registry::verbosity::high) { + return; + } + const bool skipped = state.test.state == impl::test_case_state::skipped; #if SNITCH_WITH_TIMINGS From a3502c0213a7780588acc27181736e2ad15729bf Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:25:11 +0100 Subject: [PATCH 16/20] Implement section reporting for console reporter --- src/snitch_reporter_console.cpp | 13 ++++++++-- .../data/expected/reporter_console_full | 26 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/snitch_reporter_console.cpp b/src/snitch_reporter_console.cpp index 4ef2d8f6..15bf1bf6 100644 --- a/src/snitch_reporter_console.cpp +++ b/src/snitch_reporter_console.cpp @@ -162,8 +162,17 @@ void reporter::report(const registry& r, const event::data& event) noexcept { make_colored(full_name, r.with_color, color::highlight1), "\n"); #endif }, - [&](const snitch::event::section_started&) {}, - [&](const snitch::event::section_ended&) {}, + [&](const snitch::event::section_started& e) { + r.print( + make_colored("entering section:", r.with_color, color::status), " ", + make_colored(e.id.name, r.with_color, color::highlight1), " at ", + e.location.file, ":", e.location.line, "\n"); + }, + [&](const snitch::event::section_ended& e) { + r.print( + make_colored("leaving section:", r.with_color, color::status), " ", + make_colored(e.id.name, r.with_color, color::highlight1), "\n"); + }, [&](const snitch::event::test_case_skipped& e) { r.print(make_colored("skipped: ", r.with_color, color::skipped)); print_location(r, e.id, e.sections, e.captures, e.location); diff --git a/tests/approval_tests/data/expected/reporter_console_full b/tests/approval_tests/data/expected/reporter_console_full index 915665ac..369a1dbd 100644 --- a/tests/approval_tests/data/expected/reporter_console_full +++ b/tests/approval_tests/data/expected/reporter_console_full @@ -254,11 +254,15 @@ failed: running test case "test unexpected throw fail" unexpected std::exception caught; message: unexpected error finished: test unexpected throw fail (*) starting: test unexpected throw in section fail at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* +entering section: section 2 at *testing_reporters.cpp:* failed: running test case "test unexpected throw in section fail" in section "section 1" in section "section 2" somewhere inside section at *testing_reporters.cpp:* unexpected std::exception caught; message: unexpected error +leaving section: section 2 +leaving section: section 1 finished: test unexpected throw in section fail (*) starting: test unexpected throw in check fail at *testing_reporters.cpp:* failed: running test case "test unexpected throw in check fail" @@ -266,10 +270,12 @@ failed: running test case "test unexpected throw in check fail" unexpected std::exception caught; message: unexpected error finished: test unexpected throw in check fail (*) starting: test unexpected throw in check & section fail at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* failed: running test case "test unexpected throw in check & section fail" in section "section 1" somewhere inside check at *testing_reporters.cpp:* unexpected std::exception caught; message: unexpected error +leaving section: section 1 finished: test unexpected throw in check & section fail (*) starting: test SKIP at *testing_reporters.cpp:* skipped: running test case "test SKIP" @@ -307,31 +313,38 @@ passed: running test case "test multiple INFO" no exception caught finished: test multiple INFO (*) starting: test SECTION at *testing_reporters.cpp:* +entering section: section at *testing_reporters.cpp:* failed: running test case "test SECTION" in section "section" at *testing_reporters.cpp:* failure +leaving section: section passed: running test case "test SECTION" at *testing_reporters.cpp:* no exception caught finished: test SECTION (*) starting: test multiple SECTION at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" in section "section 1" at *testing_reporters.cpp:* failure 1 +leaving section: section 1 failed: running test case "test multiple SECTION" at *testing_reporters.cpp:* failure 7 +entering section: section 2 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" in section "section 2" at *testing_reporters.cpp:* failure 2 +entering section: section 2.1 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" in section "section 2" in section "section 2.1" at *testing_reporters.cpp:* failure 3 +leaving section: section 2.1 failed: running test case "test multiple SECTION" at *testing_reporters.cpp:* failure 7 @@ -339,22 +352,27 @@ failed: running test case "test multiple SECTION" in section "section 2" at *testing_reporters.cpp:* failure 2 +entering section: section 2.2 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" in section "section 2" in section "section 2.2" at *testing_reporters.cpp:* failure 4 +entering section: section 2.2.1 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" in section "section 2" in section "section 2.2" in section "section 2.2.1" at *testing_reporters.cpp:* failure 5 +leaving section: section 2.2.1 failed: running test case "test multiple SECTION" in section "section 2" in section "section 2.2" at *testing_reporters.cpp:* failure 6 +leaving section: section 2.2 +leaving section: section 2 failed: running test case "test multiple SECTION" at *testing_reporters.cpp:* failure 7 @@ -363,22 +381,26 @@ passed: running test case "test multiple SECTION" no exception caught finished: test multiple SECTION (*) starting: test SECTION & INFO at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* failed: running test case "test SECTION & INFO" in section "section 1" at *testing_reporters.cpp:* with info 1 with info 2 failure 1 +leaving section: section 1 failed: running test case "test SECTION & INFO" at *testing_reporters.cpp:* with info 1 failure 3 +entering section: section 2 at *testing_reporters.cpp:* failed: running test case "test SECTION & INFO" in section "section 2" at *testing_reporters.cpp:* with info 1 with info 3 failure 2 +leaving section: section 2 failed: running test case "test SECTION & INFO" at *testing_reporters.cpp:* with info 1 @@ -388,11 +410,15 @@ passed: running test case "test SECTION & INFO" no exception caught finished: test SECTION & INFO (*) starting: test SKIP in SECTION at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* +entering section: section 2 at *testing_reporters.cpp:* skipped: running test case "test SKIP in SECTION" in section "section 1" in section "section 2" at *testing_reporters.cpp:* stopping here +leaving section: section 2 +leaving section: section 1 finished: test SKIP in SECTION (*) ========================================== error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) From f489838d8965bd2eb6c9c0bd045fe1b7594c462e Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:30:52 +0100 Subject: [PATCH 17/20] Add approval test with capture --- .../data/expected/reporter_catch2_xml_default | 43 ++++++++++++++++- .../data/expected/reporter_catch2_xml_full | 46 ++++++++++++++++++- .../expected/reporter_catch2_xml_list_tests | 9 ++++ .../data/expected/reporter_console_default | 22 ++++++++- .../data/expected/reporter_console_full | 31 ++++++++++++- .../data/expected/reporter_console_list_tests | 3 +- .../data/expected/reporter_console_withcolor | 22 ++++++++- .../data/expected/reporter_teamcity_default | 10 ++++ .../data/expected/reporter_teamcity_full | 11 +++++ .../expected/reporter_teamcity_list_tests | 1 + tests/testing_reporters.cpp | 16 +++++++ 11 files changed, 206 insertions(+), 8 deletions(-) diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index 0df66f75..4c619f9f 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -388,6 +388,45 @@ + +
+ + i := 1 + + + j := 2 + + + failure 1 + + +
+ + i := 1 + + + failure 3 + +
+ + i := 1 + + + j := 3 + + + failure 2 + + +
+ + i := 1 + + + failure 3 + + +
@@ -412,6 +451,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index 64746b6d..b804a355 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -529,6 +529,48 @@ + +
+ + i := 1 + + + j := 2 + + + failure 1 + + +
+ + i := 1 + + + failure 3 + +
+ + i := 1 + + + j := 3 + + + failure 2 + + +
+ + i := 1 + + + failure 3 + + + no exception caught + + +
@@ -553,6 +595,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests index a2eb2f40..50acafc6 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests @@ -378,6 +378,15 @@ * + + test SECTION & CAPTURE + + + + *testing_reporters.cpp + * + + test SKIP in SECTION diff --git a/tests/approval_tests/data/expected/reporter_console_default b/tests/approval_tests/data/expected/reporter_console_default index 457e9995..e7ab4200 100644 --- a/tests/approval_tests/data/expected/reporter_console_default +++ b/tests/approval_tests/data/expected/reporter_console_default @@ -169,10 +169,30 @@ failed: running test case "test SECTION & INFO" at *testing_reporters.cpp:* with info 1 failure 3 +failed: running test case "test SECTION & CAPTURE" + in section "section 1" + at *testing_reporters.cpp:* + with i := 1 + with j := 2 + failure 1 +failed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + with i := 1 + failure 3 +failed: running test case "test SECTION & CAPTURE" + in section "section 2" + at *testing_reporters.cpp:* + with i := 1 + with j := 3 + failure 2 +failed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + with i := 1 + failure 3 skipped: running test case "test SKIP in SECTION" in section "section 1" in section "section 2" at *testing_reporters.cpp:* stopping here ========================================== -error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_full b/tests/approval_tests/data/expected/reporter_console_full index 369a1dbd..d96914f7 100644 --- a/tests/approval_tests/data/expected/reporter_console_full +++ b/tests/approval_tests/data/expected/reporter_console_full @@ -409,6 +409,35 @@ passed: running test case "test SECTION & INFO" at *testing_reporters.cpp:* no exception caught finished: test SECTION & INFO (*) +starting: test SECTION & CAPTURE at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* +failed: running test case "test SECTION & CAPTURE" + in section "section 1" + at *testing_reporters.cpp:* + with i := 1 + with j := 2 + failure 1 +leaving section: section 1 +failed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + with i := 1 + failure 3 +entering section: section 2 at *testing_reporters.cpp:* +failed: running test case "test SECTION & CAPTURE" + in section "section 2" + at *testing_reporters.cpp:* + with i := 1 + with j := 3 + failure 2 +leaving section: section 2 +failed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + with i := 1 + failure 3 +passed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + no exception caught +finished: test SECTION & CAPTURE (*) starting: test SKIP in SECTION at *testing_reporters.cpp:* entering section: section 1 at *testing_reporters.cpp:* entering section: section 2 at *testing_reporters.cpp:* @@ -421,4 +450,4 @@ leaving section: section 2 leaving section: section 1 finished: test SKIP in SECTION (*) ========================================== -error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_list_tests b/tests/approval_tests/data/expected/reporter_console_list_tests index 207fd3fd..685f23d4 100644 --- a/tests/approval_tests/data/expected/reporter_console_list_tests +++ b/tests/approval_tests/data/expected/reporter_console_list_tests @@ -53,5 +53,6 @@ Matching test cases: test SECTION test multiple SECTION test SECTION & INFO + test SECTION & CAPTURE test SKIP in SECTION -43 matching test cases +44 matching test cases diff --git a/tests/approval_tests/data/expected/reporter_console_withcolor b/tests/approval_tests/data/expected/reporter_console_withcolor index a6efab59..55178903 100644 --- a/tests/approval_tests/data/expected/reporter_console_withcolor +++ b/tests/approval_tests/data/expected/reporter_console_withcolor @@ -169,10 +169,30 @@ at *testing_reporters.cpp:* with info 1 failure 3 +failed: running test case "test SECTION & CAPTURE" + in section "section 1" + at *testing_reporters.cpp:* + with i := 1 + with j := 2 + failure 1 +failed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + with i := 1 + failure 3 +failed: running test case "test SECTION & CAPTURE" + in section "section 2" + at *testing_reporters.cpp:* + with i := 1 + with j := 3 + failure 2 +failed: running test case "test SECTION & CAPTURE" + at *testing_reporters.cpp:* + with i := 1 + failure 3 skipped: running test case "test SKIP in SECTION" in section "section 1" in section "section 2" at *testing_reporters.cpp:* stopping here ========================================== -error: some tests failed (25 out of 43 test cases, 86 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_teamcity_default b/tests/approval_tests/data/expected/reporter_teamcity_default index 0020ba29..4afc5ad0 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_default +++ b/tests/approval_tests/data/expected/reporter_teamcity_default @@ -150,6 +150,16 @@ ##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] +##teamCity[testStarted name='test SECTION & CAPTURE'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 1|ni := 1|nj := 2|n failure 1'] +##teamCity[blockClosed name='section 1'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[blockOpened name='section 2' description=''] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 2|ni := 1|nj := 3|n failure 2'] +##teamCity[blockClosed name='section 2'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[testFinished name='test SECTION & CAPTURE' duration='*'] ##teamCity[testStarted name='test SKIP in SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_full b/tests/approval_tests/data/expected/reporter_teamcity_full index 9b529a14..9b1b91f1 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_full +++ b/tests/approval_tests/data/expected/reporter_teamcity_full @@ -192,6 +192,17 @@ ##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] ##teamCity[testStdOut name='test SECTION & INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] +##teamCity[testStarted name='test SECTION & CAPTURE'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 1|ni := 1|nj := 2|n failure 1'] +##teamCity[blockClosed name='section 1'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[blockOpened name='section 2' description=''] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 2|ni := 1|nj := 3|n failure 2'] +##teamCity[blockClosed name='section 2'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[testStdOut name='test SECTION & CAPTURE' out='*testing_reporters.cpp:*|n no exception caught'] +##teamCity[testFinished name='test SECTION & CAPTURE' duration='*'] ##teamCity[testStarted name='test SKIP in SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_list_tests b/tests/approval_tests/data/expected/reporter_teamcity_list_tests index 15bfd7b6..c9000a5f 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_list_tests +++ b/tests/approval_tests/data/expected/reporter_teamcity_list_tests @@ -40,6 +40,7 @@ test multiple INFO test SECTION test multiple SECTION test SECTION & INFO +test SECTION & CAPTURE test SKIP in SECTION test escape |||'|n|r|[|] test escape very long diff --git a/tests/testing_reporters.cpp b/tests/testing_reporters.cpp index 060cb2d9..467e3bb2 100644 --- a/tests/testing_reporters.cpp +++ b/tests/testing_reporters.cpp @@ -195,6 +195,22 @@ void register_tests_for_reporters(snitch::registry& r) { SNITCH_FAIL_CHECK("failure 3"); }); + r.add({"test SECTION & CAPTURE"}, SNITCH_CURRENT_LOCATION, []() { + int i = 1; + SNITCH_CAPTURE(i); + SNITCH_SECTION("section 1") { + int j = 2; + SNITCH_CAPTURE(j); + SNITCH_FAIL_CHECK("failure 1"); + } + SNITCH_SECTION("section 2") { + int j = 3; + SNITCH_CAPTURE(j); + SNITCH_FAIL_CHECK("failure 2"); + } + SNITCH_FAIL_CHECK("failure 3"); + }); + r.add({"test SKIP in SECTION"}, SNITCH_CURRENT_LOCATION, []() { SNITCH_SECTION("section 1") { SNITCH_SECTION("section 2") { From 8add4699428ed99f3cd979cb858acbf58672ca8b Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Fri, 27 Sep 2024 21:33:20 +0100 Subject: [PATCH 18/20] Remove redundant section information in TeamCity reporter --- src/snitch_reporter_teamcity.cpp | 5 +- .../data/expected/reporter_teamcity_allfail | 4 +- .../data/expected/reporter_teamcity_default | 46 +++++++++---------- .../data/expected/reporter_teamcity_full | 46 +++++++++---------- 4 files changed, 49 insertions(+), 52 deletions(-) diff --git a/src/snitch_reporter_teamcity.cpp b/src/snitch_reporter_teamcity.cpp index 573a8b40..af276bca 100644 --- a/src/snitch_reporter_teamcity.cpp +++ b/src/snitch_reporter_teamcity.cpp @@ -44,11 +44,8 @@ void print_assertion(const registry& r, const assertion& msg) noexcept { small_string buffer; r.print("'", make_escaped(buffer, msg.location.file), ":", msg.location.line, "|n"); - for (const auto& s : msg.sections) { - r.print(make_escaped(buffer, s.id.name), "|n"); - } for (const auto& c : msg.captures) { - r.print(make_escaped(buffer, c), "|n"); + r.print("with ", make_escaped(buffer, c), "|n"); } constexpr std::string_view indent = " "; diff --git a/tests/approval_tests/data/expected/reporter_teamcity_allfail b/tests/approval_tests/data/expected/reporter_teamcity_allfail index 2e0020d0..640f9692 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_allfail +++ b/tests/approval_tests/data/expected/reporter_teamcity_allfail @@ -52,7 +52,7 @@ ##teamCity[testStarted name='test unexpected throw in section fail'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n unexpected std::exception caught; message: unexpected error'] +##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 2'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in section fail' duration='*'] @@ -61,7 +61,7 @@ ##teamCity[testFinished name='test unexpected throw in check fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check & section fail'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|nsection 1|n unexpected std::exception caught; message: unexpected error'] +##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] ##teamCity[testSuiteFinished name='test "* fail*"'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_default b/tests/approval_tests/data/expected/reporter_teamcity_default index 4afc5ad0..8a3168f2 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_default +++ b/tests/approval_tests/data/expected/reporter_teamcity_default @@ -89,7 +89,7 @@ ##teamCity[testStarted name='test unexpected throw in section fail'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n unexpected std::exception caught; message: unexpected error'] +##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 2'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in section fail' duration='*'] @@ -98,72 +98,72 @@ ##teamCity[testFinished name='test unexpected throw in check fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check & section fail'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|nsection 1|n unexpected std::exception caught; message: unexpected error'] +##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] ##teamCity[testFinished name='test SKIP' duration='*'] ##teamCity[testStarted name='test INFO'] -##teamCity[testFailed name='test INFO' message='*testing_reporters.cpp:*|ninfo|n failure'] +##teamCity[testFailed name='test INFO' message='*testing_reporters.cpp:*|nwith info|n failure'] ##teamCity[testFinished name='test INFO' duration='*'] ##teamCity[testStarted name='test multiple INFO'] ##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|n failure 1'] -##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 2'] -##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|ninfo 2|n failure 3'] -##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 4'] +##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 2'] +##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|nwith info 1|nwith info 2|n failure 3'] +##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 4'] ##teamCity[testFinished name='test multiple INFO' duration='*'] ##teamCity[testStarted name='test SECTION'] ##teamCity[blockOpened name='section' description=''] -##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|nsection|n failure'] +##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|n failure'] ##teamCity[blockClosed name='section'] ##teamCity[testFinished name='test SECTION' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 1|n failure 1'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 1'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 2'] ##teamCity[blockOpened name='section 2.1' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.1|n failure 3'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 3'] ##teamCity[blockClosed name='section 2.1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 2'] ##teamCity[blockOpened name='section 2.2' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 4'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 4'] ##teamCity[blockOpened name='section 2.2.1' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|nsection 2.2.1|n failure 5'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 5'] ##teamCity[blockClosed name='section 2.2.1'] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 6'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 6'] ##teamCity[blockClosed name='section 2.2'] ##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[testFinished name='test multiple SECTION' duration='*'] ##teamCity[testStarted name='test SECTION & INFO'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 1|ninfo 1|ninfo 2|n failure 1'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|nwith info 2|n failure 1'] ##teamCity[blockClosed name='section 1'] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 3'] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 2|ninfo 1|ninfo 3|n failure 2'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|nwith info 3|n failure 2'] ##teamCity[blockClosed name='section 2'] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 3'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] ##teamCity[testStarted name='test SECTION & CAPTURE'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 1|ni := 1|nj := 2|n failure 1'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|nwith j := 2|n failure 1'] ##teamCity[blockClosed name='section 1'] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|n failure 3'] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 2|ni := 1|nj := 3|n failure 2'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|nwith j := 3|n failure 2'] ##teamCity[blockClosed name='section 2'] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|n failure 3'] ##teamCity[testFinished name='test SECTION & CAPTURE' duration='*'] ##teamCity[testStarted name='test SKIP in SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n stopping here'] +##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|n stopping here'] ##teamCity[blockClosed name='section 2'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test SKIP in SECTION' duration='*'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_full b/tests/approval_tests/data/expected/reporter_teamcity_full index 9b1b91f1..cd4999da 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_full +++ b/tests/approval_tests/data/expected/reporter_teamcity_full @@ -126,7 +126,7 @@ ##teamCity[testStarted name='test unexpected throw in section fail'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n unexpected std::exception caught; message: unexpected error'] +##teamCity[testFailed name='test unexpected throw in section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 2'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in section fail' duration='*'] @@ -135,47 +135,47 @@ ##teamCity[testFinished name='test unexpected throw in check fail' duration='*'] ##teamCity[testStarted name='test unexpected throw in check & section fail'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|nsection 1|n unexpected std::exception caught; message: unexpected error'] +##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] ##teamCity[testFinished name='test SKIP' duration='*'] ##teamCity[testStarted name='test INFO'] -##teamCity[testFailed name='test INFO' message='*testing_reporters.cpp:*|ninfo|n failure'] +##teamCity[testFailed name='test INFO' message='*testing_reporters.cpp:*|nwith info|n failure'] ##teamCity[testStdOut name='test INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test INFO' duration='*'] ##teamCity[testStarted name='test multiple INFO'] ##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|n failure 1'] -##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 2'] -##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|ninfo 2|n failure 3'] -##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 4'] +##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 2'] +##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|nwith info 1|nwith info 2|n failure 3'] +##teamCity[testFailed name='test multiple INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 4'] ##teamCity[testStdOut name='test multiple INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test multiple INFO' duration='*'] ##teamCity[testStarted name='test SECTION'] ##teamCity[blockOpened name='section' description=''] -##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|nsection|n failure'] +##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|n failure'] ##teamCity[blockClosed name='section'] ##teamCity[testStdOut name='test SECTION' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 1|n failure 1'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 1'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 2'] ##teamCity[blockOpened name='section 2.1' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.1|n failure 3'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 3'] ##teamCity[blockClosed name='section 2.1'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|n failure 2'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 2'] ##teamCity[blockOpened name='section 2.2' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 4'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 4'] ##teamCity[blockOpened name='section 2.2.1' description=''] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|nsection 2.2.1|n failure 5'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 5'] ##teamCity[blockClosed name='section 2.2.1'] -##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|nsection 2|nsection 2.2|n failure 6'] +##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 6'] ##teamCity[blockClosed name='section 2.2'] ##teamCity[blockClosed name='section 2'] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 7'] @@ -183,30 +183,30 @@ ##teamCity[testFinished name='test multiple SECTION' duration='*'] ##teamCity[testStarted name='test SECTION & INFO'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 1|ninfo 1|ninfo 2|n failure 1'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|nwith info 2|n failure 1'] ##teamCity[blockClosed name='section 1'] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 3'] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nsection 2|ninfo 1|ninfo 3|n failure 2'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|nwith info 3|n failure 2'] ##teamCity[blockClosed name='section 2'] -##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|ninfo 1|n failure 3'] +##teamCity[testFailed name='test SECTION & INFO' message='*testing_reporters.cpp:*|nwith info 1|n failure 3'] ##teamCity[testStdOut name='test SECTION & INFO' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION & INFO' duration='*'] ##teamCity[testStarted name='test SECTION & CAPTURE'] ##teamCity[blockOpened name='section 1' description=''] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 1|ni := 1|nj := 2|n failure 1'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|nwith j := 2|n failure 1'] ##teamCity[blockClosed name='section 1'] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|n failure 3'] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nsection 2|ni := 1|nj := 3|n failure 2'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|nwith j := 3|n failure 2'] ##teamCity[blockClosed name='section 2'] -##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|ni := 1|n failure 3'] +##teamCity[testFailed name='test SECTION & CAPTURE' message='*testing_reporters.cpp:*|nwith i := 1|n failure 3'] ##teamCity[testStdOut name='test SECTION & CAPTURE' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION & CAPTURE' duration='*'] ##teamCity[testStarted name='test SKIP in SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[blockOpened name='section 2' description=''] -##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|nsection 1|nsection 2|n stopping here'] +##teamCity[testIgnored name='test SKIP in SECTION' message='*testing_reporters.cpp:*|n stopping here'] ##teamCity[blockClosed name='section 2'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test SKIP in SECTION' duration='*'] From 46cdd03fda3a9cae5e4d885b9019c304df16ab09 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sat, 28 Sep 2024 10:23:29 +0100 Subject: [PATCH 19/20] Add notify_exception_handled() for explicit clearing of held state; fixes #179 too --- README.md | 8 ++ include/snitch/snitch_macros_exceptions.hpp | 12 ++- include/snitch/snitch_test_data.hpp | 14 ++++ src/snitch_capture.cpp | 4 +- src/snitch_registry.cpp | 13 +--- src/snitch_section.cpp | 4 +- src/snitch_test_data.cpp | 28 ++++++- .../data/expected/reporter_catch2_xml_default | 22 +++++- .../data/expected/reporter_catch2_xml_full | 25 +++++- .../expected/reporter_catch2_xml_list_tests | 18 +++++ .../data/expected/reporter_console_default | 10 ++- .../data/expected/reporter_console_full | 21 ++++- .../data/expected/reporter_console_list_tests | 6 +- .../data/expected/reporter_console_withcolor | 10 ++- .../data/expected/reporter_teamcity_default | 10 +++ .../data/expected/reporter_teamcity_full | 11 +++ .../expected/reporter_teamcity_list_tests | 2 + tests/runtime_tests/capture.cpp | 22 +++++- tests/runtime_tests/section.cpp | 75 +++++++++++++++++- tests/testing_event.cpp | 78 +++++++++++++++++-- tests/testing_event.hpp | 25 ++++-- tests/testing_reporters.cpp | 13 ++++ 22 files changed, 387 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index c60a3905..4e006814 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t - [Run-time and compile-time](#run-time-and-compile-time) - [Exception checks](#exception-checks) - [Miscellaneous](#miscellaneous) + - [Advanced API](#advanced-api) - [Tags](#tags) - [Matchers](#matchers) - [Sections](#sections) @@ -481,6 +482,13 @@ This reports the current test case as "skipped". Any previously reported status This is similar to `SKIP`, except that the test case continues. Further failure will not be reported. This is only recommended as an alternative to `SKIP()` when exceptions cannot be used. +### Advanced API + +`snitch::notify_exception_handled();` + +If handling exceptions explicitly with a `try/catch` block in a test case, this should be called at the end of the `catch` block. This clears up internal state that would have been used to report that exception, had it not been handled. Calling this is not strictly necessary in most cases, but omitting it can lead to confusing contextual data (incorrect section/capture/info) if another exception is thrown afterwards and not handled. + + ### Tags Tags are assigned to each test case using the [Test case macros](#test-case-macros), as a single string. Within this string, individual tags must be surrounded by square brackets, with no white-space between tags (although white space within a tag is allowed). For example: diff --git a/include/snitch/snitch_macros_exceptions.hpp b/include/snitch/snitch_macros_exceptions.hpp index 83400b66..883e20e9 100644 --- a/include/snitch/snitch_macros_exceptions.hpp +++ b/include/snitch/snitch_macros_exceptions.hpp @@ -17,6 +17,7 @@ SNITCH_NO_EXCEPTION_THROWN = true; \ } catch (const __VA_ARGS__&) { \ snitch::registry::report_assertion(true, #__VA_ARGS__ " was thrown as expected"); \ + snitch::notify_exception_handled(); \ } catch (...) { \ try { \ throw; \ @@ -25,12 +26,12 @@ false, \ #__VA_ARGS__ " expected but other std::exception thrown; message: ", \ e.what()); \ - MAYBE_ABORT; \ } catch (...) { \ snitch::registry::report_assertion( \ false, #__VA_ARGS__ " expected but other unknown exception thrown"); \ - MAYBE_ABORT; \ } \ + snitch::notify_exception_handled(); \ + MAYBE_ABORT; \ } \ if (SNITCH_NO_EXCEPTION_THROWN) { \ snitch::registry::report_assertion( \ @@ -58,12 +59,14 @@ false, "could not match caught " #EXCEPTION " with expected content: ", \ SNITCH_TEMP_MATCHER.describe_match( \ e, snitch::matchers::match_status::failed)); \ + snitch::notify_exception_handled(); \ MAYBE_ABORT; \ } else { \ snitch::registry::report_assertion( \ true, "caught " #EXCEPTION " matched expected content: ", \ SNITCH_TEMP_MATCHER.describe_match( \ e, snitch::matchers::match_status::matched)); \ + snitch::notify_exception_handled(); \ } \ } catch (...) { \ try { \ @@ -72,12 +75,12 @@ snitch::registry::report_assertion( \ false, #EXCEPTION " expected but other std::exception thrown; message: ", \ e.what()); \ - MAYBE_ABORT; \ } catch (...) { \ snitch::registry::report_assertion( \ false, #EXCEPTION " expected but other unknown exception thrown"); \ - MAYBE_ABORT; \ } \ + snitch::notify_exception_handled(); \ + MAYBE_ABORT; \ } \ if (SNITCH_NO_EXCEPTION_THROWN) { \ snitch::registry::report_assertion( \ @@ -111,6 +114,7 @@ false, "expected " #__VA_ARGS__ \ " not to throw but it threw an unknown exception"); \ } \ + snitch::notify_exception_handled(); \ MAYBE_ABORT; \ } \ } while (0) diff --git a/include/snitch/snitch_test_data.hpp b/include/snitch/snitch_test_data.hpp index 733216be..e58d7e2d 100644 --- a/include/snitch/snitch_test_data.hpp +++ b/include/snitch/snitch_test_data.hpp @@ -374,4 +374,18 @@ struct scoped_test_check { }; } // namespace snitch::impl +namespace snitch { +#if SNITCH_WITH_EXCEPTIONS +/*! \brief Notify the testing framework that an exception was manually handled. + * \details If handling exceptions explicitly with a `try/catch` block in a test case, + * this should be called at the end of the `catch` block. This clears up internal state + * that would have been used to report that exception, had it not been handled. Calling + * this is not strictly necessary in most cases, but omitting it can lead to confusing + * contextual data (incorrect section/capture/info) if another exception is thrown afterwards + * and not handled. + */ +SNITCH_EXPORT void notify_exception_handled() noexcept; +#endif +} // namespace snitch + #endif diff --git a/src/snitch_capture.cpp b/src/snitch_capture.cpp index 0c406cc8..c98498ec 100644 --- a/src/snitch_capture.cpp +++ b/src/snitch_capture.cpp @@ -92,7 +92,9 @@ small_string& add_capture(test_state& state) { } #if SNITCH_WITH_EXCEPTIONS - state.held_info.reset(); + if (std::uncaught_exceptions() == 0) { + notify_exception_handled(); + } #endif state.info.captures.grow(1); diff --git a/src/snitch_registry.cpp b/src/snitch_registry.cpp index 63ac7cac..36a74562 100644 --- a/src/snitch_registry.cpp +++ b/src/snitch_registry.cpp @@ -456,7 +456,8 @@ void report_assertion_impl( register_assertion(success, state); #if SNITCH_WITH_EXCEPTIONS - const bool use_held_info = state.unhandled_exception && state.held_info.has_value(); + const bool use_held_info = (state.unhandled_exception || std::uncaught_exceptions() > 0) && + state.held_info.has_value(); const auto captures_buffer = impl::make_capture_buffer( use_held_info ? state.held_info.value().captures : state.info.captures); @@ -655,15 +656,7 @@ impl::test_state registry::run(impl::test_case& test) noexcept { } if (state.unhandled_exception) { - auto& current_section = state.held_info.has_value() - ? state.held_info.value().sections.current_section - : state.info.sections.current_section; - - // Close all sections that were left open by the exception. - while (!current_section.empty()) { - registry::report_section_ended(current_section.back()); - current_section.pop_back(); - } + notify_exception_handled(); } state.unhandled_exception = false; diff --git a/src/snitch_section.cpp b/src/snitch_section.cpp index 0f4ccd8b..8cc72db8 100644 --- a/src/snitch_section.cpp +++ b/src/snitch_section.cpp @@ -71,7 +71,9 @@ section_entry_checker::~section_entry_checker() { section_entry_checker::operator bool() { #if SNITCH_WITH_EXCEPTIONS - state.held_info.reset(); + if (std::uncaught_exceptions() == 0) { + notify_exception_handled(); + } #endif auto& sections = state.info.sections; diff --git a/src/snitch_test_data.cpp b/src/snitch_test_data.cpp index c1d90f5c..e37665a2 100644 --- a/src/snitch_test_data.cpp +++ b/src/snitch_test_data.cpp @@ -1,5 +1,7 @@ #include "snitch/snitch_test_data.hpp" +#include "snitch/snitch_registry.hpp" + #if SNITCH_WITH_EXCEPTIONS # include #endif @@ -36,8 +38,11 @@ void pop_location(test_state& test) noexcept { scoped_test_check::scoped_test_check(const source_location& location) noexcept : test(get_current_test()) { + #if SNITCH_WITH_EXCEPTIONS - test.held_info.reset(); + if (std::uncaught_exceptions() == 0) { + notify_exception_handled(); + } #endif push_location(test, {location.file, location.line, location_type::in_check}); @@ -59,3 +64,24 @@ scoped_test_check::~scoped_test_check() noexcept { pop_location(test); } } // namespace snitch::impl + +namespace snitch { +#if SNITCH_WITH_EXCEPTIONS +void notify_exception_handled() noexcept { + auto& state = impl::get_current_test(); + if (!state.held_info.has_value()) { + return; + } + + // Close all sections that were left open by the exception. + auto& current_held_section = state.held_info.value().sections.current_section; + const auto& current_section = state.info.sections.current_section; + while (current_held_section.size() > current_section.size()) { + registry::report_section_ended(current_held_section.back()); + current_held_section.pop_back(); + } + + state.held_info.reset(); +} +#endif +} // namespace snitch diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_default b/tests/approval_tests/data/expected/reporter_catch2_xml_default index 4c619f9f..a842aaf5 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_default +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_default @@ -250,6 +250,15 @@
+ +
+ + unexpected std::exception caught; message: unexpected error + + +
+ + not interesting @@ -301,6 +310,15 @@
+ +
+ + failure + + +
+ +
@@ -451,6 +469,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_full b/tests/approval_tests/data/expected/reporter_catch2_xml_full index b804a355..ca44b684 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_full +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_full @@ -376,6 +376,15 @@
+ +
+ + unexpected std::exception caught; message: unexpected error + + +
+ + not interesting @@ -436,6 +445,18 @@ + +
+ + failure + + +
+ + no exception caught + + +
@@ -595,6 +616,6 @@ - - + + diff --git a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests index 50acafc6..c8af5592 100644 --- a/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests +++ b/tests/approval_tests/data/expected/reporter_catch2_xml_list_tests @@ -324,6 +324,15 @@ * + + test unexpected throw in check & section mayfail + + [!mayfail] + + *testing_reporters.cpp + * + + test SKIP @@ -360,6 +369,15 @@ * + + test SECTION mayfail + + [!mayfail] + + *testing_reporters.cpp + * + + test multiple SECTION diff --git a/tests/approval_tests/data/expected/reporter_console_default b/tests/approval_tests/data/expected/reporter_console_default index e7ab4200..9b26116f 100644 --- a/tests/approval_tests/data/expected/reporter_console_default +++ b/tests/approval_tests/data/expected/reporter_console_default @@ -80,6 +80,10 @@ failed: running test case "test unexpected throw in check & section fail" in section "section 1" somewhere inside check at *testing_reporters.cpp:* unexpected std::exception caught; message: unexpected error +allowed failure: running test case "test unexpected throw in check & section mayfail" + in section "section 1" + somewhere inside check at *testing_reporters.cpp:* + unexpected std::exception caught; message: unexpected error skipped: running test case "test SKIP" at *testing_reporters.cpp:* not interesting @@ -107,6 +111,10 @@ failed: running test case "test SECTION" in section "section" at *testing_reporters.cpp:* failure +allowed failure: running test case "test SECTION mayfail" + in section "section" + at *testing_reporters.cpp:* + failure failed: running test case "test multiple SECTION" in section "section 1" at *testing_reporters.cpp:* @@ -195,4 +203,4 @@ skipped: running test case "test SKIP in SECTION" at *testing_reporters.cpp:* stopping here ========================================== -error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 46 test cases, 94 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_full b/tests/approval_tests/data/expected/reporter_console_full index d96914f7..50a6e87d 100644 --- a/tests/approval_tests/data/expected/reporter_console_full +++ b/tests/approval_tests/data/expected/reporter_console_full @@ -277,6 +277,14 @@ failed: running test case "test unexpected throw in check & section fail" unexpected std::exception caught; message: unexpected error leaving section: section 1 finished: test unexpected throw in check & section fail (*) +starting: test unexpected throw in check & section mayfail at *testing_reporters.cpp:* +entering section: section 1 at *testing_reporters.cpp:* +allowed failure: running test case "test unexpected throw in check & section mayfail" + in section "section 1" + somewhere inside check at *testing_reporters.cpp:* + unexpected std::exception caught; message: unexpected error +leaving section: section 1 +finished: test unexpected throw in check & section mayfail (*) starting: test SKIP at *testing_reporters.cpp:* skipped: running test case "test SKIP" at *testing_reporters.cpp:* @@ -323,6 +331,17 @@ passed: running test case "test SECTION" at *testing_reporters.cpp:* no exception caught finished: test SECTION (*) +starting: test SECTION mayfail at *testing_reporters.cpp:* +entering section: section at *testing_reporters.cpp:* +allowed failure: running test case "test SECTION mayfail" + in section "section" + at *testing_reporters.cpp:* + failure +leaving section: section +passed: running test case "test SECTION mayfail" + at *testing_reporters.cpp:* + no exception caught +finished: test SECTION mayfail (*) starting: test multiple SECTION at *testing_reporters.cpp:* entering section: section 1 at *testing_reporters.cpp:* failed: running test case "test multiple SECTION" @@ -450,4 +469,4 @@ leaving section: section 2 leaving section: section 1 finished: test SKIP in SECTION (*) ========================================== -error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 46 test cases, 94 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_console_list_tests b/tests/approval_tests/data/expected/reporter_console_list_tests index 685f23d4..98ed91d1 100644 --- a/tests/approval_tests/data/expected/reporter_console_list_tests +++ b/tests/approval_tests/data/expected/reporter_console_list_tests @@ -47,12 +47,16 @@ Matching test cases: test unexpected throw in section fail test unexpected throw in check fail test unexpected throw in check & section fail + test unexpected throw in check & section mayfail + [!mayfail] test SKIP test INFO test multiple INFO test SECTION + test SECTION mayfail + [!mayfail] test multiple SECTION test SECTION & INFO test SECTION & CAPTURE test SKIP in SECTION -44 matching test cases +46 matching test cases diff --git a/tests/approval_tests/data/expected/reporter_console_withcolor b/tests/approval_tests/data/expected/reporter_console_withcolor index 55178903..e0a969d2 100644 --- a/tests/approval_tests/data/expected/reporter_console_withcolor +++ b/tests/approval_tests/data/expected/reporter_console_withcolor @@ -80,6 +80,10 @@ in section "section 1" somewhere inside check at *testing_reporters.cpp:* unexpected std::exception caught; message: unexpected error +allowed failure: running test case "test unexpected throw in check & section mayfail" + in section "section 1" + somewhere inside check at *testing_reporters.cpp:* + unexpected std::exception caught; message: unexpected error skipped: running test case "test SKIP" at *testing_reporters.cpp:* not interesting @@ -107,6 +111,10 @@ in section "section" at *testing_reporters.cpp:* failure +allowed failure: running test case "test SECTION mayfail" + in section "section" + at *testing_reporters.cpp:* + failure failed: running test case "test multiple SECTION" in section "section 1" at *testing_reporters.cpp:* @@ -195,4 +203,4 @@ at *testing_reporters.cpp:* stopping here ========================================== -error: some tests failed (26 out of 44 test cases, 91 assertions, 2 test cases skipped, * seconds) +error: some tests failed (26 out of 46 test cases, 94 assertions, 2 test cases skipped, * seconds) diff --git a/tests/approval_tests/data/expected/reporter_teamcity_default b/tests/approval_tests/data/expected/reporter_teamcity_default index 8a3168f2..e56401f4 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_default +++ b/tests/approval_tests/data/expected/reporter_teamcity_default @@ -101,6 +101,11 @@ ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] +##teamCity[testStarted name='test unexpected throw in check & section mayfail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[testStdOut name='test unexpected throw in check & section mayfail' out='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] +##teamCity[testFinished name='test unexpected throw in check & section mayfail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] ##teamCity[testFinished name='test SKIP' duration='*'] @@ -118,6 +123,11 @@ ##teamCity[testFailed name='test SECTION' message='*testing_reporters.cpp:*|n failure'] ##teamCity[blockClosed name='section'] ##teamCity[testFinished name='test SECTION' duration='*'] +##teamCity[testStarted name='test SECTION mayfail'] +##teamCity[blockOpened name='section' description=''] +##teamCity[testStdOut name='test SECTION mayfail' out='*testing_reporters.cpp:*|n failure'] +##teamCity[blockClosed name='section'] +##teamCity[testFinished name='test SECTION mayfail' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 1'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_full b/tests/approval_tests/data/expected/reporter_teamcity_full index cd4999da..a88e33cd 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_full +++ b/tests/approval_tests/data/expected/reporter_teamcity_full @@ -138,6 +138,11 @@ ##teamCity[testFailed name='test unexpected throw in check & section fail' message='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] ##teamCity[blockClosed name='section 1'] ##teamCity[testFinished name='test unexpected throw in check & section fail' duration='*'] +##teamCity[testStarted name='test unexpected throw in check & section mayfail'] +##teamCity[blockOpened name='section 1' description=''] +##teamCity[testStdOut name='test unexpected throw in check & section mayfail' out='*testing_reporters.cpp:*|n unexpected std::exception caught; message: unexpected error'] +##teamCity[blockClosed name='section 1'] +##teamCity[testFinished name='test unexpected throw in check & section mayfail' duration='*'] ##teamCity[testStarted name='test SKIP'] ##teamCity[testIgnored name='test SKIP' message='*testing_reporters.cpp:*|n not interesting'] ##teamCity[testFinished name='test SKIP' duration='*'] @@ -158,6 +163,12 @@ ##teamCity[blockClosed name='section'] ##teamCity[testStdOut name='test SECTION' out='*testing_reporters.cpp:*|n no exception caught'] ##teamCity[testFinished name='test SECTION' duration='*'] +##teamCity[testStarted name='test SECTION mayfail'] +##teamCity[blockOpened name='section' description=''] +##teamCity[testStdOut name='test SECTION mayfail' out='*testing_reporters.cpp:*|n failure'] +##teamCity[blockClosed name='section'] +##teamCity[testStdOut name='test SECTION mayfail' out='*testing_reporters.cpp:*|n no exception caught'] +##teamCity[testFinished name='test SECTION mayfail' duration='*'] ##teamCity[testStarted name='test multiple SECTION'] ##teamCity[blockOpened name='section 1' description=''] ##teamCity[testFailed name='test multiple SECTION' message='*testing_reporters.cpp:*|n failure 1'] diff --git a/tests/approval_tests/data/expected/reporter_teamcity_list_tests b/tests/approval_tests/data/expected/reporter_teamcity_list_tests index c9000a5f..d14c4853 100644 --- a/tests/approval_tests/data/expected/reporter_teamcity_list_tests +++ b/tests/approval_tests/data/expected/reporter_teamcity_list_tests @@ -34,10 +34,12 @@ test unexpected throw fail test unexpected throw in section fail test unexpected throw in check fail test unexpected throw in check & section fail +test unexpected throw in check & section mayfail test SKIP test INFO test multiple INFO test SECTION +test SECTION mayfail test multiple SECTION test SECTION & INFO test SECTION & CAPTURE diff --git a/tests/runtime_tests/capture.cpp b/tests/runtime_tests/capture.cpp index 15ab2bf2..8bceaf06 100644 --- a/tests/runtime_tests/capture.cpp +++ b/tests/runtime_tests/capture.cpp @@ -274,7 +274,7 @@ TEST_CASE("capture", "[test macros]") { CHECK_CAPTURES("j := 2"); } - SECTION("with handled exception then unhandled no capture") { + SECTION("with handled exception then unhandled no capture missing notify") { framework.test_case.func = []() { try { int i = 1; @@ -288,10 +288,26 @@ TEST_CASE("capture", "[test macros]") { framework.run_test(); REQUIRE(framework.get_num_failures() == 1u); - // FIXME: expected nothing - // https://github.com/snitch-org/snitch/issues/179 CHECK_CAPTURES("i := 1"); } + + SECTION("with handled exception then unhandled no capture") { + framework.test_case.func = []() { + try { + int i = 1; + SNITCH_CAPTURE(i); + throw std::runtime_error("bad"); + } catch (...) { + snitch::notify_exception_handled(); + } + + throw std::runtime_error("bad"); + }; + + framework.run_test(); + REQUIRE(framework.get_num_failures() == 1u); + CHECK_NO_CAPTURE; + } #endif } diff --git a/tests/runtime_tests/section.cpp b/tests/runtime_tests/section.cpp index fa545143..dfd57479 100644 --- a/tests/runtime_tests/section.cpp +++ b/tests/runtime_tests/section.cpp @@ -8,6 +8,16 @@ using namespace std::literals; +#if SNITCH_WITH_EXCEPTIONS +struct destructor_asserter { + bool pass = true; + + ~destructor_asserter() { + SNITCH_CHECK(pass); + } +}; +#endif + SNITCH_WARNING_PUSH SNITCH_WARNING_DISABLE_UNREACHABLE @@ -149,6 +159,49 @@ TEST_CASE("section", "[test macros]") { CHECK_SECTIONS("section 1"); CHECK_CASE(snitch::test_case_state::failed, 1u, 1u); } + + SECTION("unexpected throw with destructor assert ok") { + framework.test_case.func = []() { + SNITCH_SECTION("section 1") { + destructor_asserter a{.pass = true}; + SNITCH_SECTION("section 2") { + throw std::runtime_error("no can do"); + } + } + }; + + framework.run_test(); + + REQUIRE(framework.get_num_failures() == 1u); + + CHECK_SECTIONS("section 1", "section 2"); + CHECK_CASE(snitch::test_case_state::failed, 2u, 1u); + } + + SECTION("unexpected throw with destructor assert nok") { + framework.test_case.func = []() { + SNITCH_SECTION("section 1") { + destructor_asserter a{.pass = false}; + SNITCH_SECTION("section 2") { + throw std::runtime_error("no can do"); + } + } + }; + + framework.run_test(); + + // This is what we want: + // REQUIRE(framework.get_num_failures() == 2u); + // CHECK_SECTIONS_FOR_FAILURE(0u, "section 1", "section 2"); // exception + // CHECK_SECTIONS_FOR_FAILURE(1u, "section 1"); // destructor + // CHECK_CASE(snitch::test_case_state::failed, 2u, 2u); + + // This is what we get: + REQUIRE(framework.get_num_failures() == 2u); + CHECK_SECTIONS_FOR_FAILURE(0u, "section 1", "section 2"); // destructor + CHECK_SECTIONS_FOR_FAILURE(1u, "section 1", "section 2"); // exception + CHECK_CASE(snitch::test_case_state::failed, 2u, 2u); + } #endif SECTION("nested sections varying depth") { @@ -418,7 +471,7 @@ TEST_CASE("section", "[test macros]") { CHECK_SECTIONS("section 2"); } - SECTION("with handled exception then unhandled no section") { + SECTION("with handled exception then unhandled no section missing notify") { framework.test_case.func = []() { try { SNITCH_SECTION("section 1") { @@ -432,10 +485,26 @@ TEST_CASE("section", "[test macros]") { framework.run_test(); REQUIRE(framework.get_num_failures() == 1u); - // FIXME: expected nothing - // https://github.com/snitch-org/snitch/issues/179 CHECK_SECTIONS("section 1"); } + + SECTION("with handled exception then unhandled no section") { + framework.test_case.func = []() { + try { + SNITCH_SECTION("section 1") { + throw std::runtime_error("bad"); + } + } catch (...) { + snitch::notify_exception_handled(); + } + + throw std::runtime_error("bad"); + }; + + framework.run_test(); + REQUIRE(framework.get_num_failures() == 1u); + CHECK_NO_SECTION; + } #endif } diff --git a/tests/testing_event.cpp b/tests/testing_event.cpp index 55498e48..8a4bfab6 100644 --- a/tests/testing_event.cpp +++ b/tests/testing_event.cpp @@ -68,7 +68,7 @@ void copy_assertion_data(snitch::small_string_span pool, U& c, const T& e) { } template -std::optional +std::pair, std::size_t> get_nth_event(snitch::small_vector_span events, std::size_t id) { auto iter = events.cbegin(); bool first = true; @@ -87,9 +87,9 @@ get_nth_event(snitch::small_vector_span events, std::s } while (id != 0); if (iter == events.cend()) { - return {}; + return {std::nullopt, events.size()}; } else { - return std::get(*iter); + return {std::get(*iter), iter - events.cbegin()}; } } @@ -211,12 +211,12 @@ owning_event::data deep_copy(snitch::small_string_span pool, const snitch::event std::optional get_failure_event(snitch::small_vector_span events, std::size_t id) { - return get_nth_event(events, id); + return get_nth_event(events, id).first; } std::optional get_success_event(snitch::small_vector_span events, std::size_t id) { - return get_nth_event(events, id); + return get_nth_event(events, id).first; } std::optional get_test_id(const owning_event::data& e) noexcept { @@ -278,12 +278,76 @@ void mock_framework::run_test() { std::optional mock_framework::get_failure_event(std::size_t id) const { - return get_nth_event(events, id); + return get_nth_event(events, id).first; } std::optional mock_framework::get_success_event(std::size_t id) const { - return get_nth_event(events, id); + return get_nth_event(events, id).first; +} + +bool mock_framework::check_balanced_section_events() const { + bool test_case_started = false; + snitch::small_vector sections; + for (const auto& e : events) { + bool good = std::visit( + snitch::overload{ + [&](const owning_event::section_started& s) { + if (!test_case_started) { + return false; + } + sections.push_back(s.id); + return true; + }, + [&](const owning_event::section_ended& s) { + if (!test_case_started) { + return false; + } + if (sections.empty()) { + return false; + } + if (sections.back().name != s.id.name || + sections.back().description != s.id.description) { + return false; + } + + sections.pop_back(); + return true; + }, + [&](const owning_event::test_case_started&) { + test_case_started = true; + return sections.empty(); + }, + [&](const owning_event::test_case_ended&) { + test_case_started = false; + return sections.empty(); + }, + [](const auto&) { return true; }}, + e); + + if (!good) { + return false; + } + } + + return sections.empty(); +} + +snitch::small_vector +mock_framework::get_sections_for_failure_event(std::size_t id) const { + auto [event, pos] = get_nth_event(events, id); + + snitch::small_vector sections; + for (std::size_t i = 0; i < pos; ++i) { + std::visit( + snitch::overload{ + [&](const owning_event::section_started& s) { sections.push_back(s.id.name); }, + [&](const owning_event::section_ended&) { sections.pop_back(); }, + [](const auto&) {}}, + events[i]); + } + + return sections; } std::size_t mock_framework::get_num_registered_tests() const { diff --git a/tests/testing_event.hpp b/tests/testing_event.hpp index 4012803f..358fb869 100644 --- a/tests/testing_event.hpp +++ b/tests/testing_event.hpp @@ -59,12 +59,12 @@ struct section_started { }; struct section_ended { - snitch::section_id id = {}; - snitch::source_location location = {}; - bool skipped = false; - std::size_t assertion_count = 0; - std::size_t assertion_failure_count = 0; - std::size_t allowed_assertion_failure_count = 0; + snitch::section_id id = {}; + snitch::source_location location = {}; + bool skipped = false; + std::size_t assertion_count = 0; + std::size_t assertion_failure_count = 0; + std::size_t allowed_assertion_failure_count = 0; #if SNITCH_WITH_TIMINGS float duration = 0.0f; #endif @@ -217,6 +217,11 @@ struct mock_framework { std::optional get_failure_event(std::size_t id = 0) const; std::optional get_success_event(std::size_t id = 0) const; + bool check_balanced_section_events() const; + + snitch::small_vector + get_sections_for_failure_event(std::size_t id = 0) const; + std::size_t get_num_registered_tests() const; std::size_t get_num_runs() const; std::size_t get_num_failures() const; @@ -386,16 +391,21 @@ struct has_expr_data { #define CHECK_SECTIONS_FOR_FAILURE(FAILURE_ID, ...) \ do { \ + INFO("failure ID: ", FAILURE_ID); \ + REQUIRE(framework.check_balanced_section_events()); \ auto failure = framework.get_failure_event(FAILURE_ID); \ REQUIRE(failure.has_value()); \ const char* EXPECTED_SECTIONS[] = {__VA_ARGS__}; \ REQUIRE( \ failure.value().sections.size() == sizeof(EXPECTED_SECTIONS) / sizeof(const char*)); \ + const auto section_from_events = framework.get_sections_for_failure_event(FAILURE_ID); \ + REQUIRE(section_from_events.size() == sizeof(EXPECTED_SECTIONS) / sizeof(const char*)); \ std::size_t SECTION_INDEX = 0; \ for (std::string_view SECTION_NAME : EXPECTED_SECTIONS) { \ CHECK( \ failure.value().sections[SECTION_INDEX].id.name == \ std::string_view{SECTION_NAME}); \ + CHECK(section_from_events[SECTION_INDEX] == std::string_view{SECTION_NAME}); \ ++SECTION_INDEX; \ } \ } while (0) @@ -404,9 +414,12 @@ struct has_expr_data { #define CHECK_NO_SECTION_FOR_FAILURE(FAILURE_ID) \ do { \ + REQUIRE(framework.check_balanced_section_events()); \ auto failure = framework.get_failure_event(FAILURE_ID); \ REQUIRE(failure.has_value()); \ CHECK(failure.value().sections.empty()); \ + const auto section_from_events = framework.get_sections_for_failure_event(FAILURE_ID); \ + CHECK(section_from_events.empty()); \ } while (0) #define CHECK_NO_SECTION CHECK_NO_SECTION_FOR_FAILURE(0u) diff --git a/tests/testing_reporters.cpp b/tests/testing_reporters.cpp index 467e3bb2..afce1e78 100644 --- a/tests/testing_reporters.cpp +++ b/tests/testing_reporters.cpp @@ -136,6 +136,13 @@ void register_tests_for_reporters(snitch::registry& r) { SNITCH_CHECK(throw_unexpectedly() == 0); } }); + r.add( + {"test unexpected throw in check & section mayfail", "[!mayfail]"}, SNITCH_CURRENT_LOCATION, + []() { + SNITCH_SECTION("section 1") { + SNITCH_CHECK(throw_unexpectedly() == 0); + } + }); #endif r.add({"test SKIP"}, SNITCH_CURRENT_LOCATION, []() { SNITCH_SKIP("not interesting"); }); @@ -162,6 +169,12 @@ void register_tests_for_reporters(snitch::registry& r) { } }); + r.add({"test SECTION mayfail", "[!mayfail]"}, SNITCH_CURRENT_LOCATION, []() { + SNITCH_SECTION("section") { + SNITCH_FAIL_CHECK("failure"); + } + }); + r.add({"test multiple SECTION"}, SNITCH_CURRENT_LOCATION, []() { SNITCH_SECTION("section 1") { SNITCH_FAIL_CHECK("failure 1"); From c8bc7e06d289867d624acca3a4201e0ce6170a16 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 29 Sep 2024 20:53:48 +0100 Subject: [PATCH 20/20] Update benchmarks --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4e006814..43816f77 100644 --- a/README.md +++ b/README.md @@ -247,22 +247,22 @@ Results for Debug builds: | **Debug** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ | |-----------------|----------|----------|-----------|------------| -| Build framework | 4.0s | 42s | 2.1s | 0s | -| Build tests | 71s | 75s | 76s | 117s | -| Build all | 75s | 117s | 78s | 117s | +| Build framework | 4.2s | 42s | 2.1s | 0s | +| Build tests | 70s | 75s | 76s | 117s | +| Build all | 74s | 117s | 78s | 117s | | Run tests | 44ms | 67ms | 63ms | 14ms | -| Library size | 8.3MB | 33.5MB | 2.8MB | 0MB | -| Executable size | 36.4MB | 47.7MB | 38.6MB | 51.8MB | +| Library size | 9.2MB | 33.5MB | 2.8MB | 0MB | +| Executable size | 37.0MB | 47.7MB | 38.6MB | 51.8MB | Results for Release builds: | **Release** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ | |-----------------|----------|----------|-----------|------------| -| Build framework | 5.5s | 48s | 3.7s | 0s | +| Build framework | 5.7s | 48s | 3.7s | 0s | | Build tests | 146s | 233s | 210s | 289s | | Build all | 152s | 281s | 214s | 289s | | Run tests | 26ms | 37ms | 42ms | 5ms | -| Library size | 1.3MB | 2.5MB | 0.39MB | 0MB | +| Library size | 1.4MB | 2.5MB | 0.39MB | 0MB | | Executable size | 10.2MB | 17.4MB | 15.5MB | 11.4MB | Notes: