Skip to content

Commit

Permalink
Merge pull request #183 from snitch-org/cschreib/catch2-compat
Browse files Browse the repository at this point in the history 
Fix Catch2 XML format compatibility
  • Loading branch information
@cschreib
cschreib authored Oct 15, 2024
2 parents 2a1808f + c8bc7e0 commit 74bdd8e
Show file tree
Hide file tree
Showing 38 changed files with 1,297 additions and 300 deletions.
4 changes: 3 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ set(SNITCH_INCLUDES
${PROJECT_SOURCE_DIR}/include/snitch/snitch_string.hpp
${PROJECT_SOURCE_DIR}/include/snitch/snitch_string_utility.hpp
${PROJECT_SOURCE_DIR}/include/snitch/snitch_test_data.hpp
${PROJECT_SOURCE_DIR}/include/snitch/snitch_time.hpp
${PROJECT_SOURCE_DIR}/include/snitch/snitch_type_id.hpp
${PROJECT_SOURCE_DIR}/include/snitch/snitch_type_name.hpp
${PROJECT_SOURCE_DIR}/include/snitch/snitch_vector.hpp
Expand All @@ -111,7 +112,8 @@ set(SNITCH_SOURCES_INDIVIDUAL
${PROJECT_SOURCE_DIR}/src/snitch_reporter_teamcity.cpp
${PROJECT_SOURCE_DIR}/src/snitch_section.cpp
${PROJECT_SOURCE_DIR}/src/snitch_string_utility.cpp
${PROJECT_SOURCE_DIR}/src/snitch_test_data.cpp)
${PROJECT_SOURCE_DIR}/src/snitch_test_data.cpp
${PROJECT_SOURCE_DIR}/src/snitch_time.cpp)

if (SNITCH_UNITY_BUILD)
set(SNITCH_SOURCES ${PROJECT_SOURCE_DIR}/src/snitch.cpp)
Expand Down
22 changes: 15 additions & 7 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t
- [Run-time and compile-time](#run-time-and-compile-time)
- [Exception checks](#exception-checks)
- [Miscellaneous](#miscellaneous)
- [Advanced API](#advanced-api)
- [Tags](#tags)
- [Matchers](#matchers)
- [Sections](#sections)
Expand Down Expand Up @@ -246,22 +247,22 @@ Results for Debug builds:

| **Debug** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ |
|-----------------|----------|----------|-----------|------------|
| Build framework | 4.0s | 42s | 2.1s | 0s |
| Build tests | 71s | 75s | 76s | 117s |
| Build all | 75s | 117s | 78s | 117s |
| Build framework | 4.2s | 42s | 2.1s | 0s |
| Build tests | 70s | 75s | 76s | 117s |
| Build all | 74s | 117s | 78s | 117s |
| Run tests | 44ms | 67ms | 63ms | 14ms |
| Library size | 8.3MB | 33.5MB | 2.8MB | 0MB |
| Executable size | 36.4MB | 47.7MB | 38.6MB | 51.8MB |
| Library size | 9.2MB | 33.5MB | 2.8MB | 0MB |
| Executable size | 37.0MB | 47.7MB | 38.6MB | 51.8MB |

Results for Release builds:

| **Release** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ |
|-----------------|----------|----------|-----------|------------|
| Build framework | 5.5s | 48s | 3.7s | 0s |
| Build framework | 5.7s | 48s | 3.7s | 0s |
| Build tests | 146s | 233s | 210s | 289s |
| Build all | 152s | 281s | 214s | 289s |
| Run tests | 26ms | 37ms | 42ms | 5ms |
| Library size | 1.3MB | 2.5MB | 0.39MB | 0MB |
| Library size | 1.4MB | 2.5MB | 0.39MB | 0MB |
| Executable size | 10.2MB | 17.4MB | 15.5MB | 11.4MB |

Notes:
Expand Down Expand Up @@ -481,6 +482,13 @@ This reports the current test case as "skipped". Any previously reported status
This is similar to `SKIP`, except that the test case continues. Further failure will not be reported. This is only recommended as an alternative to `SKIP()` when exceptions cannot be used.


### Advanced API

`snitch::notify_exception_handled();`

If handling exceptions explicitly with a `try/catch` block in a test case, this should be called at the end of the `catch` block. This clears up internal state that would have been used to report that exception, had it not been handled. Calling this is not strictly necessary in most cases, but omitting it can lead to confusing contextual data (incorrect section/capture/info) if another exception is thrown afterwards and not handled.


### Tags

Tags are assigned to each test case using the [Test case macros](#test-case-macros), as a single string. Within this string, individual tags must be surrounded by square brackets, with no white-space between tags (although white space within a tag is allowed). For example:
Expand Down
12 changes: 8 additions & 4 deletions include/snitch/snitch_macros_exceptions.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
SNITCH_NO_EXCEPTION_THROWN = true; \
} catch (const __VA_ARGS__&) { \
snitch::registry::report_assertion(true, #__VA_ARGS__ " was thrown as expected"); \
snitch::notify_exception_handled(); \
} catch (...) { \
try { \
throw; \
Expand All @@ -25,12 +26,12 @@
false, \
#__VA_ARGS__ " expected but other std::exception thrown; message: ", \
e.what()); \
MAYBE_ABORT; \
} catch (...) { \
snitch::registry::report_assertion( \
false, #__VA_ARGS__ " expected but other unknown exception thrown"); \
MAYBE_ABORT; \
} \
snitch::notify_exception_handled(); \
MAYBE_ABORT; \
} \
if (SNITCH_NO_EXCEPTION_THROWN) { \
snitch::registry::report_assertion( \
Expand Down Expand Up @@ -58,12 +59,14 @@
false, "could not match caught " #EXCEPTION " with expected content: ", \
SNITCH_TEMP_MATCHER.describe_match( \
e, snitch::matchers::match_status::failed)); \
snitch::notify_exception_handled(); \
MAYBE_ABORT; \
} else { \
snitch::registry::report_assertion( \
true, "caught " #EXCEPTION " matched expected content: ", \
SNITCH_TEMP_MATCHER.describe_match( \
e, snitch::matchers::match_status::matched)); \
snitch::notify_exception_handled(); \
} \
} catch (...) { \
try { \
Expand All @@ -72,12 +75,12 @@
snitch::registry::report_assertion( \
false, #EXCEPTION " expected but other std::exception thrown; message: ", \
e.what()); \
MAYBE_ABORT; \
} catch (...) { \
snitch::registry::report_assertion( \
false, #EXCEPTION " expected but other unknown exception thrown"); \
MAYBE_ABORT; \
} \
snitch::notify_exception_handled(); \
MAYBE_ABORT; \
} \
if (SNITCH_NO_EXCEPTION_THROWN) { \
snitch::registry::report_assertion( \
Expand Down Expand Up @@ -111,6 +114,7 @@
false, "expected " #__VA_ARGS__ \
" not to throw but it threw an unknown exception"); \
} \
snitch::notify_exception_handled(); \
MAYBE_ABORT; \
} \
} while (0)
Expand Down
2 changes: 1 addition & 1 deletion include/snitch/snitch_macros_misc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

#define SNITCH_SECTION(...) \
if (snitch::impl::section_entry_checker SNITCH_MACRO_CONCAT(section_id_, __COUNTER__){ \
{{__VA_ARGS__}, SNITCH_CURRENT_LOCATION}, snitch::impl::get_current_test()})
{__VA_ARGS__}, SNITCH_CURRENT_LOCATION, snitch::impl::get_current_test()})

#define SNITCH_CAPTURE(...) \
auto SNITCH_MACRO_CONCAT(capture_id_, __COUNTER__) = \
Expand Down
6 changes: 6 additions & 0 deletions include/snitch/snitch_registry.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,12 @@ class registry {
// Internal API; do not use.
SNITCH_EXPORT static void report_skipped(std::string_view message) noexcept;

// Internal API; do not use.
SNITCH_EXPORT static void report_section_started(const section& sec) noexcept;

// Internal API; do not use.
SNITCH_EXPORT static void report_section_ended(const section& sec) noexcept;

// Internal API; do not use.
SNITCH_EXPORT impl::test_state run(impl::test_case& test) noexcept;

Expand Down
7 changes: 4 additions & 3 deletions include/snitch/snitch_section.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@

namespace snitch::impl {
struct section_entry_checker {
section data = {};
test_state& state;
bool entered = false;
section_id id = {};
source_location location = {};
test_state& state;
bool entered = false;

SNITCH_EXPORT ~section_entry_checker();

Expand Down
58 changes: 58 additions & 0 deletions include/snitch/snitch_test_data.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include "snitch/snitch_config.hpp"
#include "snitch/snitch_string.hpp"
#include "snitch/snitch_time.hpp"
#include "snitch/snitch_vector.hpp"

#include <cstddef>
Expand Down Expand Up @@ -46,6 +47,18 @@ struct section {
section_id id = {};
/// Location (file, line)
source_location location = {};

/// Counts all assertions; passed, failed, or allowed failed
std::size_t assertion_count = 0;
/// Counts failed assertions
std::size_t assertion_failure_count = 0;
/// Counts allowed failed assertions (e.g., [!shouldfail] and [!mayfail])
std::size_t allowed_assertion_failure_count = 0;

#if SNITCH_WITH_TIMINGS
/// Time index of the instant when the section was first entered.
time_point_t start_time = 0;
#endif
};

/// List of test case filters
Expand Down Expand Up @@ -177,6 +190,35 @@ struct test_case_ended {
bool failure_allowed = false;
};

struct section_started {
/// Identifiers (name, description)
section_id id = {};
/// Location (file, line)
source_location location = {};
};

struct section_ended {
/// Identifiers (name, description)
section_id id = {};
/// Location (file, line)
source_location location = {};

/// Whether the section has been skipped.
bool skipped = false;

/// Counts all assertions; passed, failed, or allowed failed
std::size_t assertion_count = 0;
/// Counts failed assertions
std::size_t assertion_failure_count = 0;
/// Counts allowed failed assertions (e.g., [!shouldfail] and [!mayfail])
std::size_t allowed_assertion_failure_count = 0;

#if SNITCH_WITH_TIMINGS
/// Section duration, in seconds
float duration = 0.0f;
#endif
};

struct assertion_failed {
const test_id& id;
section_info sections = {};
Expand Down Expand Up @@ -232,6 +274,8 @@ using data = std::variant<
test_run_ended,
test_case_started,
test_case_ended,
section_started,
section_ended,
assertion_failed,
assertion_succeeded,
test_case_skipped,
Expand Down Expand Up @@ -330,4 +374,18 @@ struct scoped_test_check {
};
} // namespace snitch::impl

namespace snitch {
#if SNITCH_WITH_EXCEPTIONS
/*! \brief Notify the testing framework that an exception was manually handled.
* \details If handling exceptions explicitly with a `try/catch` block in a test case,
* this should be called at the end of the `catch` block. This clears up internal state
* that would have been used to report that exception, had it not been handled. Calling
* this is not strictly necessary in most cases, but omitting it can lead to confusing
* contextual data (incorrect section/capture/info) if another exception is thrown afterwards
* and not handled.
*/
SNITCH_EXPORT void notify_exception_handled() noexcept;
#endif
} // namespace snitch

#endif
17 changes: 17 additions & 0 deletions include/snitch/snitch_time.hpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#ifndef SNITCH_TIME_HPP
#define SNITCH_TIME_HPP

#include "snitch/snitch_config.hpp"

#if SNITCH_WITH_TIMINGS

namespace snitch {
using time_point_t = std::size_t;

SNITCH_EXPORT time_point_t get_current_time() noexcept;

SNITCH_EXPORT float get_duration_in_seconds(time_point_t start, time_point_t end) noexcept;
} // namespace snitch

#endif
#endif
1 change: 1 addition & 0 deletions src/snitch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@
#include "snitch_section.cpp"
#include "snitch_string_utility.cpp"
#include "snitch_test_data.cpp"
#include "snitch_time.cpp"
4 changes: 3 additions & 1 deletion src/snitch_capture.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,9 @@ small_string<max_capture_length>& add_capture(test_state& state) {
}

#if SNITCH_WITH_EXCEPTIONS
state.held_info.reset();
if (std::uncaught_exceptions() == 0) {
notify_exception_handled();
}
#endif

state.info.captures.grow(1);
Expand Down
Loading

0 comments on commit 74bdd8e

Please sign in to comment.