From dc47ff8f7fb2d07dd80ba009215b8efbef1a57ff Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Tue, 10 Jan 2023 21:52:20 +0000 Subject: [PATCH 01/17] Added is_match --- include/snitch/snitch.hpp | 2 + src/snitch.cpp | 50 +++++++++ tests/runtime_tests/string_utility.cpp | 149 +++++++++++++++++++++++++ 3 files changed, 201 insertions(+) diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 9406de5d..30cec831 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -580,6 +580,8 @@ bool append_or_truncate(small_string_span ss, Args&&... args) noexcept { [[nodiscard]] bool replace_all( small_string_span string, std::string_view pattern, std::string_view replacement) noexcept; +[[nodiscard]] bool is_match(std::string_view string, std::string_view regex) noexcept; + template concept matcher_for = requires(const T& m, const U& value) { { m.match(value) } -> convertible_to; diff --git a/src/snitch.cpp b/src/snitch.cpp index cb5ef8f7..8a75c9f1 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -218,6 +218,56 @@ bool replace_all( return !overflow; } } + +bool is_match(std::string_view string, std::string_view regex) noexcept { + // An empty regex matches any string; early exit. + // An empty string matches an empty regex (exit here) or any regex containing + // only wildcards (exit later). + if ((string.empty() && regex.empty()) || regex.empty()) { + return true; + } + + const std::size_t regex_size = regex.size(); + const std::size_t string_size = string.size(); + + // Iterate characters of the regex string and exit at first non-match. + for (std::size_t j = 0; j < regex_size; ++j) { + if (regex[j] == '*') { + // Wildcard is found; if this is the last character of the regex + // then any further content will be a match; early exit. + if (j == regex_size - 1) { + return true; + } + + // Discard what has already been matched. If there are no more characters in the + // string after discarding, then we only match if the regex contains only + // wildcards from there on. + regex = regex.substr(j + 1); + const std::size_t remaining = string_size >= j ? string_size - j : 0u; + if (remaining == 0u) { + return regex.find_first_not_of('*') == regex.npos; + } + + // Otherwise, we loop over all remaining characters of the string and look + // for a match when starting from each of them. + for (std::size_t o = 0; o < remaining; ++o) { + if (is_match(string.substr(j + o), regex)) { + return true; + } + } + + return false; + } else if (j >= string_size || regex[j] != string[j]) { + // Regular character is found; not a match if not an exact match in the string. + return false; + } + } + + // We have finished reading the regex string and did not find either a definite non-match + // or a definite match. This means we did not have any wildcard left, hence that we need + // an exact match. Therefore, only match if the string size is the same as the regex. + return regex_size == string_size; +} } // namespace snitch namespace snitch::impl { diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index 9827d0b6..4f09542d 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -355,3 +355,152 @@ TEMPLATE_TEST_CASE( CHECK(std::string_view(s) == "abaca"); } } + +TEST_CASE("is_match", "[utility]") { + SECTION("empty") { + CHECK(snitch::is_match(""sv, ""sv)); + } + + SECTION("empty regex") { + CHECK(snitch::is_match("abc"sv, ""sv)); + } + + SECTION("empty string") { + CHECK(!snitch::is_match(""sv, "abc"sv)); + } + + SECTION("no wildcard match") { + CHECK(snitch::is_match("abc"sv, "abc"sv)); + } + + SECTION("no wildcard not match") { + CHECK(!snitch::is_match("abc"sv, "cba"sv)); + } + + SECTION("no wildcard not match smaller regex") { + CHECK(!snitch::is_match("abc"sv, "ab"sv)); + CHECK(!snitch::is_match("abc"sv, "bc"sv)); + CHECK(!snitch::is_match("abc"sv, "a"sv)); + CHECK(!snitch::is_match("abc"sv, "b"sv)); + CHECK(!snitch::is_match("abc"sv, "c"sv)); + } + + SECTION("no wildcard not match larger regex") { + CHECK(!snitch::is_match("abc"sv, "abcd"sv)); + CHECK(!snitch::is_match("abc"sv, "zabc"sv)); + CHECK(!snitch::is_match("abc"sv, "abcdefghijkl"sv)); + } + + SECTION("single wildcard match") { + CHECK(snitch::is_match("abc"sv, "*"sv)); + CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "*"sv)); + CHECK(snitch::is_match(""sv, "*"sv)); + } + + SECTION("start wildcard match") { + CHECK(snitch::is_match("abc"sv, "*bc"sv)); + CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "*bc"sv)); + CHECK(snitch::is_match("bc"sv, "*bc"sv)); + } + + SECTION("start wildcard not match") { + CHECK(!snitch::is_match("abd"sv, "*bc"sv)); + CHECK(!snitch::is_match("azzzzzzzzzzbd"sv, "*bc"sv)); + CHECK(!snitch::is_match("bd"sv, "*bc"sv)); + } + + SECTION("end wildcard match") { + CHECK(snitch::is_match("abc"sv, "ab*"sv)); + CHECK(snitch::is_match("abccccccccccc"sv, "ab*"sv)); + CHECK(snitch::is_match("ab"sv, "ab*"sv)); + } + + SECTION("end wildcard not match") { + CHECK(!snitch::is_match("adc"sv, "ab*"sv)); + CHECK(!snitch::is_match("adccccccccccc"sv, "ab*"sv)); + CHECK(!snitch::is_match("ad"sv, "ab*"sv)); + } + + SECTION("mid wildcard match") { + CHECK(snitch::is_match("ab_cd"sv, "ab*cd"sv)); + CHECK(snitch::is_match("abasdasdasdcd"sv, "ab*cd"sv)); + CHECK(snitch::is_match("abcd"sv, "ab*cd"sv)); + } + + SECTION("mid wildcard not match") { + CHECK(!snitch::is_match("adcd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("abcc"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("accccccccccd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("ab"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("abc"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("abd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("cd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("bcd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("acd"sv, "ab*cd"sv)); + } + + SECTION("multi wildcard match") { + CHECK(snitch::is_match("zab_cdw"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("zzzzzzabcccccccccccdwwwwwww"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("ab_cdw"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("zabcdw"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("zab_cd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("ababcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcdabcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcdabcc"sv, "*ab*cd*"sv)); + } + + SECTION("multi wildcard not match") { + CHECK(!snitch::is_match("zad_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zac_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zaa_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zdb_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zcb_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zbb_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_ddw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_bdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_adw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_ccw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_cbw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_caw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("ab_"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("ab"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("cd"sv, "*ab*cd*"sv)); + } + + SECTION("double wildcard match") { + CHECK(snitch::is_match("abc"sv, "**"sv)); + CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "**"sv)); + CHECK(snitch::is_match(""sv, "**"sv)); + CHECK(snitch::is_match("abc"sv, "abc**"sv)); + CHECK(snitch::is_match("abc"sv, "ab**"sv)); + CHECK(snitch::is_match("abc"sv, "a**"sv)); + CHECK(snitch::is_match("abc"sv, "**abc"sv)); + CHECK(snitch::is_match("abc"sv, "**bc"sv)); + CHECK(snitch::is_match("abc"sv, "**c"sv)); + CHECK(snitch::is_match("abc"sv, "ab**c"sv)); + CHECK(snitch::is_match("abc"sv, "a**bc"sv)); + CHECK(snitch::is_match("abc"sv, "a**c"sv)); + } + + SECTION("double wildcard not match") { + CHECK(!snitch::is_match("abc"sv, "abd**"sv)); + CHECK(!snitch::is_match("abc"sv, "ad**"sv)); + CHECK(!snitch::is_match("abc"sv, "d**"sv)); + CHECK(!snitch::is_match("abc"sv, "**abd"sv)); + CHECK(!snitch::is_match("abc"sv, "**bd"sv)); + CHECK(!snitch::is_match("abc"sv, "**d"sv)); + CHECK(!snitch::is_match("abc"sv, "ab**d"sv)); + CHECK(!snitch::is_match("abc"sv, "a**d"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**abc"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**ab"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**a"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**def"sv)); + } +} From 78fd23cd6962edf43caa43056ddecf2a808f6106 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Tue, 10 Jan 2023 21:52:54 +0000 Subject: [PATCH 02/17] Added is_filter_match --- include/snitch/snitch.hpp | 2 ++ src/snitch.cpp | 10 ++++++++++ tests/runtime_tests/string_utility.cpp | 11 +++++++++++ 3 files changed, 23 insertions(+) diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 30cec831..5bc43493 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -582,6 +582,8 @@ bool append_or_truncate(small_string_span ss, Args&&... args) noexcept { [[nodiscard]] bool is_match(std::string_view string, std::string_view regex) noexcept; +[[nodiscard]] bool is_filter_match(std::string_view name, std::string_view filter) noexcept; + template concept matcher_for = requires(const T& m, const U& value) { { m.match(value) } -> convertible_to; diff --git a/src/snitch.cpp b/src/snitch.cpp index 8a75c9f1..d5034b10 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -268,6 +268,16 @@ bool is_match(std::string_view string, std::string_view regex) noexcept { // an exact match. Therefore, only match if the string size is the same as the regex. return regex_size == string_size; } + +bool is_filter_match(std::string_view name, std::string_view filter) noexcept { + bool expected = true; + if (filter.size() > 1 && filter[0] == '~') { + filter = filter.substr(1); + expected = false; + } + + return is_match(name, filter) == expected; +} } // namespace snitch namespace snitch::impl { diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index 4f09542d..22466441 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -504,3 +504,14 @@ TEST_CASE("is_match", "[utility]") { CHECK(!snitch::is_match("abc"sv, "abc**def"sv)); } } + +TEST_CASE("is_filter_match", "[utility]") { + CHECK(snitch::is_filter_match("abc"sv, "abc"sv)); + CHECK(snitch::is_filter_match("abc"sv, "ab*"sv)); + CHECK(snitch::is_filter_match("abc"sv, "*bc"sv)); + CHECK(snitch::is_filter_match("abc"sv, "*"sv)); + CHECK(!snitch::is_filter_match("abc"sv, "~abc"sv)); + CHECK(!snitch::is_filter_match("abc"sv, "~ab*"sv)); + CHECK(!snitch::is_filter_match("abc"sv, "~*bc"sv)); + CHECK(!snitch::is_filter_match("abc"sv, "~*"sv)); +} From 37fdd3d6ffeb8aff2d1486bd3ae7d8a1ca42a362 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Tue, 10 Jan 2023 21:54:22 +0000 Subject: [PATCH 03/17] Use regex for filtering tests --- README.md | 18 +++++++++++++++++- src/snitch.cpp | 5 +---- tests/runtime_tests/registry.cpp | 8 ++++---- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index daa95a2d..184ad00b 100644 --- a/README.md +++ b/README.md @@ -653,7 +653,7 @@ An example reporter for _Teamcity_ is included for demonstration, see `include/s ### Default main function The default `main()` function provided in _snitch_ offers the following command-line API: - - positional argument for filtering tests by name. + - positional argument for filtering tests by name, see below. - `-h,--help`: show command line help. - `-l,--list-tests`: list all tests. - ` --list-tags`: list all tags. @@ -662,6 +662,22 @@ The default `main()` function provided in _snitch_ offers the following command- - `-v,--verbosity [quiet|normal|high]`: select level of detail for the default reporter. - ` --color [always|never]`: enable/disable colors in the default reporter. +The positional argument is used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If a filter is provided, then hidden tests will no longer be excluded. This reproduces the behavior of _Catch2_. + +A filter may contain any number of "wildcard" character, `*`, which can represent zero or more characters. For example: + - `ab*` will match all test cases with names starting with `ab`. + - `*cd` will match all test cases with names ending with `cd`. + - `ab*cd` will match all test cases with names starting with `ab` and ending with `cd`. + - `abcd` will only match the test case with name `abcd`. + - `*` will match all test cases. + + If the filter starts with `~`, then it is negated: + - `~ab*` will match all test cases with names NOT starting with `ab`. + - `~*cd` will match all test cases with names NOT ending with `cd`. + - `~ab*cd` will match all test cases with names NOT starting with `ab` or NOT ending with `cd`. + - `~abcd` will match all test cases except the test case with name `abcd`. + - `~*` will match no test case. + ### Using your own main function diff --git a/src/snitch.cpp b/src/snitch.cpp index d5034b10..7f146116 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -1074,10 +1074,7 @@ bool registry::run_tests_matching_name( std::string_view run_name, std::string_view name_filter) noexcept { small_string buffer; return ::run_tests(*this, run_name, [&](const test_case& t) { - std::string_view v = make_full_name(buffer, t.id); - - // TODO: use regex here? - return v.find(name_filter) != v.npos; + return is_filter_match(make_full_name(buffer, t.id), name_filter); }); } diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index e71e4c44..90cea086 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -526,7 +526,7 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all pass") { - framework.registry.run_tests_matching_name("test_app", "are you"); + framework.registry.run_tests_matching_name("test_app", "*are you"); CHECK(test_called); CHECK(!test_called_other_tag); @@ -547,7 +547,7 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all failed") { - framework.registry.run_tests_matching_name("test_app", "lights"); + framework.registry.run_tests_matching_name("test_app", "*lights*"); CHECK(!test_called); CHECK(test_called_other_tag); @@ -568,7 +568,7 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all skipped") { - framework.registry.run_tests_matching_name("test_app", "cup"); + framework.registry.run_tests_matching_name("test_app", "*cup"); CHECK(!test_called); CHECK(!test_called_other_tag); @@ -872,7 +872,7 @@ TEST_CASE("run tests cli", "[registry]") { } SECTION("test filter") { - const arg_vector args = {"test", "how many"}; + const arg_vector args = {"test", "how many*"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); framework.registry.run_tests(*input); From 8ff16bec9b3efa43881fed44c5a6cba3228c3b08 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Wed, 11 Jan 2023 09:37:59 +0000 Subject: [PATCH 04/17] Allow wildcards in tag filters --- CMakeLists.txt | 2 + README.md | 10 ++--- include/snitch/snitch.hpp | 2 + include/snitch/snitch_config.hpp.config | 3 ++ src/snitch.cpp | 56 ++++++++----------------- tests/runtime_tests/registry.cpp | 37 ++++++++++++++-- tests/testing_event.hpp | 2 +- 7 files changed, 64 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 48ad9975..7c5b18af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ set(SNITCH_MAX_NESTED_SECTIONS 8 CACHE STRING "Maximum depth of nested sec set(SNITCH_MAX_EXPR_LENGTH 1024 CACHE STRING "Maximum length of a printed expression when reporting failure.") set(SNITCH_MAX_MESSAGE_LENGTH 1024 CACHE STRING "Maximum length of error or status messages.") set(SNITCH_MAX_TEST_NAME_LENGTH 1024 CACHE STRING "Maximum length of a test case name.") +set(SNITCH_MAX_TAG_LENGTH 256 CACHE STRING "Maximum length of a test tag.") set(SNITCH_MAX_CAPTURES 8 CACHE STRING "Maximum number of captured expressions in a test case.") set(SNITCH_MAX_CAPTURE_LENGTH 256 CACHE STRING "Maximum length of a captured expression.") set(SNITCH_MAX_UNIQUE_TAGS 1024 CACHE STRING "Maximum number of unique tags in a test application.") @@ -59,6 +60,7 @@ if (SNITCH_CREATE_LIBRARY) SNITCH_MAX_EXPR_LENGTH=${SNITCH_MAX_EXPR_LENGTH} SNITCH_MAX_MESSAGE_LENGTH=${SNITCH_MAX_MESSAGE_LENGTH} SNITCH_MAX_TEST_NAME_LENGTH=${SNITCH_MAX_TEST_NAME_LENGTH} + SNITCH_MAX_TAG_LENGTH=${SNITCH_MAX_TAG_LENGTH} SNITCH_MAX_UNIQUE_TAGS=${SNITCH_MAX_UNIQUE_TAGS} SNITCH_MAX_COMMAND_LINE_ARGS=${SNITCH_MAX_COMMAND_LINE_ARGS} SNITCH_DEFINE_MAIN=$ diff --git a/README.md b/README.md index 184ad00b..18ebb2ac 100644 --- a/README.md +++ b/README.md @@ -658,11 +658,11 @@ The default `main()` function provided in _snitch_ offers the following command- - `-l,--list-tests`: list all tests. - ` --list-tags`: list all tags. - ` --list-tests-with-tag`: list all tests with a given tag. - - `-t,--tags`: filter tests by tags instead of by name. - - `-v,--verbosity [quiet|normal|high]`: select level of detail for the default reporter. - - ` --color [always|never]`: enable/disable colors in the default reporter. + - `-t,--tags `: filter tests by tags, see below. + - `-v,--verbosity `: select level of detail for the default reporter. + - ` --color `: enable/disable colors in the default reporter. -The positional argument is used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If a filter is provided, then hidden tests will no longer be excluded. This reproduces the behavior of _Catch2_. +The positional argument is used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If a filter is provided, then hidden tests will no longer be excluded. This reproduces the behavior of _Catch2_. If the `-t` (or `--tags`) option is given, the filtering is applied to tags instead of test case names. A filter may contain any number of "wildcard" character, `*`, which can represent zero or more characters. For example: - `ab*` will match all test cases with names starting with `ab`. @@ -671,7 +671,7 @@ A filter may contain any number of "wildcard" character, `*`, which can represen - `abcd` will only match the test case with name `abcd`. - `*` will match all test cases. - If the filter starts with `~`, then it is negated: +If the filter starts with `~`, then it is negated: - `~ab*` will match all test cases with names NOT starting with `ab`. - `~*cd` will match all test cases with names NOT ending with `cd`. - `~ab*cd` will match all test cases with names NOT starting with `ab` or NOT ending with `cd`. diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 5bc43493..3a7701d4 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -33,6 +33,8 @@ constexpr std::size_t max_message_length = SNITCH_MAX_MESSAGE_LENGTH; // Maximum length of a full test case name. // The full test case name includes the base name, plus any type. constexpr std::size_t max_test_name_length = SNITCH_MAX_TEST_NAME_LENGTH; +// Maximum length of a tag, including brackets. +constexpr std::size_t max_tag_length = SNITCH_MAX_TAG_LENGTH; // Maximum number of captured expressions in a test case. constexpr std::size_t max_captures = SNITCH_MAX_CAPTURES; // Maximum length of a captured expression. diff --git a/include/snitch/snitch_config.hpp.config b/include/snitch/snitch_config.hpp.config index a69aea39..6f9dac9a 100644 --- a/include/snitch/snitch_config.hpp.config +++ b/include/snitch/snitch_config.hpp.config @@ -24,6 +24,9 @@ #if !defined(SNITCH_MAX_TEST_NAME_LENGTH) # define SNITCH_MAX_TEST_NAME_LENGTH ${SNITCH_MAX_TEST_NAME_LENGTH} #endif +#if !defined(SNITCH_MAX_TAG_LENGTH) +# define SNITCH_MAX_TAG_LENGTH ${SNITCH_MAX_TAG_LENGTH} +#endif #if !defined(SNITCH_MAX_CAPTURES) # define SNITCH_MAX_CAPTURES ${SNITCH_MAX_CAPTURES} #endif diff --git a/src/snitch.cpp b/src/snitch.cpp index 7f146116..29752bcf 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -538,14 +538,6 @@ void for_each_raw_tag(std::string_view s, F&& callback) noexcept { callback(s.substr(last_pos)); } -std::string_view get_tag_name(std::string_view tag) { - if (tag.size() < 2u || tag[0] != '[' || tag[tag.size() - 1u] != ']') { - return {}; - } - - return tag.substr(1u, tag.size() - 2u); -} - namespace tags { struct ignored {}; struct may_fail {}; @@ -556,32 +548,34 @@ using parsed_tag = std::variant void for_each_tag(std::string_view s, F&& callback) noexcept { + small_string buffer; + for_each_raw_tag(s, [&](std::string_view t) { // Look for "ignore" tags, which is either "[.]" // or a a tag starting with ".", like "[.integration]". if (t == "[.]"sv) { // This is a pure "ignore" tag, add this to the list of special tags. callback(tags::parsed_tag{tags::ignored{}}); - return; + } else if (t.starts_with("[."sv)) { + // This is a combined "ignore" + normal tag, add the "ignore" to the list of special + // tags, and continue with the normal tag. + callback(tags::parsed_tag{tags::ignored{}}); + callback(tags::parsed_tag{std::string_view("[.]")}); + + buffer.clear(); + if (!append(buffer, "[", t.substr(2u))) { + terminate_with("tag is too long"); + } + + t = buffer; } if (t == "[!mayfail]") { callback(tags::parsed_tag{tags::may_fail{}}); - return; } if (t == "[!shouldfail]") { callback(tags::parsed_tag{tags::should_fail{}}); - return; - } - - if (t.starts_with("[."sv)) { - // This is a combined "ignore" + normal tag, add the "ignore" to the list of special - // tags, and continue with the normal tag. - callback(tags::parsed_tag{tags::ignored{}}); - t = t.substr(2u, t.size() - 3u); - } else { - t = t.substr(1u, t.size() - 2u); } callback(tags::parsed_tag(t)); @@ -1079,18 +1073,11 @@ bool registry::run_tests_matching_name( } bool registry::run_tests_with_tag(std::string_view run_name, std::string_view tag_filter) noexcept { - tag_filter = get_tag_name(tag_filter); - if (tag_filter.empty()) { - print( - make_colored("error:", with_color, color::fail), - " tag must be of the form '[tag_name]'."); - std::terminate(); - } - return ::run_tests(*this, run_name, [&](const test_case& t) { bool selected = false; for_each_tag(t.id.tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); vs != nullptr && *vs == tag_filter) { + if (auto* vs = std::get_if(&v); + vs != nullptr && is_filter_match(*vs, tag_filter)) { selected = true; } }); @@ -1132,18 +1119,11 @@ void registry::list_all_tests() const noexcept { } void registry::list_tests_with_tag(std::string_view tag) const noexcept { - tag = get_tag_name(tag); - if (tag.empty()) { - print( - make_colored("error:", with_color, color::fail), - " tag must be of the form '[tag_name]'."); - std::terminate(); - } - list_tests(*this, [&](const test_case& t) { bool selected = false; for_each_tag(t.id.tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); vs != nullptr && *vs == tag) { + if (auto* vs = std::get_if(&v); + vs != nullptr && is_filter_match(*vs, tag)) { selected = true; } }); diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index 90cea086..f79fb4c3 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -610,6 +610,27 @@ TEST_CASE("run tests", "[registry]") { } } + SECTION("run tests filtered tags wildcard") { + framework.registry.run_tests_with_tag("test_app", "*tag]"); + + CHECK(test_called); + CHECK(test_called_other_tag); + CHECK(test_called_skipped); + CHECK(test_called_int); + CHECK(test_called_float); + CHECK(test_called_hidden1); + CHECK(!test_called_hidden2); + + if (r == reporter::print) { + CHECK( + framework.messages == + contains_substring("some tests failed (3 out of 6 test cases, 3 assertions")); + } else { + CHECK(framework.get_num_runs() == 6u); + CHECK_RUN(false, 6u, 3u, 1u, 3u); + } + } + SECTION("run tests special tag [.]") { framework.registry.run_tests_with_tag("test_app", "[hidden]"); @@ -697,14 +718,16 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("[other_tag]")); CHECK(framework.messages == contains_substring("[tag with spaces]")); CHECK(framework.messages == contains_substring("[hidden]")); - CHECK(framework.messages != contains_substring("[.]")); + CHECK(framework.messages == contains_substring("[.]")); CHECK(framework.messages != contains_substring("[.hidden]")); + CHECK(framework.messages == contains_substring("[!shouldfail]")); + CHECK(framework.messages == contains_substring("[!mayfail]")); } SECTION("list_tests_with_tag") { for (auto tag : {"[tag]"sv, "[other_tag]"sv, "[skipped]"sv, "[tag with spaces]"sv, "[wrong_tag]"sv, - "[hidden]"sv, "[.]"sv, "[.hidden]"sv}) { + "[hidden]"sv, "[.]"sv, "[.hidden]"sv, "*tag]"sv}) { CAPTURE(tag); framework.messages.clear(); @@ -738,10 +761,16 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("how many templated lights [int]")); CHECK( framework.messages == contains_substring("how many templated lights [float]")); - } else if (tag == "[hidden]"sv) { + } else if (tag == "[hidden]"sv || tag == "[.]"sv) { CHECK(framework.messages == contains_substring("hidden test 1")); CHECK(framework.messages == contains_substring("hidden test 2")); - } else if (tag == "[wrong_tag]"sv || tag == "[.]"sv || tag == "[.hidden]"sv) { + } else if (tag == "*tag]"sv) { + CHECK(framework.messages == contains_substring("how are you")); + CHECK(framework.messages == contains_substring("how many lights")); + CHECK(framework.messages == contains_substring("drink from the cup")); + CHECK(framework.messages == contains_substring("how many templated lights")); + CHECK(framework.messages == contains_substring("hidden test 1")); + } else if (tag == "[wrong_tag]"sv || tag == "[.hidden]"sv) { CHECK(framework.messages.empty()); } } diff --git a/tests/testing_event.hpp b/tests/testing_event.hpp index 7f8b5079..1156afc5 100644 --- a/tests/testing_event.hpp +++ b/tests/testing_event.hpp @@ -49,7 +49,7 @@ struct mock_framework { .func = nullptr, .state = snitch::impl::test_case_state::not_run}; - snitch::small_vector events; + snitch::small_vector events; snitch::small_string<4086> messages; void report(const snitch::registry&, const snitch::event::data& e) noexcept; From 6a71dff8be0a5f13d2289b4454b01cf6e8b621f8 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Thu, 12 Jan 2023 08:57:17 +0000 Subject: [PATCH 05/17] Allow repeatable CLI arguments --- src/snitch.cpp | 72 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/src/snitch.cpp b/src/snitch.cpp index 29752bcf..28ba26ac 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -1163,13 +1163,15 @@ using namespace std::literals; constexpr std::size_t max_arg_names = 2; -enum class argument_type { optional, mandatory }; +namespace argument_type { +enum type { optional = 0b00, mandatory = 0b01, repeatable = 0b10 }; +} struct expected_argument { small_vector names; std::optional value_name; std::string_view description; - argument_type type = argument_type::optional; + argument_type::type type = argument_type::optional; }; using expected_arguments = small_vector; @@ -1189,6 +1191,26 @@ std::string_view extract_executable(std::string_view path) { return path; } +bool is_option(const expected_argument& e) { + return !e.names.empty(); +} + +bool is_option(const cli::argument& a) { + return !a.name.empty(); +} + +bool has_value(const expected_argument& e) { + return e.value_name.has_value(); +} + +bool is_mandatory(const expected_argument& e) { + return (e.type & argument_type::mandatory) != 0; +} + +bool is_repeatable(const expected_argument& e) { + return (e.type & argument_type::repeatable) != 0; +} + std::optional parse_arguments( int argc, const char* const argv[], @@ -1205,7 +1227,8 @@ std::optional parse_arguments( small_vector expected_found; for (const auto& e : expected) { expected_found.push_back(false); - if (!e.names.empty()) { + + if (is_option(e)) { if (e.names.size() == 1) { if (!e.names[0].starts_with('-')) { terminate_with("option name must start with '-' or '--'"); @@ -1216,7 +1239,7 @@ std::optional parse_arguments( } } } else { - if (!e.value_name.has_value()) { + if (!has_value(e)) { terminate_with("positional argument must have a value name"); } } @@ -1225,12 +1248,15 @@ std::optional parse_arguments( // Parse for (int argi = 1; argi < argc; ++argi) { std::string_view arg(argv[argi]); + if (arg.starts_with('-')) { + // Options start with dashes. bool found = false; + for (std::size_t arg_index = 0; arg_index < expected.size(); ++arg_index) { const auto& e = expected[arg_index]; - if (e.names.empty()) { + if (!is_option(e)) { continue; } @@ -1240,7 +1266,7 @@ std::optional parse_arguments( found = true; - if (expected_found[arg_index]) { + if (expected_found[arg_index] && !is_repeatable(e)) { console_print( make_colored("error:", settings.with_color, color::error), " duplicate command line argument '", arg, "'\n"); @@ -1250,7 +1276,7 @@ std::optional parse_arguments( expected_found[arg_index] = true; - if (e.value_name) { + if (has_value(e)) { if (argi + 1 == argc) { console_print( make_colored("error:", settings.with_color, color::error), @@ -1276,11 +1302,17 @@ std::optional parse_arguments( " unknown command line argument '", arg, "'\n"); } } else { + // If no dash, this is a positional argument. bool found = false; + for (std::size_t arg_index = 0; arg_index < expected.size(); ++arg_index) { const auto& e = expected[arg_index]; - if (!e.names.empty() || expected_found[arg_index]) { + if (is_option(e)) { + continue; + } + + if (expected_found[arg_index] && !is_repeatable(e)) { continue; } @@ -1302,8 +1334,8 @@ std::optional parse_arguments( for (std::size_t arg_index = 0; arg_index < expected.size(); ++arg_index) { const auto& e = expected[arg_index]; - if (e.type == argument_type::mandatory && !expected_found[arg_index]) { - if (e.names.empty()) { + if (!expected_found[arg_index] && is_mandatory(e)) { + if (!is_option(e)) { console_print( make_colored("error:", settings.with_color, color::error), " missing positional argument '<", *e.value_name, ">'\n"); @@ -1339,16 +1371,22 @@ void print_help( // Print command line usage example console_print(make_colored("Usage:", settings.with_color, color::pass), "\n"); console_print(" ", program_name); - if (std::any_of(expected.cbegin(), expected.cend(), [](auto& e) { return !e.names.empty(); })) { + if (std::any_of(expected.cbegin(), expected.cend(), [](auto& e) { return is_option(e); })) { console_print(" [options...]"); } for (const auto& e : expected) { - if (e.names.empty()) { - if (e.type == argument_type::mandatory) { + if (!is_option(e)) { + if (!is_mandatory(e) && !is_repeatable(e)) { + console_print(" [<", *e.value_name, ">]"); + } else if (is_mandatory(e) && !is_repeatable(e)) { console_print(" <", *e.value_name, ">"); + } else if (!is_mandatory(e) && is_repeatable(e)) { + console_print(" [<", *e.value_name, ">...]"); + } else if (is_mandatory(e) && is_repeatable(e)) { + console_print(" <", *e.value_name, ">..."); } else { - console_print(" [<", *e.value_name, ">]"); + terminate_with("unhandled argument type"); } } } @@ -1361,7 +1399,7 @@ void print_help( heading.clear(); bool success = true; - if (!e.names.empty()) { + if (is_option(e)) { if (e.names[0].starts_with("--")) { success = success && append(heading, " "); } @@ -1372,7 +1410,7 @@ void print_help( success = success && append(heading, ", ", e.names[1]); } - if (e.value_name) { + if (has_value(e)) { success = success && append(heading, " <", *e.value_name, ">"); } } else { @@ -1438,7 +1476,7 @@ get_positional_argument(const cli::input& args, std::string_view name) noexcept std::optional ret; auto iter = std::find_if(args.arguments.cbegin(), args.arguments.cend(), [&](const auto& arg) { - return arg.name.empty() && arg.value_name == name; + return !is_option(arg) && arg.value_name == name; }); if (iter != args.arguments.cend()) { From 21a1d17183e2114686b05424261603e17f79ffbf Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sat, 14 Jan 2023 10:11:19 +0000 Subject: [PATCH 06/17] Use instead of [type] in full name --- README.md | 2 +- src/snitch.cpp | 29 ++++++++++++------------- tests/runtime_tests/registry.cpp | 36 ++++++++++++++++---------------- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 18ebb2ac..9fd46038 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ This must be called at namespace, global, or class scope; not inside a function `TEMPLATE_TEST_CASE(NAME, TAGS, TYPES...) { /* test code for TestType */ }` -This is similar to `TEST_CASE`, except that it declares a new test case for each of the types listed in `TYPES...`. Within the test body, the current type can be accessed as `TestType`. If you tend to reuse the same list of types for multiple test cases, then `TEMPLATE_LIST_TEST_CASE()` is recommended instead. +This is similar to `TEST_CASE`, except that it declares a new test case for each of the types listed in `TYPES...`. Within the test body, the current type can be accessed as `TestType`. The full name of the test, used when filtering tests by name, is `"NAME "`. If you tend to reuse the same list of types for multiple test cases, then `TEMPLATE_LIST_TEST_CASE()` is recommended instead. `TEMPLATE_LIST_TEST_CASE(NAME, TAGS, TYPES) { /* test code for TestType */ }` diff --git a/src/snitch.cpp b/src/snitch.cpp index 28ba26ac..b3c2beb5 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -688,26 +688,11 @@ bool run_tests(registry& r, std::string_view run_name, F&& predicate) noexcept { return success; } -template -void list_tests(const registry& r, F&& predicate) noexcept { - for (const test_case& t : r) { - if (!predicate(t)) { - continue; - } - - if (!t.id.type.empty()) { - r.print(t.id.name, " [", t.id.type, "]\n"); - } else { - r.print(t.id.name, "\n"); - } - } -} - std::string_view make_full_name(small_string& buffer, const test_id& id) noexcept { buffer.clear(); if (id.type.length() != 0) { - if (!append(buffer, id.name, " [", id.type, "]")) { + if (!append(buffer, id.name, " <", id.type, ">")) { return {}; } } else { @@ -719,6 +704,18 @@ make_full_name(small_string& buffer, const test_id& id) no return buffer.str(); } +template +void list_tests(const registry& r, F&& predicate) noexcept { + small_string buffer; + for (const test_case& t : r) { + if (!predicate(t)) { + continue; + } + + r.print(make_full_name(buffer, t.id), "\n"); + } +} + void set_state(test_case& t, impl::test_case_state s) noexcept { if (static_cast>(t.state) < static_cast>(s)) { diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index f79fb4c3..cc2e668e 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -146,8 +146,8 @@ TEST_CASE("add template test", "[registry]") { CHECK(test_called == false); CHECK(test_called_int == true); CHECK(test_called_float == false); - CHECK(framework.messages == contains_substring("starting: how many lights [int]")); - CHECK(framework.messages == contains_substring("finished: how many lights [int]")); + CHECK(framework.messages == contains_substring("starting: how many lights ")); + CHECK(framework.messages == contains_substring("finished: how many lights ")); } SECTION("run float default reporter") { @@ -157,8 +157,8 @@ TEST_CASE("add template test", "[registry]") { CHECK(test_called == false); CHECK(test_called_int == false); CHECK(test_called_float == true); - CHECK(framework.messages == contains_substring("starting: how many lights [float]")); - CHECK(framework.messages == contains_substring("finished: how many lights [float]")); + CHECK(framework.messages == contains_substring("starting: how many lights ")); + CHECK(framework.messages == contains_substring("finished: how many lights ")); } SECTION("run int custom reporter") { @@ -704,8 +704,8 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); - CHECK(framework.messages == contains_substring("how many templated lights [float]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); + CHECK(framework.messages == contains_substring("how many templated lights ")); CHECK(framework.messages == contains_substring("hidden test 1")); CHECK(framework.messages == contains_substring("hidden test 2")); } @@ -737,30 +737,30 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); CHECK( - framework.messages == contains_substring("how many templated lights [float]")); + framework.messages == contains_substring("how many templated lights ")); } else if (tag == "[other_tag]"sv) { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages != contains_substring("drink from the cup")); - CHECK(framework.messages != contains_substring("how many templated lights [int]")); + CHECK(framework.messages != contains_substring("how many templated lights ")); CHECK( - framework.messages != contains_substring("how many templated lights [float]")); + framework.messages != contains_substring("how many templated lights ")); } else if (tag == "[skipped]"sv) { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages != contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages != contains_substring("how many templated lights [int]")); + CHECK(framework.messages != contains_substring("how many templated lights ")); CHECK( - framework.messages != contains_substring("how many templated lights [float]")); + framework.messages != contains_substring("how many templated lights ")); } else if (tag == "[tag with spaces]"sv) { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages != contains_substring("how many lights")); CHECK(framework.messages != contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); CHECK( - framework.messages == contains_substring("how many templated lights [float]")); + framework.messages == contains_substring("how many templated lights ")); } else if (tag == "[hidden]"sv || tag == "[.]"sv) { CHECK(framework.messages == contains_substring("hidden test 1")); CHECK(framework.messages == contains_substring("hidden test 2")); @@ -871,8 +871,8 @@ TEST_CASE("run tests cli", "[registry]") { CHECK(framework.messages == contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); - CHECK(framework.messages == contains_substring("how many templated lights [float]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); + CHECK(framework.messages == contains_substring("how many templated lights ")); } SECTION("--list-tags") { @@ -896,8 +896,8 @@ TEST_CASE("run tests cli", "[registry]") { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages != contains_substring("drink from the cup")); - CHECK(framework.messages != contains_substring("how many templated lights [int]")); - CHECK(framework.messages != contains_substring("how many templated lights [float]")); + CHECK(framework.messages != contains_substring("how many templated lights ")); + CHECK(framework.messages != contains_substring("how many templated lights ")); } SECTION("test filter") { From f9104a1a960ce60d229eb3efe3085d7993481bf1 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sat, 14 Jan 2023 17:43:32 +0000 Subject: [PATCH 07/17] Support escaped characters in is_match --- README.md | 9 +++++++ src/snitch.cpp | 36 ++++++++++++++++++-------- tests/runtime_tests/string_utility.cpp | 12 +++++++++ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9fd46038..d30fc8c0 100644 --- a/README.md +++ b/README.md @@ -678,6 +678,15 @@ If the filter starts with `~`, then it is negated: - `~abcd` will match all test cases except the test case with name `abcd`. - `~*` will match no test case. +**Note:** To match the actual character `*` in a test name, the `*` in the filter must be escaped using a backslash `\`, for example `\*`. Be mindful that most shells (Bash, etc.) will also require the backslash itself be escaped to be interpreted as an actual backslash in _snitch_: + +| Bash | _snitch_ | matches | +|---------|----------|--------------------------------------------| +| `\\` | `\` | nothing (ill-formed filter) | +| `\\*` | `\*` | exactly the `*` character | +| `\\\\` | `\\` | exactly the `\` character | +| `\\\\*` | `\\*` | any string starting with the `\` character | + ### Using your own main function diff --git a/src/snitch.cpp b/src/snitch.cpp index b3c2beb5..4cfe498a 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -231,19 +231,33 @@ bool is_match(std::string_view string, std::string_view regex) noexcept { const std::size_t string_size = string.size(); // Iterate characters of the regex string and exit at first non-match. - for (std::size_t j = 0; j < regex_size; ++j) { - if (regex[j] == '*') { + std::size_t js = 0; + for (std::size_t jr = 0; jr < regex_size; ++jr, ++js) { + bool escaped = false; + if (regex[jr] == '\\') { + // Escaped character, look ahead ignoring special characters. + ++jr; + if (jr >= regex_size) { + // Nothing left to escape; the regex is ill-formed. + return false; + } + + escaped = true; + } + + if (!escaped && regex[jr] == '*') { // Wildcard is found; if this is the last character of the regex // then any further content will be a match; early exit. - if (j == regex_size - 1) { + if (jr == regex_size - 1) { return true; } - // Discard what has already been matched. If there are no more characters in the - // string after discarding, then we only match if the regex contains only - // wildcards from there on. - regex = regex.substr(j + 1); - const std::size_t remaining = string_size >= j ? string_size - j : 0u; + // Discard what has already been matched. + regex = regex.substr(jr + 1); + + // If there are no more characters in the string after discarding, then we only match if + // the regex contains only wildcards from there on. + const std::size_t remaining = string_size >= js ? string_size - js : 0u; if (remaining == 0u) { return regex.find_first_not_of('*') == regex.npos; } @@ -251,13 +265,13 @@ bool is_match(std::string_view string, std::string_view regex) noexcept { // Otherwise, we loop over all remaining characters of the string and look // for a match when starting from each of them. for (std::size_t o = 0; o < remaining; ++o) { - if (is_match(string.substr(j + o), regex)) { + if (is_match(string.substr(js + o), regex)) { return true; } } return false; - } else if (j >= string_size || regex[j] != string[j]) { + } else if (js >= string_size || regex[jr] != string[js]) { // Regular character is found; not a match if not an exact match in the string. return false; } @@ -266,7 +280,7 @@ bool is_match(std::string_view string, std::string_view regex) noexcept { // We have finished reading the regex string and did not find either a definite non-match // or a definite match. This means we did not have any wildcard left, hence that we need // an exact match. Therefore, only match if the string size is the same as the regex. - return regex_size == string_size; + return js == string_size; } bool is_filter_match(std::string_view name, std::string_view filter) noexcept { diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index 22466441..4e96f7de 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -503,6 +503,18 @@ TEST_CASE("is_match", "[utility]") { CHECK(!snitch::is_match("abc"sv, "abc**a"sv)); CHECK(!snitch::is_match("abc"sv, "abc**def"sv)); } + + SECTION("string contains wildcard & escaped wildcard") { + CHECK(snitch::is_match("a*c"sv, "a\\*c"sv)); + CHECK(snitch::is_match("a*"sv, "a\\*"sv)); + CHECK(snitch::is_match("*a"sv, "\\*a"sv)); + CHECK(snitch::is_match("a*"sv, "a*"sv)); + CHECK(snitch::is_match("a\\b"sv, "a\\\\b"sv)); + CHECK(snitch::is_match("a"sv, "\\a"sv)); + CHECK(!snitch::is_match("a"sv, "a\\"sv)); + CHECK(!snitch::is_match("a"sv, "a\\\\"sv)); + CHECK(!snitch::is_match("a"sv, "\\\\a"sv)); + } } TEST_CASE("is_filter_match", "[utility]") { From d2a5927d9faacb49f34d8f2c47ed53b69f9954f1 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sat, 14 Jan 2023 17:45:16 +0000 Subject: [PATCH 08/17] Added for_each_positional_argument --- include/snitch/snitch.hpp | 5 +++++ src/snitch.cpp | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 3a7701d4..64c0e65c 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -1165,6 +1165,11 @@ std::optional get_option(const cli::input& args, std::string_view std::optional get_positional_argument(const cli::input& args, std::string_view name) noexcept; + +void for_each_positional_argument( + const cli::input& args, + std::string_view name, + const small_function& callback) noexcept; } // namespace snitch::cli // Test registry. diff --git a/src/snitch.cpp b/src/snitch.cpp index 4cfe498a..0e403fa5 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -1496,6 +1496,24 @@ get_positional_argument(const cli::input& args, std::string_view name) noexcept return ret; } + +void for_each_positional_argument( + const cli::input& args, + std::string_view name, + const small_function& callback) noexcept { + + auto iter = args.arguments.cbegin(); + while (iter != args.arguments.cend()) { + iter = std::find_if(iter, args.arguments.cend(), [&](const auto& arg) { + return !is_option(arg) && arg.value_name == name; + }); + + if (iter != args.arguments.cend()) { + callback(*iter->value); + ++iter; + } + } +} } // namespace snitch::cli namespace snitch { From f460740d184fc9942c7365dee6dc52facd9db454 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sat, 14 Jan 2023 17:47:43 +0000 Subject: [PATCH 09/17] Added is_filter_match_* --- include/snitch/snitch.hpp | 6 ++- src/snitch.cpp | 40 ++++++++++++++----- tests/runtime_tests/string_utility.cpp | 53 ++++++++++++++++++++++---- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 64c0e65c..349b8106 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -584,7 +584,11 @@ bool append_or_truncate(small_string_span ss, Args&&... args) noexcept { [[nodiscard]] bool is_match(std::string_view string, std::string_view regex) noexcept; -[[nodiscard]] bool is_filter_match(std::string_view name, std::string_view filter) noexcept; +[[nodiscard]] bool is_filter_match_name(std::string_view name, std::string_view filter) noexcept; + +[[nodiscard]] bool is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept; + +[[nodiscard]] bool is_filter_match_id(const test_id& id, std::string_view filter) noexcept; template concept matcher_for = requires(const T& m, const U& value) { diff --git a/src/snitch.cpp b/src/snitch.cpp index 0e403fa5..2010d38d 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -282,16 +282,6 @@ bool is_match(std::string_view string, std::string_view regex) noexcept { // an exact match. Therefore, only match if the string size is the same as the regex. return js == string_size; } - -bool is_filter_match(std::string_view name, std::string_view filter) noexcept { - bool expected = true; - if (filter.size() > 1 && filter[0] == '~') { - filter = filter.substr(1); - expected = false; - } - - return is_match(name, filter) == expected; -} } // namespace snitch namespace snitch::impl { @@ -757,6 +747,36 @@ small_vector make_capture_buffer(const capture_s } // namespace namespace snitch { +bool is_filter_match_name(std::string_view name, std::string_view filter) noexcept { + bool expected = true; + if (filter.size() > 1 && filter[0] == '~') { + filter = filter.substr(1); + expected = false; + } + + return is_match(name, filter) == expected; +} + +bool is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept { + bool selected = false; + for_each_tag(tags, [&](const tags::parsed_tag& v) { + if (auto* vs = std::get_if(&v); + vs != nullptr && is_filter_match_name(*vs, filter)) { + selected = true; + } + }); + + return selected; +} + +bool is_filter_match_id(const test_id& id, std::string_view filter) noexcept { + if (filter.starts_with("[") || filter.starts_with("~[")) { + return is_filter_match_tags(id.tags, filter); + } else { + return is_filter_match_name(id.name, filter); + } +} + const char* registry::add(const test_id& id, test_ptr func) noexcept { if (test_list.size() == test_list.capacity()) { print( diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index 4e96f7de..40785ba3 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -518,12 +518,49 @@ TEST_CASE("is_match", "[utility]") { } TEST_CASE("is_filter_match", "[utility]") { - CHECK(snitch::is_filter_match("abc"sv, "abc"sv)); - CHECK(snitch::is_filter_match("abc"sv, "ab*"sv)); - CHECK(snitch::is_filter_match("abc"sv, "*bc"sv)); - CHECK(snitch::is_filter_match("abc"sv, "*"sv)); - CHECK(!snitch::is_filter_match("abc"sv, "~abc"sv)); - CHECK(!snitch::is_filter_match("abc"sv, "~ab*"sv)); - CHECK(!snitch::is_filter_match("abc"sv, "~*bc"sv)); - CHECK(!snitch::is_filter_match("abc"sv, "~*"sv)); + CHECK(snitch::is_filter_match_name("abc"sv, "abc"sv)); + CHECK(snitch::is_filter_match_name("abc"sv, "ab*"sv)); + CHECK(snitch::is_filter_match_name("abc"sv, "*bc"sv)); + CHECK(snitch::is_filter_match_name("abc"sv, "*"sv)); + CHECK(!snitch::is_filter_match_name("abc"sv, "~abc"sv)); + CHECK(!snitch::is_filter_match_name("abc"sv, "~ab*"sv)); + CHECK(!snitch::is_filter_match_name("abc"sv, "~*bc"sv)); + CHECK(!snitch::is_filter_match_name("abc"sv, "~*"sv)); +} + +TEST_CASE("is_filter_match_tag", "[utility]") { + CHECK(snitch::is_filter_match_tags("[tag1]"sv, "[tag1]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag1]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag2]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[tug*]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2][.]"sv, "[.]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][.tag2]"sv, "[.]"sv)); + CHECK(snitch::is_filter_match_tags("[.tag1][tag2]"sv, "[.]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[.]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2][!mayfail]"sv, "[!mayfail]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[!mayfail]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2][!shouldfail]"sv, "[!shouldfail]"sv)); + CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[!shouldfail]"sv)); + + CHECK(!snitch::is_filter_match_tags("[tag1]"sv, "[tag2]"sv)); + CHECK(!snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag3]"sv)); + CHECK(!snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tug*]*"sv)); + CHECK(!snitch::is_filter_match_tags("[tag1][tag2]"sv, "[.]"sv)); + CHECK(!snitch::is_filter_match_tags("[.tag1][tag2]"sv, "[.tag1]"sv)); + CHECK(!snitch::is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag1]"sv)); + CHECK(!snitch::is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag2]"sv)); +} + +TEST_CASE("is_filter_match_id", "[utility]") { + CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "abc"sv)); + CHECK(!snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~abc"sv)); + CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "ab*"sv)); + CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag1]"sv)); + CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag2]"sv)); + CHECK(!snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag3]"sv)); + CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~[tag3]"sv)); + CHECK(snitch::is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "\\[weird]"sv)); + CHECK(!snitch::is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "[weird]"sv)); } From 844ae9a6f1debf966df44520182b8b510f0e41c9 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sat, 14 Jan 2023 17:55:45 +0000 Subject: [PATCH 10/17] Refactor registry::run_* and CLI to allow multiple filters --- README.md | 23 +-- include/snitch/snitch.hpp | 8 +- src/snitch.cpp | 283 ++++++++++++++----------------- tests/runtime_tests/cli.cpp | 41 ++++- tests/runtime_tests/registry.cpp | 49 ++++-- 5 files changed, 226 insertions(+), 178 deletions(-) diff --git a/README.md b/README.md index d30fc8c0..6340c7ee 100644 --- a/README.md +++ b/README.md @@ -653,16 +653,15 @@ An example reporter for _Teamcity_ is included for demonstration, see `include/s ### Default main function The default `main()` function provided in _snitch_ offers the following command-line API: - - positional argument for filtering tests by name, see below. + - positional arguments for filtering tests by name, see below. - `-h,--help`: show command line help. - `-l,--list-tests`: list all tests. - ` --list-tags`: list all tags. - ` --list-tests-with-tag`: list all tests with a given tag. - - `-t,--tags `: filter tests by tags, see below. - `-v,--verbosity `: select level of detail for the default reporter. - ` --color `: enable/disable colors in the default reporter. -The positional argument is used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If a filter is provided, then hidden tests will no longer be excluded. This reproduces the behavior of _Catch2_. If the `-t` (or `--tags`) option is given, the filtering is applied to tags instead of test case names. +The positional arguments are used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If at lease one filter is provided, then hidden tests will no longer be excluded by default. This reproduces the behavior of _Catch2_. A filter may contain any number of "wildcard" character, `*`, which can represent zero or more characters. For example: - `ab*` will match all test cases with names starting with `ab`. @@ -678,14 +677,18 @@ If the filter starts with `~`, then it is negated: - `~abcd` will match all test cases except the test case with name `abcd`. - `~*` will match no test case. -**Note:** To match the actual character `*` in a test name, the `*` in the filter must be escaped using a backslash `\`, for example `\*`. Be mindful that most shells (Bash, etc.) will also require the backslash itself be escaped to be interpreted as an actual backslash in _snitch_: +If the filter starts with `[` or `~[`, then it applies to the test case tags, else it applies to the test case name. -| Bash | _snitch_ | matches | -|---------|----------|--------------------------------------------| -| `\\` | `\` | nothing (ill-formed filter) | -| `\\*` | `\*` | exactly the `*` character | -| `\\\\` | `\\` | exactly the `\` character | -| `\\\\*` | `\\*` | any string starting with the `\` character | +**Note:** To match the actual character `*` in a test name, the `*` in the filter must be escaped using a backslash, like `\*`. In general, any character located after a single backslash will be interpreted as a regular character, with no special meaning. Be mindful that most shells (Bash, etc.) will also require the backslash itself be escaped to be interpreted as an actual backslash in _snitch_. The table below shows examples of how edge-cases are handled: + +| Bash | _snitch_ | matches | +|---------|----------|---------------------------------------------| +| `\\` | `\` | nothing (ill-formed filter) | +| `\\*` | `\*` | any name which is exactly the `*` character | +| `\\\\` | `\\` | any name which is exactly the `\` character | +| `\\\\*` | `\\*` | any name starting with the `\` character | +| `[a*` | `[a*` | any tag starting with `[a` | +| `\\[a*` | `\[a*` | any name starting with `[a` | ### Using your own main function diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 349b8106..2b7a82d2 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -1253,9 +1253,11 @@ class registry { impl::test_state run(impl::test_case& test) noexcept; - bool run_all_tests(std::string_view run_name) noexcept; - bool run_tests_matching_name(std::string_view run_name, std::string_view name_filter) noexcept; - bool run_tests_with_tag(std::string_view run_name, std::string_view tag_filter) noexcept; + bool run_tests(std::string_view run_name) noexcept; + + bool run_selected_tests( + std::string_view run_name, + const small_function& filter) noexcept; bool run_tests(const cli::input& args) noexcept; diff --git a/src/snitch.cpp b/src/snitch.cpp index 2010d38d..d8fba690 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -586,112 +586,6 @@ void for_each_tag(std::string_view s, F&& callback) noexcept { }); } -template -bool run_tests(registry& r, std::string_view run_name, F&& predicate) noexcept { - if (!r.report_callback.empty()) { - r.report_callback(r, event::test_run_started{run_name}); - } else if (is_at_least(r.verbose, registry::verbosity::normal)) { - r.print( - make_colored("starting tests with ", r.with_color, color::highlight2), - make_colored("snitch v" SNITCH_FULL_VERSION "\n", r.with_color, color::highlight1)); - r.print("==========================================\n"); - } - - bool success = true; - std::size_t run_count = 0; - std::size_t fail_count = 0; - std::size_t skip_count = 0; - std::size_t assertion_count = 0; - -#if SNITCH_WITH_TIMINGS - using clock = std::chrono::high_resolution_clock; - auto time_start = clock::now(); -#endif - - for (test_case& t : r) { - if (!predicate(t)) { - continue; - } - - auto state = r.run(t); - - ++run_count; - assertion_count += state.asserts; - - switch (t.state) { - case impl::test_case_state::success: { - // Nothing to do - break; - } - case impl::test_case_state::failed: { - ++fail_count; - success = false; - break; - } - case impl::test_case_state::skipped: { - ++skip_count; - break; - } - case impl::test_case_state::not_run: { - // Unreachable - break; - } - } - } - -#if SNITCH_WITH_TIMINGS - auto time_end = clock::now(); - float duration = std::chrono::duration(time_end - time_start).count(); -#endif - - if (!r.report_callback.empty()) { -#if SNITCH_WITH_TIMINGS - r.report_callback( - r, event::test_run_ended{ - .name = run_name, - .success = success, - .run_count = run_count, - .fail_count = fail_count, - .skip_count = skip_count, - .assertion_count = assertion_count, - .duration = duration}); -#else - r.report_callback( - r, event::test_run_ended{ - .name = run_name, - .success = success, - .run_count = run_count, - .fail_count = fail_count, - .skip_count = skip_count, - .assertion_count = assertion_count}); -#endif - } else if (is_at_least(r.verbose, registry::verbosity::normal)) { - r.print("==========================================\n"); - - if (success) { - r.print( - make_colored("success:", r.with_color, color::pass), " all tests passed (", - run_count, " test cases, ", assertion_count, " assertions"); - } else { - r.print( - make_colored("error:", r.with_color, color::fail), " some tests failed (", - fail_count, " out of ", run_count, " test cases, ", assertion_count, " assertions"); - } - - if (skip_count > 0) { - r.print(", ", skip_count, " test cases skipped"); - } - -#if SNITCH_WITH_TIMINGS - r.print(", ", duration, " seconds"); -#endif - - r.print(")\n"); - } - - return success; -} - std::string_view make_full_name(small_string& buffer, const test_id& id) noexcept { buffer.clear(); @@ -1082,39 +976,127 @@ test_state registry::run(test_case& test) noexcept { return state; } -bool registry::run_all_tests(std::string_view run_name) noexcept { - return ::run_tests(*this, run_name, [](const test_case& t) { - bool selected = true; - for_each_tag(t.id.tags, [&](const tags::parsed_tag& s) { - if (std::holds_alternative(s)) { - selected = false; - } - }); +bool registry::run_selected_tests( + std::string_view run_name, + const small_function& predicate) noexcept { - return selected; - }); -} + if (!report_callback.empty()) { + report_callback(*this, event::test_run_started{run_name}); + } else if (is_at_least(verbose, registry::verbosity::normal)) { + print( + make_colored("starting tests with ", with_color, color::highlight2), + make_colored("snitch v" SNITCH_FULL_VERSION "\n", with_color, color::highlight1)); + print("==========================================\n"); + } -bool registry::run_tests_matching_name( - std::string_view run_name, std::string_view name_filter) noexcept { - small_string buffer; - return ::run_tests(*this, run_name, [&](const test_case& t) { - return is_filter_match(make_full_name(buffer, t.id), name_filter); - }); + bool success = true; + std::size_t run_count = 0; + std::size_t fail_count = 0; + std::size_t skip_count = 0; + std::size_t assertion_count = 0; + +#if SNITCH_WITH_TIMINGS + using clock = std::chrono::high_resolution_clock; + auto time_start = clock::now(); +#endif + + for (test_case& t : *this) { + if (!predicate(t.id)) { + continue; + } + + auto state = run(t); + + ++run_count; + assertion_count += state.asserts; + + switch (t.state) { + case impl::test_case_state::success: { + // Nothing to do + break; + } + case impl::test_case_state::failed: { + ++fail_count; + success = false; + break; + } + case impl::test_case_state::skipped: { + ++skip_count; + break; + } + case impl::test_case_state::not_run: { + // Unreachable + break; + } + } + } + +#if SNITCH_WITH_TIMINGS + auto time_end = clock::now(); + float duration = std::chrono::duration(time_end - time_start).count(); +#endif + + if (!report_callback.empty()) { +#if SNITCH_WITH_TIMINGS + report_callback( + *this, event::test_run_ended{ + .name = run_name, + .success = success, + .run_count = run_count, + .fail_count = fail_count, + .skip_count = skip_count, + .assertion_count = assertion_count, + .duration = duration}); +#else + report_callback( + *this, event::test_run_ended{ + .name = run_name, + .success = success, + .run_count = run_count, + .fail_count = fail_count, + .skip_count = skip_count, + .assertion_count = assertion_count}); +#endif + } else if (is_at_least(verbose, registry::verbosity::normal)) { + print("==========================================\n"); + + if (success) { + print( + make_colored("success:", with_color, color::pass), " all tests passed (", run_count, + " test cases, ", assertion_count, " assertions"); + } else { + print( + make_colored("error:", with_color, color::fail), " some tests failed (", fail_count, + " out of ", run_count, " test cases, ", assertion_count, " assertions"); + } + + if (skip_count > 0) { + print(", ", skip_count, " test cases skipped"); + } + +#if SNITCH_WITH_TIMINGS + print(", ", duration, " seconds"); +#endif + + print(")\n"); + } + + return success; } -bool registry::run_tests_with_tag(std::string_view run_name, std::string_view tag_filter) noexcept { - return ::run_tests(*this, run_name, [&](const test_case& t) { - bool selected = false; - for_each_tag(t.id.tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); - vs != nullptr && is_filter_match(*vs, tag_filter)) { - selected = true; +bool registry::run_tests(std::string_view run_name) noexcept { + const auto filter = [](const test_id& id) { + bool selected = true; + for_each_tag(id.tags, [&](const tags::parsed_tag& s) { + if (std::holds_alternative(s)) { + selected = false; } }); return selected; - }); + }; + + return run_selected_tests(run_name, filter); } void registry::list_all_tags() const noexcept { @@ -1150,17 +1132,7 @@ void registry::list_all_tests() const noexcept { } void registry::list_tests_with_tag(std::string_view tag) const noexcept { - list_tests(*this, [&](const test_case& t) { - bool selected = false; - for_each_tag(t.id.tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); - vs != nullptr && is_filter_match(*vs, tag)) { - selected = true; - } - }); - - return selected; - }); + list_tests(*this, [&](const test_case& t) { return is_filter_match_tags(t.id.tags, tag); }); } test_case* registry::begin() noexcept { @@ -1463,11 +1435,10 @@ constexpr expected_arguments expected_args = { {{"-l", "--list-tests"}, {}, "List tests by name"}, {{"--list-tags"}, {}, "List tags by name"}, {{"--list-tests-with-tag"}, {"[tag]"}, "List tests by name with a given tag"}, - {{"-t", "--tags"}, {}, "Use tags for filtering, not name"}, {{"-v", "--verbosity"}, {"quiet|normal|high"}, "Define how much gets sent to the standard output"}, {{"--color"}, {"always|never"}, "Enable/disable color in output"}, {{"-h", "--help"}, {}, "Print help"}, - {{}, {"test regex"}, "A regex to select which test cases (or tags) to run"}}; + {{}, {"test regex"}, "A regex to select which test cases to run", argument_type::repeatable}}; // clang-format on constexpr bool with_color_default = SNITCH_DEFAULT_WITH_COLOR == 1; @@ -1589,14 +1560,24 @@ bool registry::run_tests(const cli::input& args) noexcept { return true; } - if (auto opt = get_positional_argument(args, "test regex")) { - if (get_option(args, "--tags")) { - return run_tests_with_tag(args.executable, *opt->value); - } else { - return run_tests_matching_name(args.executable, *opt->value); - } + if (get_positional_argument(args, "test regex").has_value()) { + small_string buffer; + + const auto filter = [&](const test_id& id) noexcept { + bool selected = true; + + const auto callback = [&](std::string_view filter) noexcept { + selected = is_filter_match_id(id, filter); + }; + + for_each_positional_argument(args, "test regex", callback); + + return selected; + }; + + return run_selected_tests(args.executable, filter); } else { - return run_all_tests(args.executable); + return run_tests(args.executable); } } } // namespace snitch diff --git a/tests/runtime_tests/cli.cpp b/tests/runtime_tests/cli.cpp index b2242011..db279166 100644 --- a/tests/runtime_tests/cli.cpp +++ b/tests/runtime_tests/cli.cpp @@ -156,14 +156,29 @@ TEST_CASE("parse arguments positional", "[cli]") { CHECK(console.messages.empty()); } -TEST_CASE("parse arguments too many positional", "[cli]") { +TEST_CASE("parse arguments multiple positional", "[cli]") { console_output_catcher console; const arg_vector args = {"test", "arg1", "arg2"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); - REQUIRE(!input.has_value()); - CHECK(console.messages == contains_substring("too many positional arguments")); + REQUIRE(input.has_value()); + CHECK(input->executable == "test"sv); + REQUIRE(input->arguments.size() == 2u); + + CHECK(input->arguments[0].name == ""sv); + REQUIRE(input->arguments[0].value.has_value()); + REQUIRE(input->arguments[0].value_name.has_value()); + CHECK(input->arguments[0].value.value() == "arg1"sv); + CHECK(input->arguments[0].value_name.value() == "test regex"sv); + + CHECK(input->arguments[1].name == ""sv); + REQUIRE(input->arguments[1].value.has_value()); + REQUIRE(input->arguments[1].value_name.has_value()); + CHECK(input->arguments[1].value.value() == "arg2"sv); + CHECK(input->arguments[1].value_name.value() == "test regex"sv); + + CHECK(console.messages.empty()); } TEST_CASE("get option", "[cli]") { @@ -200,6 +215,9 @@ TEST_CASE("get positional argument", "[cli]") { cli_input{"at middle"sv, {"test", "--help", "arg1", "--verbosity", "high"}}, cli_input{"at start"sv, {"test", "arg1", "--help", "--verbosity", "high"}}, cli_input{"alone"sv, {"test", "arg1"}}, + cli_input{"multiple"sv, {"test", "arg1", "arg2"}}, + cli_input{ + "multiple interleaved"sv, {"test", "arg1", "--verbosity", "high", "arg2"}}, }) { #if SNITCH_TEST_WITH_SNITCH @@ -214,6 +232,23 @@ TEST_CASE("get positional argument", "[cli]") { CHECK(arg->name == ""sv); CHECK(arg->value == "arg1"sv); CHECK(arg->value_name == "test regex"sv); + + if (input->executable.starts_with("multiple")) { + std::size_t i = 0u; + + auto callback = [&](std::string_view value) noexcept { + if (i == 0u) { + CHECK(value == "arg1"sv); + } else { + CHECK(value == "arg2"sv); + } + + ++i; + }; + + snitch::cli::for_each_positional_argument(*input, "test regex", callback); + CHECK(i == 2u); + } } } diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index cc2e668e..45b0bbdf 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -503,8 +503,8 @@ TEST_CASE("run tests", "[registry]") { INFO((r == reporter::print ? "default reporter" : "custom reporter")); - SECTION("run all tests") { - framework.registry.run_all_tests("test_app"); + SECTION("run tests") { + framework.registry.run_tests("test_app"); CHECK(test_called); CHECK(test_called_other_tag); @@ -526,7 +526,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all pass") { - framework.registry.run_tests_matching_name("test_app", "*are you"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_name(id.name, "*are you"); + }); CHECK(test_called); CHECK(!test_called_other_tag); @@ -547,7 +550,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all failed") { - framework.registry.run_tests_matching_name("test_app", "*lights*"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_name(id.name, "*lights*"); + }); CHECK(!test_called); CHECK(test_called_other_tag); @@ -568,7 +574,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all skipped") { - framework.registry.run_tests_matching_name("test_app", "*cup"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_name(id.name, "*cup"); + }); CHECK(!test_called); CHECK(!test_called_other_tag); @@ -590,7 +599,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered tags") { - framework.registry.run_tests_with_tag("test_app", "[other_tag]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[other_tag]"); + }); CHECK(!test_called); CHECK(test_called_other_tag); @@ -611,7 +623,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered tags wildcard") { - framework.registry.run_tests_with_tag("test_app", "*tag]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "*tag]"); + }); CHECK(test_called); CHECK(test_called_other_tag); @@ -632,7 +647,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [.]") { - framework.registry.run_tests_with_tag("test_app", "[hidden]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[hidden]"); + }); CHECK(!test_called); CHECK(!test_called_other_tag); @@ -653,7 +671,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [!mayfail]") { - framework.registry.run_tests_with_tag("test_app", "[may fail]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[may fail]"); + }); if (r == reporter::print) { CHECK( @@ -666,7 +687,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [!shouldfail]") { - framework.registry.run_tests_with_tag("test_app", "[should fail]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[should fail]"); + }); if (r == reporter::print) { CHECK( @@ -679,7 +703,10 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [!shouldfail][!mayfail]") { - framework.registry.run_tests_with_tag("test_app", "[may+should fail]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[may+should fail]"); + }); if (r == reporter::print) { CHECK( From c30e4edc8f54ed7af504201094c2f2cb0649309a Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 10:35:02 +0000 Subject: [PATCH 11/17] Match multi filter behavior to Catch2 --- README.md | 42 +++++++---- include/snitch/snitch.hpp | 10 ++- src/snitch.cpp | 61 +++++++++++----- tests/runtime_tests/registry.cpp | 54 +++++++++++--- tests/runtime_tests/string_utility.cpp | 97 +++++++++++++++----------- 5 files changed, 179 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 6340c7ee..1ba8d87f 100644 --- a/README.md +++ b/README.md @@ -661,23 +661,35 @@ The default `main()` function provided in _snitch_ offers the following command- - `-v,--verbosity `: select level of detail for the default reporter. - ` --color `: enable/disable colors in the default reporter. -The positional arguments are used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If at lease one filter is provided, then hidden tests will no longer be excluded by default. This reproduces the behavior of _Catch2_. +The positional arguments are used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If at least one filter is provided, then hidden tests will no longer be excluded by default. This reproduces the behavior of _Catch2_. A filter may contain any number of "wildcard" character, `*`, which can represent zero or more characters. For example: - - `ab*` will match all test cases with names starting with `ab`. - - `*cd` will match all test cases with names ending with `cd`. - - `ab*cd` will match all test cases with names starting with `ab` and ending with `cd`. - - `abcd` will only match the test case with name `abcd`. - - `*` will match all test cases. - -If the filter starts with `~`, then it is negated: - - `~ab*` will match all test cases with names NOT starting with `ab`. - - `~*cd` will match all test cases with names NOT ending with `cd`. - - `~ab*cd` will match all test cases with names NOT starting with `ab` or NOT ending with `cd`. - - `~abcd` will match all test cases except the test case with name `abcd`. - - `~*` will match no test case. - -If the filter starts with `[` or `~[`, then it applies to the test case tags, else it applies to the test case name. + - `ab*` will include all test cases with names starting with `ab`. + - `*cd` will include all test cases with names ending with `cd`. + - `ab*cd` will include all test cases with names starting with `ab` and ending with `cd`. + - `abcd` will only include the test case with name `abcd`. + - `*` will include all test cases. + +If a filter starts with `~`, then it is interpreted as an exclusion: + - `~ab*` will exclude all test cases with names starting with `ab`. + - `~*cd` will exclude all test cases with names ending with `cd`. + - `~ab*cd` will exclude all test cases with names starting with `ab` and ending with `cd`. + - `~abcd` will exclude the test case with name `abcd`. + - `~*` will exclude all test cases. + +If a filter starts with `[` or `~[`, then it applies to the test case tags, else it applies to the test case name. This behavior can be bypassed by escaping the bracket `\[`, in which case the filter applies to the test case name again (see note below on escaping). + +Finally, if more than one filter is provided, then filters are applied one after the other, in the order provided. As in _Catch2_, a filter will include (or exclude with `~`) the tests that match the inclusion (or exclusion) pattern, but will leave the status of tests that do not match the filter unchanged. Filters on test names and tags can be mixed. For example, the table below shows which test is included (1) or excluded (0) after applying the three filters `a* ~*d abcd`: + +| Test name | Initial | Apply `a*` | State | Apply `~*d` | State | Apply `abcd` | State | +|-----------|---------| ------------|-------|-------------|-------|--------------|-------| +| `a` | 0 | 1 | 1 | | 1 | | 1 | +| `b` | 0 | | 0 | | 0 | | 0 | +| `c` | 0 | | 0 | | 0 | | 0 | +| `d` | 0 | | 0 | 0 | 0 | | 0 | +| `abc` | 0 | 1 | 1 | | 1 | | 1 | +| `abd` | 0 | 1 | 1 | 0 | 0 | | 0 | +| `abcd` | 0 | 1 | 1 | 0 | 0 | 1 | 1 | **Note:** To match the actual character `*` in a test name, the `*` in the filter must be escaped using a backslash, like `\*`. In general, any character located after a single backslash will be interpreted as a regular character, with no special meaning. Be mindful that most shells (Bash, etc.) will also require the backslash itself be escaped to be interpreted as an actual backslash in _snitch_. The table below shows examples of how edge-cases are handled: diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 2b7a82d2..d337b6a0 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -584,11 +584,15 @@ bool append_or_truncate(small_string_span ss, Args&&... args) noexcept { [[nodiscard]] bool is_match(std::string_view string, std::string_view regex) noexcept; -[[nodiscard]] bool is_filter_match_name(std::string_view name, std::string_view filter) noexcept; +enum class filter_result { included, excluded, not_included, not_excluded }; -[[nodiscard]] bool is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept; +[[nodiscard]] filter_result +is_filter_match_name(std::string_view name, std::string_view filter) noexcept; -[[nodiscard]] bool is_filter_match_id(const test_id& id, std::string_view filter) noexcept; +[[nodiscard]] filter_result +is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept; + +[[nodiscard]] filter_result is_filter_match_id(const test_id& id, std::string_view filter) noexcept; template concept matcher_for = requires(const T& m, const U& value) { diff --git a/src/snitch.cpp b/src/snitch.cpp index d8fba690..a484ccda 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -641,30 +641,39 @@ small_vector make_capture_buffer(const capture_s } // namespace namespace snitch { -bool is_filter_match_name(std::string_view name, std::string_view filter) noexcept { - bool expected = true; - if (filter.size() > 1 && filter[0] == '~') { - filter = filter.substr(1); - expected = false; +filter_result is_filter_match_name(std::string_view name, std::string_view filter) noexcept { + filter_result match_action = filter_result::included; + filter_result no_match_action = filter_result::not_included; + if (filter.starts_with('~')) { + filter = filter.substr(1); + match_action = filter_result::excluded; + no_match_action = filter_result::not_excluded; } - return is_match(name, filter) == expected; + return is_match(name, filter) ? match_action : no_match_action; } -bool is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept { - bool selected = false; +filter_result is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept { + filter_result match_action = filter_result::included; + filter_result no_match_action = filter_result::not_included; + if (filter.starts_with('~')) { + filter = filter.substr(1); + match_action = filter_result::excluded; + no_match_action = filter_result::not_excluded; + } + + bool match = false; for_each_tag(tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); - vs != nullptr && is_filter_match_name(*vs, filter)) { - selected = true; + if (auto* vs = std::get_if(&v); vs != nullptr && is_match(*vs, filter)) { + match = true; } }); - return selected; + return match ? match_action : no_match_action; } -bool is_filter_match_id(const test_id& id, std::string_view filter) noexcept { - if (filter.starts_with("[") || filter.starts_with("~[")) { +filter_result is_filter_match_id(const test_id& id, std::string_view filter) noexcept { + if (filter.starts_with('[') || filter.starts_with("~[")) { return is_filter_match_tags(id.tags, filter); } else { return is_filter_match_name(id.name, filter); @@ -1132,7 +1141,10 @@ void registry::list_all_tests() const noexcept { } void registry::list_tests_with_tag(std::string_view tag) const noexcept { - list_tests(*this, [&](const test_case& t) { return is_filter_match_tags(t.id.tags, tag); }); + list_tests(*this, [&](const test_case& t) { + const auto result = is_filter_match_tags(t.id.tags, tag); + return result == filter_result::included || result == filter_result::not_excluded; + }); } test_case* registry::begin() noexcept { @@ -1564,15 +1576,28 @@ bool registry::run_tests(const cli::input& args) noexcept { small_string buffer; const auto filter = [&](const test_id& id) noexcept { - bool selected = true; + std::optional selected; const auto callback = [&](std::string_view filter) noexcept { - selected = is_filter_match_id(id, filter); + switch (is_filter_match_id(id, filter)) { + case filter_result::included: selected = true; break; + case filter_result::excluded: selected = false; break; + case filter_result::not_included: + if (!selected.has_value()) { + selected = false; + } + break; + case filter_result::not_excluded: + if (!selected.has_value()) { + selected = true; + } + break; + } }; for_each_positional_argument(args, "test regex", callback); - return selected; + return selected.value(); }; return run_selected_tests(args.executable, filter); diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index 45b0bbdf..259e73a6 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -528,7 +528,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests filtered all pass") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_name(id.name, "*are you"); + return snitch::is_filter_match_name(id.name, "*are you") == + snitch::filter_result::included; }); CHECK(test_called); @@ -552,7 +553,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests filtered all failed") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_name(id.name, "*lights*"); + return snitch::is_filter_match_name(id.name, "*lights*") == + snitch::filter_result::included; }); CHECK(!test_called); @@ -576,7 +578,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests filtered all skipped") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_name(id.name, "*cup"); + return snitch::is_filter_match_name(id.name, "*cup") == + snitch::filter_result::included; }); CHECK(!test_called); @@ -601,7 +604,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests filtered tags") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_tags(id.tags, "[other_tag]"); + return snitch::is_filter_match_tags(id.tags, "[other_tag]") == + snitch::filter_result::included; }); CHECK(!test_called); @@ -625,7 +629,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests filtered tags wildcard") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_tags(id.tags, "*tag]"); + return snitch::is_filter_match_tags(id.tags, "*tag]") == + snitch::filter_result::included; }); CHECK(test_called); @@ -649,7 +654,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests special tag [.]") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_tags(id.tags, "[hidden]"); + return snitch::is_filter_match_tags(id.tags, "[hidden]") == + snitch::filter_result::included; }); CHECK(!test_called); @@ -673,7 +679,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests special tag [!mayfail]") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_tags(id.tags, "[may fail]"); + return snitch::is_filter_match_tags(id.tags, "[may fail]") == + snitch::filter_result::included; }); if (r == reporter::print) { @@ -689,7 +696,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests special tag [!shouldfail]") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_tags(id.tags, "[should fail]"); + return snitch::is_filter_match_tags(id.tags, "[should fail]") == + snitch::filter_result::included; }); if (r == reporter::print) { @@ -705,7 +713,8 @@ TEST_CASE("run tests", "[registry]") { SECTION("run tests special tag [!shouldfail][!mayfail]") { framework.registry.run_selected_tests( "test_app", [](const snitch::test_id& id) noexcept { - return snitch::is_filter_match_tags(id.tags, "[may+should fail]"); + return snitch::is_filter_match_tags(id.tags, "[may+should fail]") == + snitch::filter_result::included; }); if (r == reporter::print) { @@ -936,10 +945,35 @@ TEST_CASE("run tests cli", "[registry]") { } SECTION("test tag filter") { - const arg_vector args = {"test", "--tags", "[skipped]"}; + const arg_vector args = {"test", "[skipped]"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); framework.registry.run_tests(*input); CHECK_RUN(true, 1u, 0u, 1u, 0u); } } + +std::array readme_test_called = {false}; + +TEST_CASE("run tests cli readme example", "[registry]") { + mock_framework framework; + framework.setup_reporter_and_print(); + console_output_catcher console; + + readme_test_called = {false}; + + framework.registry.add({"a"}, []() { readme_test_called[0] = true; }); + framework.registry.add({"b"}, []() { readme_test_called[1] = true; }); + framework.registry.add({"c"}, []() { readme_test_called[2] = true; }); + framework.registry.add({"d"}, []() { readme_test_called[3] = true; }); + framework.registry.add({"abc"}, []() { readme_test_called[4] = true; }); + framework.registry.add({"abd"}, []() { readme_test_called[5] = true; }); + framework.registry.add({"abcd"}, []() { readme_test_called[6] = true; }); + + const arg_vector args = {"test", "a*", "~*d", "abcd"}; + auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); + framework.registry.run_tests(*input); + + std::array expected = {true, false, false, false, true, false, true}; + CHECK(readme_test_called == expected); +} diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index 40785ba3..f0520e30 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -517,50 +517,69 @@ TEST_CASE("is_match", "[utility]") { } } +using snitch::filter_result; +using snitch::is_filter_match_id; +using snitch::is_filter_match_name; +using snitch::is_filter_match_tags; + TEST_CASE("is_filter_match", "[utility]") { - CHECK(snitch::is_filter_match_name("abc"sv, "abc"sv)); - CHECK(snitch::is_filter_match_name("abc"sv, "ab*"sv)); - CHECK(snitch::is_filter_match_name("abc"sv, "*bc"sv)); - CHECK(snitch::is_filter_match_name("abc"sv, "*"sv)); - CHECK(!snitch::is_filter_match_name("abc"sv, "~abc"sv)); - CHECK(!snitch::is_filter_match_name("abc"sv, "~ab*"sv)); - CHECK(!snitch::is_filter_match_name("abc"sv, "~*bc"sv)); - CHECK(!snitch::is_filter_match_name("abc"sv, "~*"sv)); + CHECK(is_filter_match_name("abc"sv, "abc"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "ab*"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "*bc"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "*"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "def"sv) == filter_result::not_included); + CHECK(is_filter_match_name("abc"sv, "~abc"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~ab*"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~*bc"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~*"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~def"sv) == filter_result::not_excluded); } TEST_CASE("is_filter_match_tag", "[utility]") { - CHECK(snitch::is_filter_match_tags("[tag1]"sv, "[tag1]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag1]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag2]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[tug*]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2][.]"sv, "[.]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][.tag2]"sv, "[.]"sv)); - CHECK(snitch::is_filter_match_tags("[.tag1][tag2]"sv, "[.]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[.]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2][!mayfail]"sv, "[!mayfail]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[!mayfail]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2][!shouldfail]"sv, "[!shouldfail]"sv)); - CHECK(snitch::is_filter_match_tags("[tag1][tag2]"sv, "~[!shouldfail]"sv)); - - CHECK(!snitch::is_filter_match_tags("[tag1]"sv, "[tag2]"sv)); - CHECK(!snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tag3]"sv)); - CHECK(!snitch::is_filter_match_tags("[tag1][tag2]"sv, "[tug*]*"sv)); - CHECK(!snitch::is_filter_match_tags("[tag1][tag2]"sv, "[.]"sv)); - CHECK(!snitch::is_filter_match_tags("[.tag1][tag2]"sv, "[.tag1]"sv)); - CHECK(!snitch::is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag1]"sv)); - CHECK(!snitch::is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag2]"sv)); + CHECK(is_filter_match_tags("[tag1]"sv, "[tag1]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag1]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag2]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[tug*]"sv) == filter_result::not_excluded); + CHECK(is_filter_match_tags("[tag1][tag2][.]"sv, "[.]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][.tag2]"sv, "[.]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[.tag1][tag2]"sv, "[.]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[.]"sv) == filter_result::not_excluded); + CHECK(is_filter_match_tags("[tag1][!mayfail]"sv, "[!mayfail]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[!mayfail]"sv) == filter_result::not_excluded); + CHECK( + is_filter_match_tags("[tag1][!shouldfail]"sv, "[!shouldfail]"sv) == + filter_result::included); + CHECK( + is_filter_match_tags("[tag1][tag2]"sv, "~[!shouldfail]"sv) == filter_result::not_excluded); + + CHECK(is_filter_match_tags("[tag1]"sv, "[tag2]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag3]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tug*]*"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[.]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[.tag1][tag2]"sv, "[.tag1]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag1]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag2]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[tag1]"sv) == filter_result::excluded); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[tag2]"sv) == filter_result::excluded); } TEST_CASE("is_filter_match_id", "[utility]") { - CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "abc"sv)); - CHECK(!snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~abc"sv)); - CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "ab*"sv)); - CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag1]"sv)); - CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag2]"sv)); - CHECK(!snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag3]"sv)); - CHECK(snitch::is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~[tag3]"sv)); - CHECK(snitch::is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "\\[weird]"sv)); - CHECK(!snitch::is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "[weird]"sv)); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "abc"sv) == filter_result::included); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~abc"sv) == filter_result::excluded); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "ab*"sv) == filter_result::included); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag1]"sv) == filter_result::included); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag2]"sv) == filter_result::included); + CHECK( + is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag3]"sv) == filter_result::not_included); + CHECK( + is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~[tag3]"sv) == + filter_result::not_excluded); + CHECK( + is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "\\[weird]"sv) == + filter_result::included); + CHECK( + is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "[weird]"sv) == + filter_result::not_included); } From 6a45c59e1c7f7f230de9e743774aa7f0119535b1 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 10:37:08 +0000 Subject: [PATCH 12/17] Removed unused variable --- src/snitch.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/snitch.cpp b/src/snitch.cpp index a484ccda..f327720e 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -1573,8 +1573,6 @@ bool registry::run_tests(const cli::input& args) noexcept { } if (get_positional_argument(args, "test regex").has_value()) { - small_string buffer; - const auto filter = [&](const test_id& id) noexcept { std::optional selected; From b1d327f1cca86b73e391cdf96892be6381b85550 Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 10:44:51 +0000 Subject: [PATCH 13/17] Added missing tests for coverage --- tests/runtime_tests/registry.cpp | 8 ++++++++ tests/runtime_tests/string_utility.cpp | 1 + 2 files changed, 9 insertions(+) diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index 259e73a6..a72606f9 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -944,6 +944,14 @@ TEST_CASE("run tests cli", "[registry]") { CHECK_RUN(false, 3u, 3u, 0u, 3u); } + SECTION("test filter exclusion") { + const arg_vector args = {"test", "~*fail"}; + auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); + framework.registry.run_tests(*input); + + CHECK_RUN(false, 7u, 3u, 1u, 3u); + } + SECTION("test tag filter") { const arg_vector args = {"test", "[skipped]"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index f0520e30..d79901aa 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -478,6 +478,7 @@ TEST_CASE("is_match", "[utility]") { CHECK(snitch::is_match("abc"sv, "**"sv)); CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "**"sv)); CHECK(snitch::is_match(""sv, "**"sv)); + CHECK(snitch::is_match("abcdefg"sv, "*g*******"sv)); CHECK(snitch::is_match("abc"sv, "abc**"sv)); CHECK(snitch::is_match("abc"sv, "ab**"sv)); CHECK(snitch::is_match("abc"sv, "a**"sv)); From da62fd01fc653c0cd8b48b81d4af9ba36542a43b Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 10:52:53 +0000 Subject: [PATCH 14/17] Removed unused SNITCH_MAX_TAG_LENGTH --- CMakeLists.txt | 2 -- include/snitch/snitch.hpp | 2 -- include/snitch/snitch_config.hpp.config | 3 --- 3 files changed, 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c5b18af..48ad9975 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,6 @@ set(SNITCH_MAX_NESTED_SECTIONS 8 CACHE STRING "Maximum depth of nested sec set(SNITCH_MAX_EXPR_LENGTH 1024 CACHE STRING "Maximum length of a printed expression when reporting failure.") set(SNITCH_MAX_MESSAGE_LENGTH 1024 CACHE STRING "Maximum length of error or status messages.") set(SNITCH_MAX_TEST_NAME_LENGTH 1024 CACHE STRING "Maximum length of a test case name.") -set(SNITCH_MAX_TAG_LENGTH 256 CACHE STRING "Maximum length of a test tag.") set(SNITCH_MAX_CAPTURES 8 CACHE STRING "Maximum number of captured expressions in a test case.") set(SNITCH_MAX_CAPTURE_LENGTH 256 CACHE STRING "Maximum length of a captured expression.") set(SNITCH_MAX_UNIQUE_TAGS 1024 CACHE STRING "Maximum number of unique tags in a test application.") @@ -60,7 +59,6 @@ if (SNITCH_CREATE_LIBRARY) SNITCH_MAX_EXPR_LENGTH=${SNITCH_MAX_EXPR_LENGTH} SNITCH_MAX_MESSAGE_LENGTH=${SNITCH_MAX_MESSAGE_LENGTH} SNITCH_MAX_TEST_NAME_LENGTH=${SNITCH_MAX_TEST_NAME_LENGTH} - SNITCH_MAX_TAG_LENGTH=${SNITCH_MAX_TAG_LENGTH} SNITCH_MAX_UNIQUE_TAGS=${SNITCH_MAX_UNIQUE_TAGS} SNITCH_MAX_COMMAND_LINE_ARGS=${SNITCH_MAX_COMMAND_LINE_ARGS} SNITCH_DEFINE_MAIN=$ diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index d337b6a0..c8557fdc 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -33,8 +33,6 @@ constexpr std::size_t max_message_length = SNITCH_MAX_MESSAGE_LENGTH; // Maximum length of a full test case name. // The full test case name includes the base name, plus any type. constexpr std::size_t max_test_name_length = SNITCH_MAX_TEST_NAME_LENGTH; -// Maximum length of a tag, including brackets. -constexpr std::size_t max_tag_length = SNITCH_MAX_TAG_LENGTH; // Maximum number of captured expressions in a test case. constexpr std::size_t max_captures = SNITCH_MAX_CAPTURES; // Maximum length of a captured expression. diff --git a/include/snitch/snitch_config.hpp.config b/include/snitch/snitch_config.hpp.config index 6f9dac9a..a69aea39 100644 --- a/include/snitch/snitch_config.hpp.config +++ b/include/snitch/snitch_config.hpp.config @@ -24,9 +24,6 @@ #if !defined(SNITCH_MAX_TEST_NAME_LENGTH) # define SNITCH_MAX_TEST_NAME_LENGTH ${SNITCH_MAX_TEST_NAME_LENGTH} #endif -#if !defined(SNITCH_MAX_TAG_LENGTH) -# define SNITCH_MAX_TAG_LENGTH ${SNITCH_MAX_TAG_LENGTH} -#endif #if !defined(SNITCH_MAX_CAPTURES) # define SNITCH_MAX_CAPTURES ${SNITCH_MAX_CAPTURES} #endif From 3583e98245733979c902ff96f567ea765303582f Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 10:53:39 +0000 Subject: [PATCH 15/17] Revert "Removed unused SNITCH_MAX_TAG_LENGTH" This reverts commit da62fd01fc653c0cd8b48b81d4af9ba36542a43b. --- CMakeLists.txt | 2 ++ include/snitch/snitch.hpp | 2 ++ include/snitch/snitch_config.hpp.config | 3 +++ 3 files changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 48ad9975..7c5b18af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ set(SNITCH_MAX_NESTED_SECTIONS 8 CACHE STRING "Maximum depth of nested sec set(SNITCH_MAX_EXPR_LENGTH 1024 CACHE STRING "Maximum length of a printed expression when reporting failure.") set(SNITCH_MAX_MESSAGE_LENGTH 1024 CACHE STRING "Maximum length of error or status messages.") set(SNITCH_MAX_TEST_NAME_LENGTH 1024 CACHE STRING "Maximum length of a test case name.") +set(SNITCH_MAX_TAG_LENGTH 256 CACHE STRING "Maximum length of a test tag.") set(SNITCH_MAX_CAPTURES 8 CACHE STRING "Maximum number of captured expressions in a test case.") set(SNITCH_MAX_CAPTURE_LENGTH 256 CACHE STRING "Maximum length of a captured expression.") set(SNITCH_MAX_UNIQUE_TAGS 1024 CACHE STRING "Maximum number of unique tags in a test application.") @@ -59,6 +60,7 @@ if (SNITCH_CREATE_LIBRARY) SNITCH_MAX_EXPR_LENGTH=${SNITCH_MAX_EXPR_LENGTH} SNITCH_MAX_MESSAGE_LENGTH=${SNITCH_MAX_MESSAGE_LENGTH} SNITCH_MAX_TEST_NAME_LENGTH=${SNITCH_MAX_TEST_NAME_LENGTH} + SNITCH_MAX_TAG_LENGTH=${SNITCH_MAX_TAG_LENGTH} SNITCH_MAX_UNIQUE_TAGS=${SNITCH_MAX_UNIQUE_TAGS} SNITCH_MAX_COMMAND_LINE_ARGS=${SNITCH_MAX_COMMAND_LINE_ARGS} SNITCH_DEFINE_MAIN=$ diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index c8557fdc..d337b6a0 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -33,6 +33,8 @@ constexpr std::size_t max_message_length = SNITCH_MAX_MESSAGE_LENGTH; // Maximum length of a full test case name. // The full test case name includes the base name, plus any type. constexpr std::size_t max_test_name_length = SNITCH_MAX_TEST_NAME_LENGTH; +// Maximum length of a tag, including brackets. +constexpr std::size_t max_tag_length = SNITCH_MAX_TAG_LENGTH; // Maximum number of captured expressions in a test case. constexpr std::size_t max_captures = SNITCH_MAX_CAPTURES; // Maximum length of a captured expression. diff --git a/include/snitch/snitch_config.hpp.config b/include/snitch/snitch_config.hpp.config index a69aea39..6f9dac9a 100644 --- a/include/snitch/snitch_config.hpp.config +++ b/include/snitch/snitch_config.hpp.config @@ -24,6 +24,9 @@ #if !defined(SNITCH_MAX_TEST_NAME_LENGTH) # define SNITCH_MAX_TEST_NAME_LENGTH ${SNITCH_MAX_TEST_NAME_LENGTH} #endif +#if !defined(SNITCH_MAX_TAG_LENGTH) +# define SNITCH_MAX_TAG_LENGTH ${SNITCH_MAX_TAG_LENGTH} +#endif #if !defined(SNITCH_MAX_CAPTURES) # define SNITCH_MAX_CAPTURES ${SNITCH_MAX_CAPTURES} #endif From 4d78c2305631f44d4c4f60875bc3eb95d829af7b Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 17:11:36 +0000 Subject: [PATCH 16/17] Updated benchmarks I re-ran all benchmarks, since all runtime timings for snitch inexplicably went up significantly on my machine (even going back to previous commits with recorded runtime timings). It could be the BIOS update I did a week ago, or the kernel update. This time I used a script to run all benchmarks, which may reduce variability. --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 1ba8d87f..5bf0c98b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t - No heap allocation from the testing framework, so heap allocations from your code can be tracked precisely. - Works with exceptions disabled, albeit with a minor limitation (see [Exceptions](#exceptions) below). - No external dependency; just pure C++20 with the STL. - - Compiles template-heavy tests at least 60% faster than other testing frameworks (see [Benchmark](#benchmark)). + - Compiles template-heavy tests at least 50% faster than other testing frameworks (see Release [benchmarks](#benchmark)). - By defaults, test results are reported to the standard output, with optional coloring for readability. Test events can also be forwarded to a reporter callback for reporting to CI frameworks (Teamcity, ..., see [Reporters](#reporters)). - Limited subset of the [_Catch2_](https://github.com/catchorg/_Catch2_) API, see [Comparison with _Catch2_](#detailed-comparison-with-catch2). - Additional API not in _Catch2_, or different from _Catch2_: @@ -152,7 +152,7 @@ See the documentation for the [header-only mode](#header-only-build) for more in ## Benchmark The following benchmarks were done using real-world tests from another library ([_observable_unique_ptr_](https://github.com/cschreib/observable_unique_ptr)), which generates about 4000 test cases and 25000 checks. This library uses "typed" tests almost exclusively, where each test case is instantiated several times, each time with a different tested type (here, 25 types). Building and running the tests was done without parallelism to simplify the comparison. The benchmarks were ran on a desktop with the following specs: - - OS: Linux Mint 20.3, linux kernel 5.15.0-48-generic. + - OS: Linux Mint 20.3, linux kernel 5.15.0-56-generic. - CPU: AMD Ryzen 5 2600 (6 core). - RAM: 16GB. - Storage: NVMe. @@ -176,22 +176,22 @@ Results for Debug builds: | **Debug** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ | |-----------------|----------|----------|-----------|------------| -| Build framework | 1.7s | 64s | 2.0s | 0s | -| Build tests | 63s | 86s | 78s | 109s | -| Build all | 65s | 150s | 80s | 109s | -| Run tests | 18ms | 83ms | 60ms | 20ms | -| Library size | 2.90MB | 38.6MB | 2.8MB | 0MB | -| Executable size | 33.2MB | 49.3MB | 38.6MB | 51.9MB | +| Build framework | 2.0s | 41s | 2.0s | 0s | +| Build tests | 65s | 79s | 73s | 118s | +| Build all | 67s | 120s | 75s | 118s | +| Run tests | 31ms | 76ms | 63ms | 20ms | +| Library size | 3.3MB | 38.6MB | 2.8MB | 0MB | +| Executable size | 33.4MB | 49.3MB | 38.6MB | 51.9MB | Results for Release builds: | **Release** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ | |-----------------|----------|----------|-----------|------------| -| Build framework | 2.4s | 68s | 3.6s | 0s | -| Build tests | 135s | 264s | 216s | 281s | -| Build all | 137s | 332s | 220s | 281s | -| Run tests | 9ms | 31ms | 36ms | 10ms | -| Library size | 0.62MB | 2.6MB | 0.39MB | 0MB | +| Build framework | 2.6s | 47s | 3.5s | 0s | +| Build tests | 137s | 254s | 207s | 289s | +| Build all | 140s | 301s | 210s | 289s | +| Run tests | 24ms | 46ms | 44ms | 5ms | +| Library size | 0.65MB | 2.6MB | 0.39MB | 0MB | | Executable size | 9.8MB | 17.4MB | 15.2MB | 11.3MB | Notes: From 3497bc606f92e88900775927f963354411977d0c Mon Sep 17 00:00:00 2001 From: Corentin Schreiber Date: Sun, 15 Jan 2023 21:49:03 +0000 Subject: [PATCH 17/17] Use steady_clock instead of high_resolution_clock See notes in https://en.cppreference.com/w/cpp/chrono/high_resolution_clock --- src/snitch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/snitch.cpp b/src/snitch.cpp index f327720e..f8fa219f 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -903,7 +903,7 @@ test_state registry::run(test_case& test) noexcept { thread_current_test = &state; #if SNITCH_WITH_TIMINGS - using clock = std::chrono::high_resolution_clock; + using clock = std::chrono::steady_clock; auto time_start = clock::now(); #endif @@ -1005,7 +1005,7 @@ bool registry::run_selected_tests( std::size_t assertion_count = 0; #if SNITCH_WITH_TIMINGS - using clock = std::chrono::high_resolution_clock; + using clock = std::chrono::steady_clock; auto time_start = clock::now(); #endif