diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a2da0bf83..0af1de021 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -139,8 +139,8 @@ if(RYML_TEST_SUITE) set(tsdir ${ed}/yaml-test-suite) c4_download_remote_proj(yaml-test-suite tsdir GIT_REPOSITORY https://github.com/yaml/yaml-test-suite - GIT_TAG ed99dd31187f00d729fe160a7658f6f29c08f80b) #master) - set(suite_dir ${tsdir}/test) + GIT_TAG bcd49a2d4919c1b1ac3b9d6e5ebe6b140b5089e3) #master) + set(suite_dir ${tsdir}/src) if(NOT EXISTS ${suite_dir}) c4_err("cannot find yaml-test-suite at ${suite_dir} -- was there an error downloading the project?") endif() @@ -170,7 +170,7 @@ if(RYML_TEST_SUITE) add_test(NAME ryml-test-suite-${name}-in_json COMMAND ${tgt} --gtest_filter=*/in_json* ${suite_dir}/${tml_file}) endfunction() - file(GLOB suite_cases RELATIVE "${suite_dir}" "${suite_dir}/*.tml") + file(GLOB suite_cases RELATIVE "${suite_dir}" "${suite_dir}/*.yaml") foreach(case ${suite_cases}) ryml_add_test_from_suite(${case}) endforeach() diff --git a/test/test_suite.cpp b/test/test_suite.cpp index cc3833761..c2cd7a992 100644 --- a/test/test_suite.cpp +++ b/test/test_suite.cpp @@ -240,7 +240,7 @@ struct Approach RYML_ASSERT(events_part(CPART_OUT_YAML) == CPART_OUT_YAML_EVENTS); RYML_ASSERT(events_part(CPART_EMIT_YAML) == CPART_EMIT_YAML_EVENTS); // use bit-or to ensure calling both, so that both report a skip - return skip_part() | allowed_failure_events.skip(events_part(case_part)); + return (bool)((int)skip_part() | (int)allowed_failure_events.skip(events_part(case_part))); } void parse(size_t num, bool emit) @@ -358,43 +358,73 @@ struct Subject // some utility functions, used below -size_t find_first_after(size_t pos, std::initializer_list candidates) +void check_known_keys(csubstr filename, NodeRef const spec) { - size_t ret = npos; - for(size_t s : candidates) - if(s > pos && s < ret) - ret = s; - return ret; + for(auto node : spec.children()) + { + csubstr k = node.key(); + if(k == "name") + ; + else if(k == "from") + ; + else if(k == "yaml") + ; + else if(k == "tags") + ; + else if(k == "tree") + ; + else if(k == "json") + ; + else if(k == "dump") + ; + else if(k == "emit") + ; + else if(k == "fail") + ; + else if(k == "toke") + ; + else + C4_ERROR("%.*s: unknown tag '%.*s'", + (int)filename.len, filename.str, + (int)k.len, k.str); + } } -csubstr filter_out_indentation(csubstr src, std::string *dst) +struct SpecialCharsFilter { - if( ! src.begins_with(" ")) + std::string tmpa; + std::string tmpb; + std::string *current = &tmpa; + csubstr _do_replace(csubstr pattern, csubstr repl, csubstr subject) { - dst->assign(src.begin(), src.end()); - return c4::to_csubstr(*dst); + subject = replace_all(pattern, repl, subject, current); + if(current == &tmpa) + current = &tmpb; + else + current = &tmpa; + return subject; } - auto has_meta_comments = [](csubstr s) { - for(csubstr line : s.split('\n')) - if(line.begins_with('#')) - return true; - return false; - }; - std::string tmp; - if(has_meta_comments(src)) + // https://github.com/yaml/yaml-test-suite#special-characters + csubstr replace_normal(csubstr txt) { - tmp.reserve(src.size()); - for(csubstr line : src.split('\n')) - { - if(line.begins_with('#')) - continue; - tmp.append(line.begin(), line.end()); - tmp += '\n'; - } - src = c4::to_csubstr(tmp); + txt = _do_replace("␣", " ", txt); + txt = _do_replace("———»", "\t", txt); + txt = _do_replace("——»", "\t", txt); + txt = _do_replace("—»", "\t", txt); + txt = _do_replace("»", "\t", txt); + txt = _do_replace("↵", "\n", txt); + txt = _do_replace("←", "\r", txt); + txt = _do_replace("∎", "", txt); + txt = _do_replace("⇔", "\xef\xbb\xbf", txt); // byte order mark 0xef 0xbb 0xbf + return txt; } - return replace_all("\n ", "\n", src.sub(4), dst); -} + csubstr replace_events(csubstr txt) + { + txt = _do_replace("", " ", txt); + txt = _do_replace("", "\t", txt); + return txt; + } +}; /** all the ways that a test case can be processed are @@ -405,6 +435,7 @@ struct SuiteCase csubstr filename; std::string file_contents; + Tree tree; csubstr desc; csubstr from; csubstr tags; @@ -422,8 +453,7 @@ struct SuiteCase return c4::to_csubstr(s.unix_ro.levels[0].src); } - /** loads the several types of tests from an input test suite - * template file (tml)*/ + /** loads the several types of tests from an input test suite file */ SuiteCase(const char* filename_) { filename = c4::to_csubstr(filename_); @@ -431,153 +461,71 @@ struct SuiteCase // read the file c4::fs::file_get_contents(filename_, &file_contents); csubstr contents = c4::to_csubstr(file_contents); + #if RYML_NFO + _nfo_logf("contents:\n~~~{}~~~", contents); + #endif // now parse the file - csubstr ws = " \t\r\n"; - csubstr txt; - size_t b, e; - - // desc - RYML_CHECK(contents.begins_with("=== ")); - e = contents.find("--- from: ", 4); - RYML_CHECK(e != npos); - desc = contents.range(4, e).trimr(ws); - - // from - b = e + 4; - e = contents.find("--- tags: ", b); - RYML_CHECK(e != npos); - from = contents.range(b, e); - RYML_CHECK(from.begins_with("from: ")); - RYML_CHECK(from.size() >= 6); - from = from.sub(6).trimr(ws); - - // tags - b = e + 4; - e = contents.find("--- in-yaml", b); - RYML_CHECK(e != npos); - tags = contents.range(b, e); - RYML_CHECK(tags.begins_with("tags: ")); - RYML_CHECK(tags.size() >= 6); - tags = tags.sub(6).trimr(ws); - - expect_error = (tags.find("error") != npos); - + RYML_CHECK(contents.begins_with("---")); + parse(filename, contents, &tree); + #if RYML_NFO + c4::print("parsed:"); print_tree(tree); + #endif + NodeRef spec = tree.docref(0)[0]; + check_known_keys(filename, spec); + desc = spec["name"].val(); + from = spec["from"].val(); + tags = spec["tags"].val(); bool has_whitespace = tags.find("whitespace"); + expect_error = (tags.find("error") != npos); + if(spec.has_child("fail")) + { + bool fail = false; + spec["fail"] >> fail; + if(expect_error) + C4_CHECK(fail == expect_error); + } - size_t end_tags = e; - size_t begin_in_yaml = contents.find("--- in-yaml" , end_tags); - size_t begin_error = contents.find("--- error" , end_tags); - size_t begin_in_json = contents.find("--- in-json" , end_tags); - size_t begin_out_yaml = contents.find("--- out-yaml" , end_tags); - size_t begin_emit_yaml = contents.find("--- emit-yaml" , end_tags); - size_t begin_events = contents.find("--- test-event", end_tags); - size_t lex_token = contents.find("--- lex-token" , end_tags); - auto did_not_slurp_other_tml_tokens = [](csubstr part){ - csubstr tokens[] = {"--- in-yaml", "--- error", "--- in-json", "--- out-yaml", "--- emit-yaml", "---test-event", "--- lex-token"}; - return ! part.first_of_any_iter(std::begin(tokens), std::end(tokens)); - }; - std::initializer_list all = { - begin_in_yaml, - begin_error, - begin_in_json, - begin_out_yaml, - begin_emit_yaml, - begin_events, - lex_token, - contents.size() - }; - - // some of the examples have their code indented, - // so we need these workspaces for deindenting - std::string tmpa; - std::string tmpb; + SpecialCharsFilter filter; + csubstr txt; // in_yaml - RYML_CHECK(begin_in_yaml != npos); - size_t first_after_in_yaml = find_first_after(begin_in_yaml, all); - begin_in_yaml = 1 + contents.find('\n', begin_in_yaml); // skip this line - txt = contents.range(begin_in_yaml, first_after_in_yaml); - RYML_CHECK(did_not_slurp_other_tml_tokens(txt)); - txt = filter_out_indentation(txt, &tmpa); + txt = spec["yaml"].val(); if(has_whitespace) - { - txt = replace_all("", " ", txt, &tmpb); - txt = replace_all("", "\t", txt, &tmpa); - } + txt = filter.replace_normal(txt); in_yaml.init(filename, txt, CPART_IN_YAML, expect_error); - // error - if(begin_error != npos) - { - size_t first_after = find_first_after(begin_error, all); - begin_error = 1 + contents.find('\n', begin_error); // skip this line - txt = contents.range(begin_error, first_after); - RYML_CHECK(did_not_slurp_other_tml_tokens(txt)); - txt = filter_out_indentation(txt, &tmpa); - } - // in_json - if(begin_in_json != npos) + if(spec.has_child("json")) { - size_t first_after = find_first_after(begin_in_json, all); - begin_in_json = 1 + contents.find('\n', begin_in_json); // skip this line - txt = contents.range(begin_in_json, first_after); - RYML_CHECK(did_not_slurp_other_tml_tokens(txt)); + txt = spec["json"].val(); in_json.init(filename, txt, CPART_IN_JSON, expect_error); } // out_yaml - if(begin_out_yaml != npos) + if(spec.has_child("dump")) { - size_t first_after = find_first_after(begin_out_yaml, all); - begin_out_yaml = 1 + contents.find('\n', begin_out_yaml); // skip this line - txt = contents.range(begin_out_yaml, first_after); - RYML_CHECK(did_not_slurp_other_tml_tokens(txt)); - txt = filter_out_indentation(txt, &tmpa); + txt = spec["dump"].val(); if(has_whitespace) - { - txt = replace_all("", " ", txt, &tmpb); - txt = replace_all("", "\t", txt, &tmpa); - } + txt = filter.replace_normal(txt); out_yaml.init(filename, txt, CPART_OUT_YAML, expect_error); } // emit_yaml - if(begin_emit_yaml != npos) + if(spec.has_child("emit")) { - size_t first_after = find_first_after(begin_emit_yaml, all); - begin_emit_yaml = 1 + contents.find('\n', begin_emit_yaml); // skip this line - txt = contents.range(begin_emit_yaml, first_after); - RYML_CHECK(did_not_slurp_other_tml_tokens(txt)); - txt = filter_out_indentation(txt, &tmpa); + txt = spec["emit"].val(); if(has_whitespace) - { - txt = replace_all("", " ", txt, &tmpb); - txt = replace_all("", "\t", txt, &tmpa); - } + txt = filter.replace_normal(txt); emit_yaml.init(filename, txt, CPART_EMIT_YAML, expect_error); } // events - { - RYML_CHECK(begin_events != npos); - size_t first_after = find_first_after(begin_events, all); - begin_events = 1 + contents.find('\n', begin_events); // skip this line - txt = contents.range(begin_events, first_after); - RYML_CHECK(did_not_slurp_other_tml_tokens(txt)); - if(has_whitespace) - { - txt = replace_all("", " ", txt, &tmpb); - txt = replace_all("", "\t", txt, &tmpa); - } - events.init(filename, txt); - } - - // lex-token - { - // don't really care - } + C4_CHECK(spec.has_child("tree")); + txt = spec["tree"].val(); + if(has_whitespace) + txt = filter.replace_events(txt); + events.init(filename, txt); } void print() const diff --git a/test/test_suite/test_suite_events.cpp b/test/test_suite/test_suite_events.cpp index 40d773c5b..049313273 100644 --- a/test/test_suite/test_suite_events.cpp +++ b/test/test_suite/test_suite_events.cpp @@ -232,6 +232,7 @@ void EventsParser::parse(csubstr src, Tree *C4_RESTRICT tree_) for(csubstr line : src.split('\n')) { line = line.trimr('\r'); + line = line.triml(' '); _nfo_printf("\n\n-----------------------\n"); { size_t curr = m_stack.empty() ? tree.root_id() : m_stack.top().tree_node;