diff --git a/DEPENDENCIES b/DEPENDENCIES index e5a71de1..d9c23c77 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -4,4 +4,4 @@ jsontoolkit https://github.com/sourcemeta/jsontoolkit 7a398224cc2e76ea9ae8541a87 hydra https://github.com/sourcemeta/hydra a4a74f3cabd32f2f829f449d67339dac33f9910e alterschema https://github.com/sourcemeta/alterschema 92e370ce9c1f0582014b54d43e388ee012dfe13d jsonbinpack https://github.com/sourcemeta/jsonbinpack d777179441d3c703e1fda1187742541aa26836b5 -blaze https://github.com/sourcemeta/blaze cf0c89cd419ffb70cc334d395ac5ab1035702e30 +blaze https://github.com/sourcemeta/blaze a5b3c8e4d77a0b88e4a93f304ae75e711b30a2e6 diff --git a/docs/metaschema.markdown b/docs/metaschema.markdown index f068536c..c12c6601 100644 --- a/docs/metaschema.markdown +++ b/docs/metaschema.markdown @@ -7,7 +7,7 @@ Metaschema ```sh jsonschema metaschema [schemas-or-directories...] [--verbose/-v] [--http/-h] [--extension/-e ] - [--ignore/-i ] + [--ignore/-i ] [--trace/-t] ``` Ensure that a schema or a set of schemas are considered valid with regards to @@ -76,3 +76,9 @@ jsonschema metaschema path/to/schemas/ --ignore path/to/schemas/nested ```sh jsonschema metaschema --extension .schema.json ``` + +### Validate the metaschema of a JSON Schema with trace information + +```sh +jsonschema metaschema path/to/my/schema.json --trace +``` diff --git a/docs/validate.markdown b/docs/validate.markdown index 0eabc0a8..9eb529c8 100644 --- a/docs/validate.markdown +++ b/docs/validate.markdown @@ -7,7 +7,7 @@ Validating ```sh jsonschema validate [--http/-h] [--verbose/-v] [--resolve/-r ...] [--benchmark/-b] - [--extension/-e ] [--ignore/-i ] + [--extension/-e ] [--ignore/-i ] [--trace/-t] ``` The most popular use case of JSON Schema is to validate JSON documents. The @@ -94,3 +94,9 @@ jsonschema validate path/to/my/schema.json path/to/my/instance.json \ ```sh jsonschema validate path/to/my/schema.json path/to/my/instance.json --benchmark ``` + +### Validate a JSON instance against a schema with trace information + +```sh +jsonschema validate path/to/my/schema.json path/to/my/instance.json --trace +``` diff --git a/src/command_metaschema.cc b/src/command_metaschema.cc index c2e80a7b..2c149662 100644 --- a/src/command_metaschema.cc +++ b/src/command_metaschema.cc @@ -16,7 +16,8 @@ // TODO: Add a flag to emit output using the standard JSON Schema output format auto sourcemeta::jsonschema::cli::metaschema( const std::span &arguments) -> int { - const auto options{parse_options(arguments, {"h", "http"})}; + const auto options{parse_options(arguments, {"h", "http", "t", "trace"})}; + const auto trace{options.contains("t") || options.contains("trace")}; const auto custom_resolver{ resolver(options, options.contains("h") || options.contains("http"))}; bool result{true}; @@ -44,18 +45,25 @@ auto sourcemeta::jsonschema::cli::metaschema( cache.insert({dialect.value(), metaschema_template}); } - sourcemeta::blaze::ErrorOutput output{entry.second}; - if (sourcemeta::blaze::evaluate(cache.at(dialect.value()), entry.second, - std::ref(output))) { - log_verbose(options) - << "ok: " << std::filesystem::weakly_canonical(entry.first).string() - << "\n matches " << dialect.value() << "\n"; + if (trace) { + sourcemeta::blaze::TraceOutput output; + result = sourcemeta::blaze::evaluate(cache.at(dialect.value()), + entry.second, std::ref(output)); + print(output, std::cout); } else { - std::cerr << "fail: " - << std::filesystem::weakly_canonical(entry.first).string() - << "\n"; - print(output, std::cerr); - result = false; + sourcemeta::blaze::ErrorOutput output{entry.second}; + if (sourcemeta::blaze::evaluate(cache.at(dialect.value()), entry.second, + std::ref(output))) { + log_verbose(options) + << "ok: " << std::filesystem::weakly_canonical(entry.first).string() + << "\n matches " << dialect.value() << "\n"; + } else { + std::cerr << "fail: " + << std::filesystem::weakly_canonical(entry.first).string() + << "\n"; + print(output, std::cerr); + result = false; + } } } diff --git a/src/command_validate.cc b/src/command_validate.cc index b22449c3..c77f045a 100644 --- a/src/command_validate.cc +++ b/src/command_validate.cc @@ -19,7 +19,8 @@ // TODO: Add a flag to take a pre-compiled schema as input auto sourcemeta::jsonschema::cli::validate( const std::span &arguments) -> int { - const auto options{parse_options(arguments, {"h", "http", "b", "benchmark"})}; + const auto options{ + parse_options(arguments, {"h", "http", "b", "benchmark", "t", "trace"})}; if (options.at("").size() < 1) { std::cerr @@ -51,6 +52,7 @@ auto sourcemeta::jsonschema::cli::validate( } const auto benchmark{options.contains("b") || options.contains("benchmark")}; + const auto trace{options.contains("t") || options.contains("trace")}; const auto schema_template{sourcemeta::blaze::compile( schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver, sourcemeta::blaze::default_schema_compiler)}; @@ -73,6 +75,7 @@ auto sourcemeta::jsonschema::cli::validate( index += 1; std::ostringstream error; sourcemeta::blaze::ErrorOutput output{instance}; + sourcemeta::blaze::TraceOutput trace_output; bool subresult = true; if (benchmark) { const auto timestamp_start{ @@ -87,12 +90,18 @@ auto sourcemeta::jsonschema::cli::validate( } else { error << "error: Schema validation failure\n"; } + } else if (trace) { + subresult = sourcemeta::blaze::evaluate(schema_template, instance, + std::ref(trace_output)); } else { subresult = sourcemeta::blaze::evaluate(schema_template, instance, std::ref(output)); } - if (subresult) { + if (trace) { + print(trace_output, std::cout); + result = subresult; + } else if (subresult) { log_verbose(options) << "ok: " << std::filesystem::weakly_canonical(instance_path).string() @@ -125,6 +134,7 @@ auto sourcemeta::jsonschema::cli::validate( const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)}; std::ostringstream error; sourcemeta::blaze::ErrorOutput output{instance}; + sourcemeta::blaze::TraceOutput trace_output; bool subresult{true}; if (benchmark) { const auto timestamp_start{std::chrono::high_resolution_clock::now()}; @@ -139,12 +149,18 @@ auto sourcemeta::jsonschema::cli::validate( error << "error: Schema validation failure\n"; result = false; } + } else if (trace) { + subresult = sourcemeta::blaze::evaluate(schema_template, instance, + std::ref(trace_output)); } else { subresult = sourcemeta::blaze::evaluate(schema_template, instance, std::ref(output)); } - if (subresult) { + if (trace) { + print(trace_output, std::cout); + result = subresult; + } else if (subresult) { log_verbose(options) << "ok: " << std::filesystem::weakly_canonical(instance_path).string() diff --git a/src/main.cc b/src/main.cc index a11781d3..e75e5e0e 100644 --- a/src/main.cc +++ b/src/main.cc @@ -23,13 +23,13 @@ Global Options: validate [--http/-h] [--benchmark/-b] [--extension/-e ] - [--ignore/-i ] + [--ignore/-i ] [--trace/-t] Validate one of more instances against the given schema. metaschema [schemas-or-directories...] [--http/-h] [--extension/-e ] - [--ignore/-i ] + [--ignore/-i ] [--trace/-t] Validate that a schema or a set of schemas are valid with respect to their metaschemas. diff --git a/src/utils.cc b/src/utils.cc index b03b5844..60a9ecc1 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -189,6 +189,37 @@ auto print(const sourcemeta::blaze::ErrorOutput &output, std::ostream &stream) } } +auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream) + -> void { + for (const auto &entry : output) { + if (entry.evaluate_path.empty()) { + continue; + } + + switch (entry.type) { + case sourcemeta::blaze::TraceOutput::EntryType::Push: + stream << "-> (push) "; + break; + case sourcemeta::blaze::TraceOutput::EntryType::Pass: + stream << "<- (pass) "; + break; + case sourcemeta::blaze::TraceOutput::EntryType::Fail: + stream << "<- (fail) "; + break; + default: + assert(false); + break; + } + + stream << "\""; + sourcemeta::jsontoolkit::stringify(entry.evaluate_path, stream); + stream << "\"\n"; + stream << " at \""; + sourcemeta::jsontoolkit::stringify(entry.instance_location, stream); + stream << "\"\n"; + } +} + static auto fallback_resolver( const std::map> &options, std::string_view identifier) diff --git a/src/utils.h b/src/utils.h index 9103c2c4..aa1dc214 100644 --- a/src/utils.h +++ b/src/utils.h @@ -32,6 +32,9 @@ auto for_each_json(const std::vector &arguments, auto print(const sourcemeta::blaze::ErrorOutput &output, std::ostream &stream) -> void; +auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream) + -> void; + auto resolver(const std::map> &options, const bool remote = false) -> sourcemeta::jsontoolkit::SchemaResolver; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 75f744c4..018d88c6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -42,6 +42,8 @@ add_jsonschema_test_unix(validate/fail_schema_enoent) add_jsonschema_test_unix(validate/fail_schema_invalid_json) add_jsonschema_test_unix(validate/fail_schema_non_schema) add_jsonschema_test_unix(validate/fail_schema_unknown_dialect) +add_jsonschema_test_unix(validate/fail_trace) +add_jsonschema_test_unix(validate/pass_trace) add_jsonschema_test_unix(validate/pass_resolve) add_jsonschema_test_unix(validate/pass_resolve_custom_extension) add_jsonschema_test_unix(validate/pass_resolve_verbose) @@ -71,6 +73,8 @@ add_jsonschema_test_unix(validate/fail_many) add_jsonschema_test_unix(validate/fail_many_verbose) # Metaschema +add_jsonschema_test_unix(metaschema/pass_trace) +add_jsonschema_test_unix(metaschema/fail_trace) add_jsonschema_test_unix(metaschema/fail_directory) add_jsonschema_test_unix(metaschema/fail_single) add_jsonschema_test_unix(metaschema/fail_non_schema) diff --git a/test/metaschema/fail_trace.sh b/test/metaschema/fail_trace.sh new file mode 100755 index 00000000..463cc1c2 --- /dev/null +++ b/test/metaschema/fail_trace.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "minimum": "foo" +} +EOF + +"$1" metaschema "$TMP/schema.json" --trace > "$TMP/output.txt" \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +# Order of execution can vary + +cat << 'EOF' > "$TMP/expected-1.txt" +-> (push) "/dependencies" + at "" +<- (pass) "/dependencies" + at "" +-> (push) "/properties" + at "" +-> (push) "/properties/$schema/type" + at "/$schema" +<- (pass) "/properties/$schema/type" + at "/$schema" +-> (push) "/properties/minimum/type" + at "/minimum" +<- (fail) "/properties/minimum/type" + at "/minimum" +<- (fail) "/properties" + at "" +EOF + +cat << 'EOF' > "$TMP/expected-2.txt" +-> (push) "/dependencies" + at "" +<- (pass) "/dependencies" + at "" +-> (push) "/properties" + at "" +-> (push) "/properties/minimum/type" + at "/minimum" +<- (fail) "/properties/minimum/type" + at "/minimum" +<- (fail) "/properties" + at "" +EOF + +diff "$TMP/output.txt" "$TMP/expected-1.txt" || \ + diff "$TMP/output.txt" "$TMP/expected-2.txt" diff --git a/test/metaschema/pass_trace.sh b/test/metaschema/pass_trace.sh new file mode 100755 index 00000000..7bd0083f --- /dev/null +++ b/test/metaschema/pass_trace.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ "$schema": "http://json-schema.org/draft-04/schema#" } +EOF + +"$1" metaschema "$TMP/schema.json" --trace > "$TMP/output.txt" + +cat << 'EOF' > "$TMP/expected.txt" +-> (push) "/dependencies" + at "" +<- (pass) "/dependencies" + at "" +-> (push) "/properties" + at "" +-> (push) "/properties/$schema/type" + at "/$schema" +<- (pass) "/properties/$schema/type" + at "/$schema" +<- (pass) "/properties" + at "" +-> (push) "/type" + at "" +<- (pass) "/type" + at "" +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/validate/fail_trace.sh b/test/validate/fail_trace.sh new file mode 100755 index 00000000..b2aa83cc --- /dev/null +++ b/test/validate/fail_trace.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "type": "string" + } + } +} +EOF + +cat << 'EOF' > "$TMP/instance.json" +{ "foo": 1 } +EOF + +"$1" validate "$TMP/schema.json" "$TMP/instance.json" --trace > "$TMP/output.txt" \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.txt" +-> (push) "/properties/foo/type" + at "/foo" +<- (fail) "/properties/foo/type" + at "/foo" +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/validate/pass_trace.sh b/test/validate/pass_trace.sh new file mode 100755 index 00000000..7c85cf08 --- /dev/null +++ b/test/validate/pass_trace.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "type": "string" + } + } +} +EOF + +cat << 'EOF' > "$TMP/instance.json" +{ "foo": "bar" } +EOF + +"$1" validate "$TMP/schema.json" "$TMP/instance.json" --trace > "$TMP/output.txt" + +cat << EOF > "$TMP/expected.txt" +-> (push) "/properties/foo/type" + at "/foo" +<- (pass) "/properties/foo/type" + at "/foo" +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/vendor/blaze/src/compiler/CMakeLists.txt b/vendor/blaze/src/compiler/CMakeLists.txt index e3c40403..9e6537a3 100644 --- a/vendor/blaze/src/compiler/CMakeLists.txt +++ b/vendor/blaze/src/compiler/CMakeLists.txt @@ -3,7 +3,8 @@ noa_library(NAMESPACE sourcemeta PROJECT blaze NAME compiler PRIVATE_HEADERS error.h output.h SOURCES compile.cc compile_json.cc compile_describe.cc - compile_output.cc compile_helpers.h default_compiler.cc + compile_output_error.cc compile_output_trace.cc + compile_helpers.h default_compiler.cc default_compiler_2020_12.h default_compiler_2019_09.h default_compiler_draft7.h diff --git a/vendor/blaze/src/compiler/compile_output.cc b/vendor/blaze/src/compiler/compile_output_error.cc similarity index 100% rename from vendor/blaze/src/compiler/compile_output.cc rename to vendor/blaze/src/compiler/compile_output_error.cc diff --git a/vendor/blaze/src/compiler/compile_output_trace.cc b/vendor/blaze/src/compiler/compile_output_trace.cc new file mode 100644 index 00000000..23b67ab7 --- /dev/null +++ b/vendor/blaze/src/compiler/compile_output_trace.cc @@ -0,0 +1,79 @@ +#include + +#include + +#include // std::move +#include // std::visit + +#ifdef __clang__ +#include // abi::__cxa_demangle +#include // std::free +static auto step_name(const sourcemeta::blaze::Template::value_type &step) + -> std::string { + return std::visit( + [](const auto &value) { + int status; + std::string name{typeid(value).name()}; + char *demangled = + abi::__cxa_demangle(name.c_str(), nullptr, nullptr, &status); + if (demangled) { + name = demangled; + std::free(demangled); + } + + return name; + }, + step); +} +#else +static auto step_name(const sourcemeta::blaze::Template::value_type &) + -> std::string { + // TODO: Properly implement for GCC and MSVC + return "????"; +} +#endif + +namespace sourcemeta::blaze { + +TraceOutput::TraceOutput(const sourcemeta::jsontoolkit::WeakPointer &base) + : base_{base} {} + +auto TraceOutput::begin() const -> const_iterator { + return this->output.begin(); +} + +auto TraceOutput::end() const -> const_iterator { return this->output.end(); } + +auto TraceOutput::cbegin() const -> const_iterator { + return this->output.cbegin(); +} + +auto TraceOutput::cend() const -> const_iterator { return this->output.cend(); } + +auto TraceOutput::operator()( + const EvaluationType type, const bool result, + const Template::value_type &step, + const sourcemeta::jsontoolkit::WeakPointer &evaluate_path, + const sourcemeta::jsontoolkit::WeakPointer &instance_location, + const sourcemeta::jsontoolkit::JSON &) -> void { + const std::string step_prefix{"sourcemeta::blaze::"}; + const auto full_step_name{step_name(step)}; + const auto short_step_name{full_step_name.starts_with(step_prefix) + ? full_step_name.substr(step_prefix.size()) + : full_step_name}; + + auto effective_evaluate_path{evaluate_path.resolve_from(this->base_)}; + + if (type == EvaluationType::Pre) { + this->output.push_back({EntryType::Push, short_step_name, instance_location, + std::move(effective_evaluate_path)}); + } else if (result) { + this->output.push_back({EntryType::Pass, short_step_name, instance_location, + std::move(effective_evaluate_path)}); + } else { + this->output.push_back({EntryType::Fail, short_step_name, instance_location, + std::move(effective_evaluate_path)}); + } +} + +} // namespace sourcemeta::blaze diff --git a/vendor/blaze/src/compiler/include/sourcemeta/blaze/compiler_output.h b/vendor/blaze/src/compiler/include/sourcemeta/blaze/compiler_output.h index cfd3fa40..9dcfa8a4 100644 --- a/vendor/blaze/src/compiler/include/sourcemeta/blaze/compiler_output.h +++ b/vendor/blaze/src/compiler/include/sourcemeta/blaze/compiler_output.h @@ -44,16 +44,16 @@ namespace sourcemeta::blaze { /// /// const sourcemeta::jsontoolkit::JSON instance{5}; /// -/// sourcemeta::blaze::ErrorOutput output; +/// sourcemeta::blaze::ErrorOutput output{instance}; /// const auto result{sourcemeta::blaze::evaluate( /// schema_template, instance, std::ref(output))}; /// /// if (!result) { -/// for (const auto &trace : output) { -/// std::cerr << trace.message << "\n"; -/// sourcemeta::jsontoolkit::stringify(trace.instance_location, std::cerr); +/// for (const auto &entry : output) { +/// std::cerr << entry.message << "\n"; +/// sourcemeta::jsontoolkit::stringify(entry.instance_location, std::cerr); /// std::cerr << "\n"; -/// sourcemeta::jsontoolkit::stringify(trace.evaluate_path, std::cerr); +/// sourcemeta::jsontoolkit::stringify(entry.evaluate_path, std::cerr); /// std::cerr << "\n"; /// } /// } @@ -103,6 +103,104 @@ class SOURCEMETA_BLAZE_COMPILER_EXPORT ErrorOutput { #endif }; +/// @ingroup compiler +/// +/// An evaluation callback that reports a trace of execution. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// #include +/// #include +/// +/// #include +/// #include +/// +/// const sourcemeta::jsontoolkit::JSON schema = +/// sourcemeta::jsontoolkit::parse(R"JSON({ +/// "$schema": "https://json-schema.org/draft/2020-12/schema", +/// "type": "string" +/// })JSON"); +/// +/// const auto schema_template{sourcemeta::blaze::compile( +/// schema, sourcemeta::jsontoolkit::default_schema_walker, +/// sourcemeta::jsontoolkit::official_resolver, +/// sourcemeta::jsontoolkit::default_schema_compiler)}; +/// +/// const sourcemeta::jsontoolkit::JSON instance{5}; +/// +/// sourcemeta::blaze::TraceOutput output; +/// const auto result{sourcemeta::blaze::evaluate( +/// schema_template, instance, std::ref(output))}; +/// +/// if (!result) { +/// for (const auto &entry : output) { +// switch (entry.type) { +// case sourcemeta::blaze::TraceOutput::EntryType::Push: +// std::cerr << "-> (push) "; +// break; +// case sourcemeta::blaze::TraceOutput::EntryType::Pass: +// std::cerr << "<- (pass) "; +// break; +// case sourcemeta::blaze::TraceOutput::EntryType::Fail: +// std::cerr << "<- (fail) "; +// break; +// } +/// +/// std::cerr << entry.name << "\n"; +/// sourcemeta::jsontoolkit::stringify(entry.instance_location, std::cerr); +/// std::cerr << "\n"; +/// sourcemeta::jsontoolkit::stringify(entry.evaluate_path, std::cerr); +/// std::cerr << "\n"; +/// } +/// } +/// ``` +class SOURCEMETA_BLAZE_COMPILER_EXPORT TraceOutput { +public: + TraceOutput(const sourcemeta::jsontoolkit::WeakPointer &base = + sourcemeta::jsontoolkit::empty_weak_pointer); + + // Prevent accidental copies + TraceOutput(const ErrorOutput &) = delete; + auto operator=(const TraceOutput &) -> TraceOutput & = delete; + + enum class EntryType { Push, Pass, Fail }; + + struct Entry { + const EntryType type; + const std::string name; + const sourcemeta::jsontoolkit::WeakPointer instance_location; + const sourcemeta::jsontoolkit::WeakPointer evaluate_path; + }; + + auto operator()(const EvaluationType type, const bool result, + const Template::value_type &step, + const sourcemeta::jsontoolkit::WeakPointer &evaluate_path, + const sourcemeta::jsontoolkit::WeakPointer &instance_location, + const sourcemeta::jsontoolkit::JSON &annotation) -> void; + + using container_type = typename std::vector; + using const_iterator = typename container_type::const_iterator; + auto begin() const -> const_iterator; + auto end() const -> const_iterator; + auto cbegin() const -> const_iterator; + auto cend() const -> const_iterator; + +private: +// Exporting symbols that depends on the standard C++ library is considered +// safe. +// https://learn.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-2-c4275?view=msvc-170&redirectedfrom=MSDN +#if defined(_MSC_VER) +#pragma warning(disable : 4251) +#endif + const sourcemeta::jsontoolkit::WeakPointer base_; + container_type output; +#if defined(_MSC_VER) +#pragma warning(default : 4251) +#endif +}; + /// @ingroup compiler /// /// This function translates a step execution into a human-readable string.