Add a --trace option for validation-related commands

Signed-off-by: Juan Cruz Viotti <[email protected]>
sourcemeta · Nov 7, 2024 · 66e8d2a · 66e8d2a
1 parent 4db93a1
commit 66e8d2a
Show file tree

Hide file tree

Showing 17 changed files with 441 additions and 26 deletions.
diff --git a/DEPENDENCIES b/DEPENDENCIES
@@ -4,4 +4,4 @@ jsontoolkit https://github.com/sourcemeta/jsontoolkit 7a398224cc2e76ea9ae8541a87
 hydra https://github.com/sourcemeta/hydra a4a74f3cabd32f2f829f449d67339dac33f9910e
 alterschema https://github.com/sourcemeta/alterschema 92e370ce9c1f0582014b54d43e388ee012dfe13d
 jsonbinpack https://github.com/sourcemeta/jsonbinpack d777179441d3c703e1fda1187742541aa26836b5
-blaze https://github.com/sourcemeta/blaze cf0c89cd419ffb70cc334d395ac5ab1035702e30
+blaze https://github.com/sourcemeta/blaze a5b3c8e4d77a0b88e4a93f304ae75e711b30a2e6
diff --git a/docs/metaschema.markdown b/docs/metaschema.markdown
@@ -7,7 +7,7 @@ Metaschema
 ```sh
 jsonschema metaschema [schemas-or-directories...]
   [--verbose/-v] [--http/-h] [--extension/-e <extension>]
-  [--ignore/-i <schemas-or-directories>]
+  [--ignore/-i <schemas-or-directories>] [--trace/-t]
 ```
 
 Ensure that a schema or a set of schemas are considered valid with regards to
@@ -76,3 +76,9 @@ jsonschema metaschema path/to/schemas/ --ignore path/to/schemas/nested
 ```sh
 jsonschema metaschema --extension .schema.json
 ```
+
+### Validate the metaschema of a JSON Schema with trace information
+
+```sh
+jsonschema metaschema path/to/my/schema.json --trace
+```
diff --git a/docs/validate.markdown b/docs/validate.markdown
@@ -7,7 +7,7 @@ Validating
 ```sh
 jsonschema validate <schema.json> <instance.json|.jsonl...> [--http/-h]
   [--verbose/-v] [--resolve/-r <schemas-or-directories> ...] [--benchmark/-b]
-  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>]
+  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>] [--trace/-t]
 ```
 
 The most popular use case of JSON Schema is to validate JSON documents. The
@@ -94,3 +94,9 @@ jsonschema validate path/to/my/schema.json path/to/my/instance.json \
 ```sh
 jsonschema validate path/to/my/schema.json path/to/my/instance.json --benchmark
 ```
+
+### Validate a JSON instance against a schema with trace information
+
+```sh
+jsonschema validate path/to/my/schema.json path/to/my/instance.json --trace
+```
diff --git a/src/command_metaschema.cc b/src/command_metaschema.cc
@@ -16,7 +16,8 @@
 // TODO: Add a flag to emit output using the standard JSON Schema output format
 auto sourcemeta::jsonschema::cli::metaschema(
     const std::span<const std::string> &arguments) -> int {
-  const auto options{parse_options(arguments, {"h", "http"})};
+  const auto options{parse_options(arguments, {"h", "http", "t", "trace"})};
+  const auto trace{options.contains("t") || options.contains("trace")};
   const auto custom_resolver{
       resolver(options, options.contains("h") || options.contains("http"))};
   bool result{true};
@@ -44,18 +45,25 @@ auto sourcemeta::jsonschema::cli::metaschema(
       cache.insert({dialect.value(), metaschema_template});
     }
 
-    sourcemeta::blaze::ErrorOutput output{entry.second};
-    if (sourcemeta::blaze::evaluate(cache.at(dialect.value()), entry.second,
-                                    std::ref(output))) {
-      log_verbose(options)
-          << "ok: " << std::filesystem::weakly_canonical(entry.first).string()
-          << "\n  matches " << dialect.value() << "\n";
+    if (trace) {
+      sourcemeta::blaze::TraceOutput output;
+      result = sourcemeta::blaze::evaluate(cache.at(dialect.value()),
+                                           entry.second, std::ref(output));
+      print(output, std::cout);
     } else {
-      std::cerr << "fail: "
-                << std::filesystem::weakly_canonical(entry.first).string()
-                << "\n";
-      print(output, std::cerr);
-      result = false;
+      sourcemeta::blaze::ErrorOutput output{entry.second};
+      if (sourcemeta::blaze::evaluate(cache.at(dialect.value()), entry.second,
+                                      std::ref(output))) {
+        log_verbose(options)
+            << "ok: " << std::filesystem::weakly_canonical(entry.first).string()
+            << "\n  matches " << dialect.value() << "\n";
+      } else {
+        std::cerr << "fail: "
+                  << std::filesystem::weakly_canonical(entry.first).string()
+                  << "\n";
+        print(output, std::cerr);
+        result = false;
+      }
     }
   }
 

diff --git a/src/command_validate.cc b/src/command_validate.cc
@@ -19,7 +19,8 @@
 // TODO: Add a flag to take a pre-compiled schema as input
 auto sourcemeta::jsonschema::cli::validate(
     const std::span<const std::string> &arguments) -> int {
-  const auto options{parse_options(arguments, {"h", "http", "b", "benchmark"})};
+  const auto options{
+      parse_options(arguments, {"h", "http", "b", "benchmark", "t", "trace"})};
 
   if (options.at("").size() < 1) {
     std::cerr
@@ -51,6 +52,7 @@ auto sourcemeta::jsonschema::cli::validate(
   }
 
   const auto benchmark{options.contains("b") || options.contains("benchmark")};
+  const auto trace{options.contains("t") || options.contains("trace")};
   const auto schema_template{sourcemeta::blaze::compile(
       schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver,
       sourcemeta::blaze::default_schema_compiler)};
@@ -73,6 +75,7 @@ auto sourcemeta::jsonschema::cli::validate(
           index += 1;
           std::ostringstream error;
           sourcemeta::blaze::ErrorOutput output{instance};
+          sourcemeta::blaze::TraceOutput trace_output;
           bool subresult = true;
           if (benchmark) {
             const auto timestamp_start{
@@ -87,12 +90,18 @@ auto sourcemeta::jsonschema::cli::validate(
             } else {
               error << "error: Schema validation failure\n";
             }
+          } else if (trace) {
+            subresult = sourcemeta::blaze::evaluate(schema_template, instance,
+                                                    std::ref(trace_output));
           } else {
             subresult = sourcemeta::blaze::evaluate(schema_template, instance,
                                                     std::ref(output));
           }
 
-          if (subresult) {
+          if (trace) {
+            print(trace_output, std::cout);
+            result = subresult;
+          } else if (subresult) {
             log_verbose(options)
                 << "ok: "
                 << std::filesystem::weakly_canonical(instance_path).string()
@@ -125,6 +134,7 @@ auto sourcemeta::jsonschema::cli::validate(
       const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)};
       std::ostringstream error;
       sourcemeta::blaze::ErrorOutput output{instance};
+      sourcemeta::blaze::TraceOutput trace_output;
       bool subresult{true};
       if (benchmark) {
         const auto timestamp_start{std::chrono::high_resolution_clock::now()};
@@ -139,12 +149,18 @@ auto sourcemeta::jsonschema::cli::validate(
           error << "error: Schema validation failure\n";
           result = false;
         }
+      } else if (trace) {
+        subresult = sourcemeta::blaze::evaluate(schema_template, instance,
+                                                std::ref(trace_output));
       } else {
         subresult = sourcemeta::blaze::evaluate(schema_template, instance,
                                                 std::ref(output));
       }
 
-      if (subresult) {
+      if (trace) {
+        print(trace_output, std::cout);
+        result = subresult;
+      } else if (subresult) {
         log_verbose(options)
             << "ok: "
             << std::filesystem::weakly_canonical(instance_path).string()

diff --git a/src/main.cc b/src/main.cc
@@ -23,13 +23,13 @@ Global Options:
 
    validate <schema.json> <instance.json|.jsonl...> [--http/-h]
             [--benchmark/-b] [--extension/-e <extension>]
-            [--ignore/-i <schemas-or-directories>]
+            [--ignore/-i <schemas-or-directories>] [--trace/-t]
 
        Validate one of more instances against the given schema.
 
    metaschema [schemas-or-directories...] [--http/-h]
               [--extension/-e <extension>]
-              [--ignore/-i <schemas-or-directories>]
+              [--ignore/-i <schemas-or-directories>] [--trace/-t]
 
        Validate that a schema or a set of schemas are valid with respect
        to their metaschemas.

diff --git a/src/utils.cc b/src/utils.cc
@@ -189,6 +189,37 @@ auto print(const sourcemeta::blaze::ErrorOutput &output, std::ostream &stream)
   }
 }
 
+auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream)
+    -> void {
+  for (const auto &entry : output) {
+    if (entry.evaluate_path.empty()) {
+      continue;
+    }
+
+    switch (entry.type) {
+      case sourcemeta::blaze::TraceOutput::EntryType::Push:
+        stream << "-> (push) ";
+        break;
+      case sourcemeta::blaze::TraceOutput::EntryType::Pass:
+        stream << "<- (pass) ";
+        break;
+      case sourcemeta::blaze::TraceOutput::EntryType::Fail:
+        stream << "<- (fail) ";
+        break;
+      default:
+        assert(false);
+        break;
+    }
+
+    stream << "\"";
+    sourcemeta::jsontoolkit::stringify(entry.evaluate_path, stream);
+    stream << "\"\n";
+    stream << "   at \"";
+    sourcemeta::jsontoolkit::stringify(entry.instance_location, stream);
+    stream << "\"\n";
+  }
+}
+
 static auto fallback_resolver(
     const std::map<std::string, std::vector<std::string>> &options,
     std::string_view identifier)

diff --git a/src/utils.h b/src/utils.h
@@ -32,6 +32,9 @@ auto for_each_json(const std::vector<std::string> &arguments,
 auto print(const sourcemeta::blaze::ErrorOutput &output, std::ostream &stream)
     -> void;
 
+auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream)
+    -> void;
+
 auto resolver(const std::map<std::string, std::vector<std::string>> &options,
               const bool remote = false)
     -> sourcemeta::jsontoolkit::SchemaResolver;

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -42,6 +42,8 @@ add_jsonschema_test_unix(validate/fail_schema_enoent)
 add_jsonschema_test_unix(validate/fail_schema_invalid_json)
 add_jsonschema_test_unix(validate/fail_schema_non_schema)
 add_jsonschema_test_unix(validate/fail_schema_unknown_dialect)
+add_jsonschema_test_unix(validate/fail_trace)
+add_jsonschema_test_unix(validate/pass_trace)
 add_jsonschema_test_unix(validate/pass_resolve)
 add_jsonschema_test_unix(validate/pass_resolve_custom_extension)
 add_jsonschema_test_unix(validate/pass_resolve_verbose)
@@ -71,6 +73,8 @@ add_jsonschema_test_unix(validate/fail_many)
 add_jsonschema_test_unix(validate/fail_many_verbose)
 
 # Metaschema
+add_jsonschema_test_unix(metaschema/pass_trace)
+add_jsonschema_test_unix(metaschema/fail_trace)
 add_jsonschema_test_unix(metaschema/fail_directory)
 add_jsonschema_test_unix(metaschema/fail_single)
 add_jsonschema_test_unix(metaschema/fail_non_schema)

diff --git a/test/metaschema/fail_trace.sh b/test/metaschema/fail_trace.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "minimum": "foo"
+}
+EOF
+
+"$1" metaschema "$TMP/schema.json" --trace > "$TMP/output.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+# Order of execution can vary
+
+cat << 'EOF' > "$TMP/expected-1.txt"
+-> (push) "/dependencies"
+   at ""
+<- (pass) "/dependencies"
+   at ""
+-> (push) "/properties"
+   at ""
+-> (push) "/properties/$schema/type"
+   at "/$schema"
+<- (pass) "/properties/$schema/type"
+   at "/$schema"
+-> (push) "/properties/minimum/type"
+   at "/minimum"
+<- (fail) "/properties/minimum/type"
+   at "/minimum"
+<- (fail) "/properties"
+   at ""
+EOF
+
+cat << 'EOF' > "$TMP/expected-2.txt"
+-> (push) "/dependencies"
+   at ""
+<- (pass) "/dependencies"
+   at ""
+-> (push) "/properties"
+   at ""
+-> (push) "/properties/minimum/type"
+   at "/minimum"
+<- (fail) "/properties/minimum/type"
+   at "/minimum"
+<- (fail) "/properties"
+   at ""
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected-1.txt" || \
+  diff "$TMP/output.txt" "$TMP/expected-2.txt"
diff --git a/test/metaschema/pass_trace.sh b/test/metaschema/pass_trace.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{ "$schema": "http://json-schema.org/draft-04/schema#" }
+EOF
+
+"$1" metaschema "$TMP/schema.json" --trace > "$TMP/output.txt"
+
+cat << 'EOF' > "$TMP/expected.txt"
+-> (push) "/dependencies"
+   at ""
+<- (pass) "/dependencies"
+   at ""
+-> (push) "/properties"
+   at ""
+-> (push) "/properties/$schema/type"
+   at "/$schema"
+<- (pass) "/properties/$schema/type"
+   at "/$schema"
+<- (pass) "/properties"
+   at ""
+-> (push) "/type"
+   at ""
+<- (pass) "/type"
+   at ""
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"
diff --git a/test/validate/fail_trace.sh b/test/validate/fail_trace.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "foo": 1 }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --trace > "$TMP/output.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+-> (push) "/properties/foo/type"
+   at "/foo"
+<- (fail) "/properties/foo/type"
+   at "/foo"
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"