[WIP] Add a --trace option for validation-related commands

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>
sourcemeta · Nov 7, 2024 · 0c76ea3 · 0c76ea3
1 parent 4db93a1
commit 0c76ea3
Show file tree

Hide file tree

Showing 17 changed files with 332 additions and 18 deletions.
diff --git a/DEPENDENCIES b/DEPENDENCIES
@@ -4,4 +4,4 @@ jsontoolkit https://github.com/sourcemeta/jsontoolkit 7a398224cc2e76ea9ae8541a87
 hydra https://github.com/sourcemeta/hydra a4a74f3cabd32f2f829f449d67339dac33f9910e
 alterschema https://github.com/sourcemeta/alterschema 92e370ce9c1f0582014b54d43e388ee012dfe13d
 jsonbinpack https://github.com/sourcemeta/jsonbinpack d777179441d3c703e1fda1187742541aa26836b5
-blaze https://github.com/sourcemeta/blaze cf0c89cd419ffb70cc334d395ac5ab1035702e30
+blaze https://github.com/sourcemeta/blaze a5b3c8e4d77a0b88e4a93f304ae75e711b30a2e6
diff --git a/docs/metaschema.markdown b/docs/metaschema.markdown
@@ -7,7 +7,7 @@ Metaschema
 ```sh
 jsonschema metaschema [schemas-or-directories...]
   [--verbose/-v] [--http/-h] [--extension/-e <extension>]
-  [--ignore/-i <schemas-or-directories>]
+  [--ignore/-i <schemas-or-directories>] [--trace/-t]
 ```
 
 Ensure that a schema or a set of schemas are considered valid with regards to
@@ -76,3 +76,9 @@ jsonschema metaschema path/to/schemas/ --ignore path/to/schemas/nested
 ```sh
 jsonschema metaschema --extension .schema.json
 ```
+
+### Validate the metaschema of a JSON Schema with trace information
+
+```sh
+jsonschema metaschema path/to/my/schema.json --trace
+```
diff --git a/docs/test.markdown b/docs/test.markdown
@@ -7,7 +7,7 @@ Testing
 ```sh
 jsonschema test [schemas-or-directories...]
   [--http/-h] [--verbose/-v] [--resolve/-r <schemas-or-directories> ...]
-  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>]
+  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>] [--trace/-t]
 ```
 
 Schemas are code. As such, you should run an automated unit testing suite
@@ -109,3 +109,9 @@ jsonschema test path/to/test.json --resolve path/to/external.json
 ```sh
 jsonschema test path/to/test.json --resolve path/to/schemas --extension schema.json
 ```
+
+### Run a single test definition with trace information
+
+```sh
+jsonschema test path/to/test.json --trace
+```
diff --git a/docs/validate.markdown b/docs/validate.markdown
@@ -7,7 +7,7 @@ Validating
 ```sh
 jsonschema validate <schema.json> <instance.json|.jsonl...> [--http/-h]
   [--verbose/-v] [--resolve/-r <schemas-or-directories> ...] [--benchmark/-b]
-  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>]
+  [--extension/-e <extension>] [--ignore/-i <schemas-or-directories>] [--trace/-t]
 ```
 
 The most popular use case of JSON Schema is to validate JSON documents. The
@@ -94,3 +94,9 @@ jsonschema validate path/to/my/schema.json path/to/my/instance.json \
 ```sh
 jsonschema validate path/to/my/schema.json path/to/my/instance.json --benchmark
 ```
+
+### Validate a JSON instance against a schema with trace information
+
+```sh
+jsonschema validate path/to/my/schema.json path/to/my/instance.json --trace
+```
diff --git a/src/command_metaschema.cc b/src/command_metaschema.cc
@@ -16,7 +16,7 @@
 // TODO: Add a flag to emit output using the standard JSON Schema output format
 auto sourcemeta::jsonschema::cli::metaschema(
     const std::span<const std::string> &arguments) -> int {
-  const auto options{parse_options(arguments, {"h", "http"})};
+  const auto options{parse_options(arguments, {"h", "http", "t", "trace"})};
   const auto custom_resolver{
       resolver(options, options.contains("h") || options.contains("http"))};
   bool result{true};

diff --git a/src/command_test.cc b/src/command_test.cc
@@ -66,7 +66,7 @@ static auto get_data(const sourcemeta::jsontoolkit::JSON &test_case,
 
 auto sourcemeta::jsonschema::cli::test(
     const std::span<const std::string> &arguments) -> int {
-  const auto options{parse_options(arguments, {"h", "http"})};
+  const auto options{parse_options(arguments, {"h", "http", "t", "trace"})};
   bool result{true};
   const auto test_resolver{
       resolver(options, options.contains("h") || options.contains("http"))};

diff --git a/src/command_validate.cc b/src/command_validate.cc
@@ -19,7 +19,8 @@
 // TODO: Add a flag to take a pre-compiled schema as input
 auto sourcemeta::jsonschema::cli::validate(
     const std::span<const std::string> &arguments) -> int {
-  const auto options{parse_options(arguments, {"h", "http", "b", "benchmark"})};
+  const auto options{
+      parse_options(arguments, {"h", "http", "b", "benchmark", "t", "trace"})};
 
   if (options.at("").size() < 1) {
     std::cerr
@@ -51,6 +52,7 @@ auto sourcemeta::jsonschema::cli::validate(
   }
 
   const auto benchmark{options.contains("b") || options.contains("benchmark")};
+  const auto trace{options.contains("t") || options.contains("trace")};
   const auto schema_template{sourcemeta::blaze::compile(
       schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver,
       sourcemeta::blaze::default_schema_compiler)};
@@ -73,6 +75,7 @@ auto sourcemeta::jsonschema::cli::validate(
           index += 1;
           std::ostringstream error;
           sourcemeta::blaze::ErrorOutput output{instance};
+          sourcemeta::blaze::TraceOutput trace_output;
           bool subresult = true;
           if (benchmark) {
             const auto timestamp_start{
@@ -87,12 +90,18 @@ auto sourcemeta::jsonschema::cli::validate(
             } else {
               error << "error: Schema validation failure\n";
             }
+          } else if (trace) {
+            subresult = sourcemeta::blaze::evaluate(schema_template, instance,
+                                                    std::ref(trace_output));
           } else {
             subresult = sourcemeta::blaze::evaluate(schema_template, instance,
                                                     std::ref(output));
           }
 
-          if (subresult) {
+          if (trace) {
+            print(trace_output, std::cout);
+            result = subresult;
+          } else if (subresult) {
             log_verbose(options)
                 << "ok: "
                 << std::filesystem::weakly_canonical(instance_path).string()
@@ -125,6 +134,7 @@ auto sourcemeta::jsonschema::cli::validate(
       const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)};
       std::ostringstream error;
       sourcemeta::blaze::ErrorOutput output{instance};
+      sourcemeta::blaze::TraceOutput trace_output;
       bool subresult{true};
       if (benchmark) {
         const auto timestamp_start{std::chrono::high_resolution_clock::now()};
@@ -139,12 +149,18 @@ auto sourcemeta::jsonschema::cli::validate(
           error << "error: Schema validation failure\n";
           result = false;
         }
+      } else if (trace) {
+        subresult = sourcemeta::blaze::evaluate(schema_template, instance,
+                                                std::ref(trace_output));
       } else {
         subresult = sourcemeta::blaze::evaluate(schema_template, instance,
                                                 std::ref(output));
       }
 
-      if (subresult) {
+      if (trace) {
+        print(trace_output, std::cout);
+        result = subresult;
+      } else if (subresult) {
         log_verbose(options)
             << "ok: "
             << std::filesystem::weakly_canonical(instance_path).string()

diff --git a/src/main.cc b/src/main.cc
@@ -23,19 +23,19 @@ Global Options:
 
    validate <schema.json> <instance.json|.jsonl...> [--http/-h]
             [--benchmark/-b] [--extension/-e <extension>]
-            [--ignore/-i <schemas-or-directories>]
+            [--ignore/-i <schemas-or-directories>] [--trace/-t]
 
        Validate one of more instances against the given schema.
 
    metaschema [schemas-or-directories...] [--http/-h]
               [--extension/-e <extension>]
-              [--ignore/-i <schemas-or-directories>]
+              [--ignore/-i <schemas-or-directories>] [--trace/-t]
 
        Validate that a schema or a set of schemas are valid with respect
        to their metaschemas.
 
    test [schemas-or-directories...] [--http/-h] [--extension/-e <extension>]
-        [--ignore/-i <schemas-or-directories>]
+        [--ignore/-i <schemas-or-directories>] [--trace/-t]
 
        Run a set of unit tests against a schema.
 

diff --git a/src/utils.cc b/src/utils.cc
@@ -189,6 +189,33 @@ auto print(const sourcemeta::blaze::ErrorOutput &output, std::ostream &stream)
   }
 }
 
+auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream)
+    -> void {
+  for (const auto &entry : output) {
+    switch (entry.type) {
+      case sourcemeta::blaze::TraceOutput::EntryType::Push:
+        stream << "-> (push) ";
+        break;
+      case sourcemeta::blaze::TraceOutput::EntryType::Pass:
+        stream << "<- (pass) ";
+        break;
+      case sourcemeta::blaze::TraceOutput::EntryType::Fail:
+        stream << "<- (fail) ";
+        break;
+      default:
+        assert(false);
+        break;
+    }
+
+    stream << "\"";
+    sourcemeta::jsontoolkit::stringify(entry.evaluate_path, stream);
+    stream << "\"\n";
+    stream << "   at \"";
+    sourcemeta::jsontoolkit::stringify(entry.instance_location, stream);
+    stream << "\"\n";
+  }
+}
+
 static auto fallback_resolver(
     const std::map<std::string, std::vector<std::string>> &options,
     std::string_view identifier)

diff --git a/src/utils.h b/src/utils.h
@@ -32,6 +32,9 @@ auto for_each_json(const std::vector<std::string> &arguments,
 auto print(const sourcemeta::blaze::ErrorOutput &output, std::ostream &stream)
     -> void;
 
+auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream)
+    -> void;
+
 auto resolver(const std::map<std::string, std::vector<std::string>> &options,
               const bool remote = false)
     -> sourcemeta::jsontoolkit::SchemaResolver;

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -42,6 +42,8 @@ add_jsonschema_test_unix(validate/fail_schema_enoent)
 add_jsonschema_test_unix(validate/fail_schema_invalid_json)
 add_jsonschema_test_unix(validate/fail_schema_non_schema)
 add_jsonschema_test_unix(validate/fail_schema_unknown_dialect)
+add_jsonschema_test_unix(validate/fail_trace)
+add_jsonschema_test_unix(validate/pass_trace)
 add_jsonschema_test_unix(validate/pass_resolve)
 add_jsonschema_test_unix(validate/pass_resolve_custom_extension)
 add_jsonschema_test_unix(validate/pass_resolve_verbose)

diff --git a/test/validate/fail_trace.sh b/test/validate/fail_trace.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "foo": 1 }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --trace > "$TMP/output.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+-> (push) "/properties/foo/type"
+   at "/foo"
+<- (fail) "/properties/foo/type"
+   at "/foo"
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"
diff --git a/test/validate/pass_trace.sh b/test/validate/pass_trace.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "foo": "bar" }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --trace > "$TMP/output.txt"
+
+cat << EOF > "$TMP/expected.txt"
+-> (push) "/properties/foo/type"
+   at "/foo"
+<- (pass) "/properties/foo/type"
+   at "/foo"
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"
diff --git a/vendor/blaze/src/compiler/CMakeLists.txt b/vendor/blaze/src/compiler/CMakeLists.txt
diff --git a/vendor/blaze/src/compiler/compile_output.cc → ...laze/src/compiler/compile_output_error.cc b/vendor/blaze/src/compiler/compile_output.cc → ...laze/src/compiler/compile_output_error.cc
diff --git a/vendor/blaze/src/compiler/compile_output_trace.cc b/vendor/blaze/src/compiler/compile_output_trace.cc