diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e87e933..d19165bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file. This change ## [Unreleased] ### Added -- Optional colorized output +- [Optional colorized output](https://github.com/bhb/expound/issues/44) +- [`explain-results` and `explain-results-str` functions print human-optimized output for `clojure.spec.test.alpha/check` results](https://github.com/bhb/expound/issues/72) ## [0.5.0] - 2018-02-06 diff --git a/bin/deps.edn b/bin/deps.edn index be96fa6f..8b949938 100644 --- a/bin/deps.edn +++ b/bin/deps.edn @@ -1,4 +1,5 @@ {:deps {expound/expound {:local/root ".."} orchestra {:mvn/version "0.3.0"} + org.clojure/test.check {:mvn/version "0.9.0"} }} diff --git a/bin/sample.clj b/bin/sample.clj index f68419ac..9421df71 100644 --- a/bin/sample.clj +++ b/bin/sample.clj @@ -302,7 +302,40 @@ (display-explain :predicate-messages/score - 101)) + 101) + + (println "----- check results -----") + + (doseq [sym-to-check (st/checkable-syms)] + (println "trying to check" sym-to-check "...") + (try + (st/with-instrument-disabled + (orch.st/with-instrument-disabled + (expound/explain-results (st/check sym-to-check {:clojure.spec.test.check/opts {:num-tests 5}})))) + (catch Exception e + (println "caught exception: " (.getMessage e))))) + + (s/fdef some-func + :args (s/cat :x int?)) + + (st/with-instrument-disabled + (orch.st/with-instrument-disabled + (expound/explain-results (st/check `some-func)))) (s/fdef results-str-fn1 + :args (s/cat :x nat-int? :y nat-int?) + :ret pos-int?) + (defn results-str-fn1 [x y] + (+ x y)) + + (s/fdef results-str-fn2 + :args (s/cat :x nat-int? :y nat-int?) + :fn #(let [x (-> % :args :x) + y (-> % :args :y) + ret (-> % :ret)] + (< x ret))) + (defn results-str-fn2 [x y] + (+ x y)) + + (expound/explain-result (st/check-fn `resultsf-str-fn1 (s/spec `results-str-fn2)))) (go) diff --git a/src/expound/alpha.cljc b/src/expound/alpha.cljc index bb19dd58..08e0e6a9 100644 --- a/src/expound/alpha.cljc +++ b/src/expound/alpha.cljc @@ -57,6 +57,7 @@ ;;;;;; private ;;;;;; +(def check-header-size 45) (def header-size 35) (def section-size 25) @@ -194,10 +195,12 @@ ([size] (apply str (repeat size "-"))) ([size s] + (label size s "-")) + ([size s label-str] (ansi/color - (let [prefix (str "-- " s " ") + (let [prefix (str label-str label-str " " s " ") chars-left (- size (count prefix))] - (str prefix (apply str (repeat chars-left "-")))) + (str prefix (apply str (repeat chars-left label-str)))) :header))) (def header-label (partial label header-size)) @@ -226,18 +229,30 @@ (defn fspec-exception-failure? [failure problem] (and (not= :instrument failure) + (not= :check-failed failure) (= '(apply fn) (:pred problem)))) (defn fspec-ret-failure? [failure problem] (and (not= :instrument failure) + (not= :check-failed failure) (= :ret (first (:path problem))))) (defn fspec-fn-failure? [failure problem] (and (not= :instrument failure) + (not= :check-failed failure) (= :fn (first (:path problem))))) +(defn check-ret-failure? [failure problem] + (and + (= :check-failed failure) + (= :ret (first (:path problem))))) + +(defn check-fn-failure? [failure problem] + (and (= :check-failed failure) + (= :fn (first (:path problem))))) + (defn missing-key? [_failure problem] (let [pred (:pred problem)] (and (seq? pred) @@ -377,6 +392,12 @@ (fspec-fn-failure? failure problem) :problem/fspec-fn-failure + (check-ret-failure? failure problem) + :problem/check-ret-failure + + (check-fn-failure? failure problem) + :problem/check-fn-failure + :else :problem/unknown)) @@ -415,8 +436,7 @@ (defmethod expected-str :problem/extra-input [_type spec-name val path problems opts] (s/assert ::singleton problems) - (let [problem (first problems)] - "has extra input")) + "has extra input") (defmethod problem-group-str :problem/extra-input [_type spec-name val path problems opts] (printer/format @@ -503,6 +523,53 @@ should satisfy (printer/indent (*value-str-fn* spec-name val path (problems/value-in val path))) (expected-str _type spec-name val path problems opts))) +(defmethod expected-str :problem/check-fn-failure [_type spec-name val path problems opts] + (s/assert ::singleton problems) + (let [problem (first problems)] + (printer/format + "failed spec. Function arguments and return value + +%s + +should satisfy + +%s" + (printer/indent (ansi/color (pr-str (:val problem)) :bad-value)) + (printer/indent (ansi/color (pr-pred (:pred problem) (:spec problem)) :good-pred))))) + +(defmethod problem-group-str :problem/check-fn-failure [_type spec-name val path problems opts] + (s/assert ::singleton problems) + (printer/format + "%s + +%s + +%s" + (header-label "Function spec failed") + (ansi/color (printer/indent (pr-str (:expound/check-fn-call (first problems)))) :bad-value) + (expected-str _type spec-name val path problems opts))) + +(defmethod expected-str :problem/check-ret-failure [_type spec-name val path problems opts] + (predicate-errors problems)) + +(defmethod problem-group-str :problem/check-ret-failure [_type spec-name val path problems opts] + (printer/format + "%s + +%s + +returned an invalid value. + +%s + +%s" + (header-label "Function spec failed") + + (ansi/color (printer/indent (pr-str (:expound/check-fn-call (first problems)))) :bad-value) + + (printer/indent (*value-str-fn* spec-name val path (problems/value-in val path))) + (expected-str _type spec-name val path problems opts))) + (defmethod expected-str :problem/unknown [_type spec-name val path problems opts] (predicate-errors problems)) @@ -518,12 +585,6 @@ should satisfy (show-spec-name spec-name (printer/indent (*value-str-fn* spec-name val path (problems/value-in val path)))) (expected-str _type spec-name val path problems opts))) -(defn problem-group-str1 [type spec-name val path problems opts] - (str - (problem-group-str type spec-name val path problems opts) - "\n\n" - (if (:print-specs? opts) (relevant-specs problems) ""))) - (defn instrumentation-info [failure caller] ;; As of version 1.9.562, Clojurescript does ;; not include failure or caller info, so @@ -540,45 +601,160 @@ should satisfy (-> ed ::s/problems first :path first) nil)) -(defn printer-str [opts explain-data] +(defn print-explain-data [opts explain-data] + (if-not explain-data + "Success!\n" + (let [{:keys [::s/fn ::s/failure]} explain-data + explain-data' (problems/annotate explain-data) + caller (:expound/caller explain-data') + form (:expound/form explain-data') + problems (->> explain-data' + :expound/problems + (problems/leaf-only) + (grouped-and-sorted-problems (::s/failure explain-data)))] + + (printer/no-trailing-whitespace + (str + (ansi/color (instrumentation-info failure caller) :none) + (printer/format + "%s + +%s +%s %s %s\n" + (string/join "\n\n" (for [[[in type] probs] problems] + (str + (problem-group-str type (spec-name explain-data) form in probs opts) + "\n\n" + (if (:print-specs? opts) (relevant-specs probs) "")))) + (ansi/color (section-label) :footer) + (ansi/color "Detected" :footer) + (ansi/color (count problems) :footer) + (ansi/color (if (= 1 (count problems)) "error" "errors") :footer))))))) + +(defn minimal-fspec [form] + (let [fspec-sp (s/cat + :sym qualified-symbol? + :args (s/* + (s/cat :k #{:args :fn :ret} :v any?)))] + + (-> (s/conform fspec-sp form) + (update :args (fn [args] (filter #(some? (:v %)) args))) + (->> (s/unform fspec-sp))))) + +(defn print-check-result [check-result] + (let [{:keys [sym spec failure] :or {sym '}} check-result + ret #?(:clj (:clojure.spec.test.check/ret check-result) + :cljs (:clojure.test.check/ret check-result)) + explain-data (ex-data failure) + bad-args (or #?(:clj (:clojure.spec.test.alpha/args explain-data) + :cljs (:cljs.spec.test.alpha/args explain-data)) + (first (:fail ret))) + failure-reason (::s/failure explain-data) + sym (or sym ')] + (str + ;; CLJS does not contain symbol if function is undefined + (label check-header-size (str "Checked " sym) "=") + "\n\n" + (cond + ;; FIXME - once we have a function that can highlight + ;; a spec, use it here to make this error message clearer + #?(:clj (and failure (= :no-gen failure-reason)) + ;; Workaround for CLJS + :cljs (and + failure + (re-matches #"Unable to construct gen at.*" (.-message failure)))) + (let [path (::s/path explain-data)] + (str + #?(:clj + (str + "Unable to construct generator for " + (ansi/color (pr-str path) :error-key)) + :cljs + (.-message failure)) + " in\n\n" + (printer/indent (str (s/form (:args (:spec check-result))))) + "\n")) + + (= :no-args-spec failure-reason) + (str + "Failed to check function.\n\n" + (ansi/color (printer/indent (printer/pprint-str + (minimal-fspec (s/form spec)))) :bad-value) + "\n\nshould contain an :args spec\n") + + (= :no-fn failure-reason) + (if (some? sym) + (str + "Failed to check function.\n\n" + (ansi/color (printer/indent (pr-str sym)) :bad-value) + "\n\nis not defined\n") + ;; CLJS doesn't set the symbol + (str + "Cannot check undefined function\n")) + + (and explain-data + (= :check-failed (-> explain-data ::s/failure))) + (with-out-str + (s/*explain-out* (update + explain-data + ::s/problems + #(map + (fn [p] + (assoc p :expound/check-fn-call (concat (list sym) + bad-args))) + %)))) + + failure + (str + (ansi/color (printer/indent (printer/pprint-str + (concat (list sym) bad-args))) :bad-value) + "\n\n threw error\n\n" + (printer/pprint-str failure)) + + :else + "Success!\n")))) + +(defn explain-data? [data] + (s/valid? + (s/keys :req + [::s/problems + ::s/spec + ::s/value] + :opt + [::s/failure]) + data)) + +(defn check-result? [data] + (s/valid? + (s/keys :req-un [::spec] + :opt-un [::sym + ::failure + :clojure.spec.test.check/ret]) + data)) + +(defn printer-str [opts data] (let [opts' (merge {:show-valid-values? false :print-specs? true} opts)] - (if-not explain-data - "Success!\n" - (binding [*value-str-fn* (get opts :value-str-fn (partial value-in-context opts')) - ansi/*enable-color* (not= :none (get opts :theme :none)) - ansi/*print-styles* (case (get opts :theme :none) - :figwheel-theme - figwheel-theme - - :none - {})] - (let [{:keys [::s/fn ::s/failure]} explain-data - explain-data' (problems/annotate explain-data) - caller (:expound/caller explain-data') - form (:expound/form explain-data') - problems (->> explain-data' - :expound/problems - (problems/leaf-only) - (grouped-and-sorted-problems (::s/failure explain-data)))] - - (printer/no-trailing-whitespace - (str - (ansi/color (instrumentation-info failure caller) :none) - (printer/format - "%s + (binding [*value-str-fn* (get opts :value-str-fn (partial value-in-context opts')) + ansi/*enable-color* (not= :none (get opts :theme :none)) + ansi/*print-styles* (case (get opts :theme :none) + :figwheel-theme + figwheel-theme -%s -%s %s %s\n" - (string/join "\n\n" (for [[[in type] probs] problems] - (problem-group-str1 type (spec-name explain-data) form in probs opts'))) - (ansi/color (section-label) :footer) - (ansi/color "Detected" :footer) - (ansi/color (count problems) :footer) - (ansi/color (if (= 1 (count problems)) "error" "errors") :footer))))))))) + :none + {})] + + (cond + (or (explain-data? data) + (nil? data)) + (print-explain-data opts' data) -(s/def ::foo string?) + (check-result? data) + (print-check-result data) + + :else + (str "Unknown data:\n\n" data))))) #?(:clj (defn ns-qualify @@ -637,3 +813,20 @@ should satisfy `(do (defmsg '~k ~error-message) (s/def ~k ~spec-form)))))) + +(defn explain-result [check-result] + (when (= s/*explain-out* s/explain-printer) + (throw (ex-info "Cannot print check results with default printer. Use 'set!' or 'binding' to use Expound printer." {}))) + (s/*explain-out* check-result)) + +(defn explain-result-str [check-result] + (with-out-str (explain-result check-result))) + +(defn explain-results [check-results] + (doseq [check-result (butlast check-results)] + (explain-result check-result) + (print "\n\n")) + (explain-result (last check-results))) + +(defn explain-results-str [check-results] + (with-out-str (explain-results check-results))) diff --git a/src/expound/printer.cljc b/src/expound/printer.cljc index a6b605b8..35a26321 100644 --- a/src/expound/printer.cljc +++ b/src/expound/printer.cljc @@ -243,4 +243,3 @@ (string/join "\n" (into [(str (apply str (repeat first-line-indent " ")) line)] (map #(str (apply str (repeat rest-lines-indent " ")) %) lines)))))) - diff --git a/test/expound/alpha_test.cljc b/test/expound/alpha_test.cljc index 7e458ab1..8b5d4caf 100644 --- a/test/expound/alpha_test.cljc +++ b/test/expound/alpha_test.cljc @@ -41,6 +41,22 @@ :clj (string/replace s "pf." "clojure.")) args)) +(defn take-lines [n s] + (string/join "\n" (take n (string/split-lines s)))) + +(def inverted-ansi-codes + (reduce + (fn [m [k v]] + (assoc m (str v) k)) + {} + ansi/sgr-code)) + +(defn readable-ansi [s] + (string/replace + s + #"\x1b\[([0-9]*)m" + #(str "<" (string/upper-case (name (get inverted-ansi-codes (second %)))) ">"))) + ;; https://github.com/bhb/expound/issues/8 (deftest expound-output-ends-in-newline (is (= "\n" (str (last (expound/expound-str string? 1))))) @@ -1273,7 +1289,7 @@ Detected 1 error\n") (+ x y)) (defn no-linum [s] - (string/replace s #".cljc:\d+" ".cljc:LINUM")) + (string/replace s #"(.cljc?):\d+" "$1:LINUM")) (deftest test-instrument (st/instrument `test-instrument-adder) @@ -2439,18 +2455,293 @@ Detected 1 error :predicate-messages/score 101)))))) -(def inverted-ansi-codes - (reduce - (fn [m [k v]] - (assoc m (str v) k)) - {} - ansi/sgr-code)) +(s/fdef results-str-fn1 + :args (s/cat :x nat-int? :y nat-int?) + :ret pos-int?) +(defn results-str-fn1 [x y] + (+ x y)) -(defn readable-ansi [s] - (string/replace - s - #"\x1b\[([0-9]*)m" - #(str "<" (string/upper-case (name (get inverted-ansi-codes (second %)))) ">"))) +(s/fdef results-str-fn2 + :args (s/cat :x nat-int? :y nat-int?) + :fn #(let [x (-> % :args :x) + y (-> % :args :y) + ret (-> % :ret)] + (< x ret))) +(defn results-str-fn2 [x y] + (+ x y)) + +(s/fdef results-str-fn3 + :args (s/cat :x #{0} :y #{0}) + :ret nat-int?) +(defn results-str-fn3 [x y] + (+ x y)) + +(s/fdef results-str-fn4 + :args (s/cat :x int?) + :ret (s/coll-of int?)) +(defn results-str-fn4 [x] + [x :not-int]) + +(s/fdef results-str-fn5 + :args (s/cat :x #{1} :y #{1}) + :ret string?) +(defn results-str-fn5 + [x y] + #?(:clj (throw (Exception. "Ooop!")) + :cljs (throw (js/Error. "Oops!")))) + +(s/fdef results-str-fn6 + :args (s/cat :f fn?) + :ret any?) +(defn results-str-fn6 + [f] + (f 1)) + +(s/fdef results-str-missing-fn + :args (s/cat :x int?)) + +(s/fdef results-str-missing-args-spec + :ret int?) +(defn results-str-missing-args-spec [] 1) + +(deftest explain-results + (testing "explaining results with non-expound printer" + (is (thrown-with-msg? + #?(:cljs :default :clj Exception) + #"Cannot print check results" + (binding [s/*explain-out* s/explain-printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn1))))))) + + (testing "single bad result (failing return spec)" + (is (= (pf + "== Checked expound.alpha-test/results-str-fn1 + +-- Function spec failed ----------- + + (expound.alpha-test/results-str-fn1 0 0) + +returned an invalid value. + + 0 + +should satisfy + + pos-int? + + + +------------------------- +Detected 1 error +") + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn1))))))) + (testing "single bad result (failing fn spec)" + (is (= (pf "== Checked expound.alpha-test/results-str-fn2 + +-- Function spec failed ----------- + + (expound.alpha-test/results-str-fn2 0 0) + +failed spec. Function arguments and return value + + {:args {:x 0, :y 0}, :ret 0} + +should satisfy + + (fn + [%%] + (let + [x + (-> %% :args :x) + y + (-> %% :args :y) + ret + (-> %% :ret)] + (< x ret))) + + + +------------------------- +Detected 1 error +") + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn2))))))) + (testing "single valid result" + (is (= "== Checked expound.alpha-test/results-str-fn3 + +Success! +" + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (st/check `results-str-fn3)))))) + #?(:clj + (testing "multiple results" + (is (= "== Checked expound.alpha-test/results-str-fn2 + +-- Function spec failed ----------- + + (expound.alpha-test/results-str-fn2 0 0) + +failed spec. Function arguments and return value + + {:args {:x 0, :y 0}, :ret 0} + +should satisfy + + (fn + [%] + (let + [x + (-> % :args :x) + y + (-> % :args :y) + ret + (-> % :ret)] + (< x ret))) + + + +------------------------- +Detected 1 error + + +== Checked expound.alpha-test/results-str-fn3 + +Success! +" + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check [`results-str-fn2 `results-str-fn3])))))))) + (testing "check-fn" + (is (= "== Checked ======================== + +-- Function spec failed ----------- + + ( 0 0) + +failed spec. Function arguments and return value + + {:args {:x 0, :y 0}, :ret 0} + +should satisfy + + (fn + [%] + (let + [x + (-> % :args :x) + y + (-> % :args :y) + ret + (-> % :ret)] + (< x ret))) + + + +------------------------- +Detected 1 error +" + (binding [s/*explain-out* expound/printer] + (expound/explain-result-str (st/check-fn `results-str-fn1 (s/spec `results-str-fn2))))))) + #?(:clj (testing "custom printer" + (is (= "== Checked expound.alpha-test/results-str-fn4 + +-- Function spec failed ----------- + + (expound.alpha-test/results-str-fn4 0) + +returned an invalid value. + + [0 :not-int] + ^^^^^^^^ + +should satisfy + + int? + + + +------------------------- +Detected 1 error +" + (binding [s/*explain-out* (expound/custom-printer {:show-valid-values? true})] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn4)))))))) + (testing "exceptions raised during check" + (is (= "== Checked expound.alpha-test/results-str-fn5 + + (expound.alpha-test/results-str-fn5 1 1) + + threw error" + (binding [s/*explain-out* expound/printer] + (take-lines 5 (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn5)))))))) + (testing "colorized output" + (is (= (pf "== Checked expound.alpha-test/results-str-fn5 + + (expound.alpha-test/results-str-fn5 1 1) + + threw error") + (binding [s/*explain-out* (expound/custom-printer {:theme :figwheel-theme})] + (readable-ansi (take-lines 5 (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn5))))))))) + + (testing "failure to generate" + (is (= + #?(:clj "== Checked expound.alpha-test/results-str-fn6 + +Unable to construct generator for [:f] in + + (clojure.spec.alpha/cat :f clojure.core/fn?) +" + ;; CLJS doesn't contain correct data for check failure + + :cljs "== Checked expound.alpha-test/results-str-fn6 + +Unable to construct gen at: [:f] for: fn? in + + (cljs.spec.alpha/cat :f cljs.core/fn?) +") + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-fn6))))))) + (testing "no-fn failure" + (is (= #?(:clj "== Checked expound.alpha-test/results-str-missing-fn + +Failed to check function. + + expound.alpha-test/results-str-missing-fn + +is not defined +" + :cljs "== Checked ======================== + +Failed to check function. + + + +is not defined +") + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-missing-fn))))))) + (testing "no args spec" + (is (= (pf "== Checked expound.alpha-test/results-str-missing-args-spec + +Failed to check function. + + (pf.spec.alpha/fspec :ret pf.core/int?) + +should contain an :args spec +") + (binding [s/*explain-out* expound/printer] + (expound/explain-results-str (orch.st/with-instrument-disabled (st/check `results-str-missing-args-spec)))))))) + +#?(:clj (deftest explain-results-gen + (checking + "all functions can be checked and printed" + num-tests + [sym-to-check (gen/elements (st/checkable-syms))] + ;; Just confirm an error is not thrown + (is (string? + (expound/explain-results-str + (orch.st/with-instrument-disabled + (with-out-str + (st/check sym-to-check + {:clojure.spec.test.check/opts {:num-tests 10}}))))))))) (s/def :colorized-output/strings (s/coll-of string?)) (deftest colorized-output diff --git a/test/expound/printer_test.cljc b/test/expound/printer_test.cljc index 95962f54..4951c783 100644 --- a/test/expound/printer_test.cljc +++ b/test/expound/printer_test.cljc @@ -16,7 +16,7 @@ (is (= "expound.alpha/expound" (printer/pprint-fn expound/expound))) (is (= "" - (printer/pprint-fn #(inc %)))) + (printer/pprint-fn #(inc (inc %))))) (is (= "" (printer/pprint-fn (constantly true)))) (is (= "" @@ -55,15 +55,15 @@ (s/explain-data :print-spec-keys/key-spec {})))))) - (is (= nil - (printer/print-spec-keys - (map #(copy-key % :via :expound/via) - (::s/problems - (s/explain-data - (s/keys - :req [:print-spec-keys/field1] - :req-un [:print-spec-keys/field2]) - {})))))) + (is (nil? + (printer/print-spec-keys + (map #(copy-key % :via :expound/via) + (::s/problems + (s/explain-data + (s/keys + :req [:print-spec-keys/field1] + :req-un [:print-spec-keys/field2]) + {})))))) (is (= "| key | spec | diff --git a/test/expound/test_utils.cljc b/test/expound/test_utils.cljc index b7ac640c..6746099f 100644 --- a/test/expound/test_utils.cljc +++ b/test/expound/test_utils.cljc @@ -2,6 +2,7 @@ (:require [clojure.spec.alpha :as s] #?(:cljs [clojure.spec.test.alpha :as st] + ;; FIXME ;; orchestra is supposed to work with cljs but ;; it isn't working for me right now #_[orchestra-cljs.spec.test :as st]