From 2f4ec6285c6517ad93e1198e9e22576a51aec621 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 6 Feb 2024 17:51:49 -0500 Subject: [PATCH 01/27] Fine adjustments in prep for 'nearly good enough' XSpec support under ci/cd (in feature branch) --- src/schema-gen/InspectorXSLT/Makefile | 8 ++-- .../inspector-production-functions.xspec | 4 +- .../xspec-smoketest.xspec | 2 +- support/xspec-dev/XSPEC-BATCH.xsl | 38 +++++++++---------- support/xspec-dev/mvn-saxon-xspec-batch.sh | 2 +- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 3e7e349f..1c2767bc 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -5,22 +5,24 @@ output_folder:=$(module_path)/test_output xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) .PHONY: test -test: unit-test smoke-test ## Run all tests +test: unit-test spec-test smoke-test ## Run all tests .PHONY: spec-test -spec-test: ## Run all specification-tests +spec-test: ## Run all specification tests LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_script) \ "folder=$(module_path)/testing/tests/inspector-functional-xspec" \ "report-to=$(output_folder)/inspector-functional-tests_report.html" \ "junit-to=$(output_folder)/inspector-functional-tests_junit-report.xml" \ + "stop-on-error=yes" \ "recurse=yes" .PHONY: smoke-test -smoke-test: ## Run all smoke-tests +smoke-test: ## Run all smoke tests LOGFILE="$(output_folder)/integration-tests.log" $(xspec_script) \ "folder=$(module_path)/testing/tests/inspector-generation-xspec" \ "report-to=$(output_folder)/integration-tests_report.html" \ "junit-to=$(output_folder)/integration-tests_junit-report.xml" \ + "stop-on-error=yes" \ "recurse=yes" .PHONY: clean diff --git a/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/inspector-production-functions.xspec b/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/inspector-production-functions.xspec index b1eedf3b..54c6088b 100644 --- a/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/inspector-production-functions.xspec +++ b/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/inspector-production-functions.xspec @@ -26,7 +26,7 @@ combo multiple key-field with @pattern - oops? --> - + @@ -49,7 +49,7 @@ - + diff --git a/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/xspec-smoketest.xspec b/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/xspec-smoketest.xspec index 30cdd42d..851a167a 100644 --- a/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/xspec-smoketest.xspec +++ b/src/schema-gen/InspectorXSLT/testing/tests/inspector-generation-xspec/xspec-smoketest.xspec @@ -11,7 +11,7 @@ - + diff --git a/support/xspec-dev/XSPEC-BATCH.xsl b/support/xspec-dev/XSPEC-BATCH.xsl index 013a10e2..a8231708 100644 --- a/support/xspec-dev/XSPEC-BATCH.xsl +++ b/support/xspec-dev/XSPEC-BATCH.xsl @@ -32,35 +32,31 @@ Parameters: - $baseURI: a URI indicating runtime context - relative to which XSpecs are found - $folder: a folder, relative to baseURI - defaults to 'src' - - hint - if you want your script to run in the current directory or any arbitrary directory - and wish to hardwire its location - and it's not inside ../../src (in which case designating $folder works) - set baseURI to the script's file: URI e.g. file:/mnt/c/Users/wap1/Documents/usnistgov/metaschema-xslt/support/xspec-dev/script.sh - (or to the folder with closing slash) - and folder to '.' - this parameterization is necessary so the XSLT can locate resource - even located elsewhere from where the script is called) - + $baseURI (optional, defaults to repository root): a URI indicating a runtime context + $folder (optional, defaults to 'src'): a folder, relative to baseURI, where XSpecs are to be found + By default, the path is directed to all XSpecs in the repo src/ directory + + Reset $folder first (to navigate from the top of the repository), and if it becomes too complex, reset $baseURI + Alternatively, pass an absolute URI in for $folder and ignore the $baseURI + Alternatively, give a file path as $baseURI, set $folder to '.' and the evaluation will operate from the file location + $pattern: glob-like syntax for file name matching cf https://www.saxonica.com/html/documentation12/sourcedocs/collections/collection-directories.html '?select' - use a single file name for a single file - use (file1.xspec|file2.xspec|...|fileN.xspec) for file name literals (with URI escaping) - defaults to *.xspec (all files suffixed xspec) + Use a single file name for a single file + Use (file1.xspec|file2.xspec|...|fileN.xspec) for file name literals (with URI escaping) + Defaults to *.xspec (all files suffixed xspec) $recurse (yes|no) identify files recursively - defaults to 'yes' $stop-on-error (yes|no) hard stop on any failure, or keep going - does not stop if a test successfully runs and returns a false result - only if tests fail to run for any reason - default to 'hard' + Does not stop if a test successfully runs and returns a false result + Only if tests fail to run for any reason + Defaults to 'no' - $report-to - if ends in '.html', a single aggregated report is written to this path relative to baseURI + $report-to - if ends in '.html', a single aggregated report is written to this path relative to the folder (target) path otherwise a non-empty $report-to results in each XSpec getting its own report written into a folder of that name they will all be together and no precaution is taken for XSpecs with colliding names - take care - as Saxon will refuse (so rename one of the files or report in aggregate) + as Saxon will refuse (so rename one of the files or report in aggregate instead) if $report is not given, no external reports are written - $junit-to - if set, with any file suffix (matching regex \..+$), a JUnit report in XML is written to this path relative to base URI + $junit-to - if set, with any file suffix (matching regex \..+$), a JUnit report in XML is written to this path relative to folder path $theme works as it does in XSPEC-SINGLE.xsl (clean|classic|toybox|uswds) results: diff --git a/support/xspec-dev/mvn-saxon-xspec-batch.sh b/support/xspec-dev/mvn-saxon-xspec-batch.sh index 74a8710b..9a82d832 100755 --- a/support/xspec-dev/mvn-saxon-xspec-batch.sh +++ b/support/xspec-dev/mvn-saxon-xspec-batch.sh @@ -70,7 +70,7 @@ echo_on_error() ADDITIONAL_ARGS=$(echo "${*// /\\ }") SAXON_ARGS="-it:go -xsl:\"${XSLT_FILE}\" -init:org.nineml.coffeesacks.RegisterCoffeeSacks \ - stop-on-error=yes $ADDITIONAL_ARGS" + $ADDITIONAL_ARGS" echo "XSpec testing - logging to ${LOGFILE}" From 18173c1297aa295b38b68309430ddd87ddd6000f Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 6 Feb 2024 18:03:02 -0500 Subject: [PATCH 02/27] Adjustments for legibility --- support/xspec-dev/XSPEC-BATCH.xsl | 7 +++---- support/xspec-dev/xspec-summary-reduce.xsl | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/support/xspec-dev/XSPEC-BATCH.xsl b/support/xspec-dev/XSPEC-BATCH.xsl index a8231708..73799e17 100644 --- a/support/xspec-dev/XSPEC-BATCH.xsl +++ b/support/xspec-dev/XSPEC-BATCH.xsl @@ -163,10 +163,7 @@ - - - - @@ -175,6 +172,8 @@ { $dropped?name => string-join(', ') } + + diff --git a/support/xspec-dev/xspec-summary-reduce.xsl b/support/xspec-dev/xspec-summary-reduce.xsl index fc0f3251..e96e042e 100644 --- a/support/xspec-dev/xspec-summary-reduce.xsl +++ b/support/xspec-dev/xspec-summary-reduce.xsl @@ -23,14 +23,14 @@ - SUCCESS - { mx:give-report-counts(.) } - NO FAILURES REPORTED + SUCCESS - { mx:give-report-counts(.) } - NO FAILURES REPORTED - FAILURE - { mx:give-report-counts(.) } - { count(descendant::fail) } { mx:pluralize('FAILURE',count(descendant::fail)) } REPORTED + FAILURE - { mx:give-report-counts(.) } - { count(descendant::fail) } { mx:pluralize('FAILURE',count(descendant::fail)) } REPORTED From 85903cb6f3d87760f23a36cd4040eb543d9cf09e Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 6 Feb 2024 18:10:12 -0500 Subject: [PATCH 03/27] Another touchup --- support/xspec-dev/xspec-summary-reduce.xsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/support/xspec-dev/xspec-summary-reduce.xsl b/support/xspec-dev/xspec-summary-reduce.xsl index e96e042e..71f71713 100644 --- a/support/xspec-dev/xspec-summary-reduce.xsl +++ b/support/xspec-dev/xspec-summary-reduce.xsl @@ -17,19 +17,21 @@ XSpec summary report: { mx:enumerate('XSpec',count(report)) } - { (1 to 20) ! '--- ' } + { (1 to 14) ! ' ...' } + SUCCESS - { mx:give-report-counts(.) } - NO FAILURES REPORTED + FAILURE - { mx:give-report-counts(.) } - { count(descendant::fail) } { mx:pluralize('FAILURE',count(descendant::fail)) } REPORTED From 5fbbef42fbf7bdeebb35403e2469be1bb50512c4 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Thu, 8 Feb 2024 11:10:32 -0500 Subject: [PATCH 04/27] Buffs in view of CI/CD and local/CL use cases for testing under make including new error-on-fail option --- src/schema-gen/InspectorXSLT/Makefile | 9 +++++++ .../xspec-patterns.xspec | 4 +++- support/xspec-dev/XSPEC-BATCH.xsl | 24 ++++++++++++------- support/xspec-dev/xspec-summary-reduce.xsl | 7 +----- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 1c2767bc..7f0e43d7 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -1,5 +1,14 @@ include ../../testing/make_common.mk +# TODO (sort out): +# unit testing - XSLT production templates +# smoke testing - whether an XSLT is produced and compiles in a run +# trying on available samples? +# use testing/TEST-INSPECTOR-RUNTIME.xpl +# spec testing - functional runtime tests of the generated XSLT + +# refresh-inspector - run XSLT production for test metaschemas writing to current (and smoke test) + module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) output_folder:=$(module_path)/test_output xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) diff --git a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/xspec-patterns.xspec b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/xspec-patterns.xspec index 98ebd6e7..b8c6f52c 100644 --- a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/xspec-patterns.xspec +++ b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/xspec-patterns.xspec @@ -32,8 +32,10 @@ Modes to test: + + + - diff --git a/support/xspec-dev/XSPEC-BATCH.xsl b/support/xspec-dev/XSPEC-BATCH.xsl index 73799e17..d6163200 100644 --- a/support/xspec-dev/XSPEC-BATCH.xsl +++ b/support/xspec-dev/XSPEC-BATCH.xsl @@ -46,11 +46,12 @@ Use (file1.xspec|file2.xspec|...|fileN.xspec) for file name literals (with URI escaping) Defaults to *.xspec (all files suffixed xspec) $recurse (yes|no) identify files recursively - defaults to 'yes' + $error-on-fail (yes|no) after delivering a report, on detecting a failure in test results, terminate the process + useful under CI/CD so a failure in testing 'breaks the build' $stop-on-error (yes|no) hard stop on any failure, or keep going Does not stop if a test successfully runs and returns a false result Only if tests fail to run for any reason - Defaults to 'no' - + Defaults to 'no', but $error-on-fail=yes also sets this to yes $report-to - if ends in '.html', a single aggregated report is written to this path relative to the folder (target) path otherwise a non-empty $report-to results in each XSpec getting its own report written into a folder of that name they will all be together and no precaution is taken for XSpecs with colliding names - take care @@ -85,12 +86,13 @@ - + + no + no - - - + + @@ -174,10 +176,13 @@ + + + Failures detected - process terminating - + + - - @@ -211,6 +216,7 @@ + @@ -218,7 +224,7 @@ - + diff --git a/support/xspec-dev/xspec-summary-reduce.xsl b/support/xspec-dev/xspec-summary-reduce.xsl index 71f71713..351c588b 100644 --- a/support/xspec-dev/xspec-summary-reduce.xsl +++ b/support/xspec-dev/xspec-summary-reduce.xsl @@ -32,7 +32,7 @@ - FAILURE - { mx:give-report-counts(.) } - { count(descendant::fail) } { mx:pluralize('FAILURE',count(descendant::fail)) } REPORTED + FAILURE - { mx:give-report-counts(.) } - { count(descendant::fail) } { mx:pluralize('failure',count(descendant::fail)) => upper-case() } REPORTED @@ -66,9 +66,4 @@ { . }{ 's'[$plural] } - - - { . }{ 'S'[$plural] } - - \ No newline at end of file From a1248392335fc0fe041ab09757c6d16c034bbc69 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Thu, 8 Feb 2024 11:30:39 -0500 Subject: [PATCH 05/27] Added to script documentation --- support/xspec-dev/mvn-saxon-xspec-batch.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/support/xspec-dev/mvn-saxon-xspec-batch.sh b/support/xspec-dev/mvn-saxon-xspec-batch.sh index 9a82d832..44371b41 100755 --- a/support/xspec-dev/mvn-saxon-xspec-batch.sh +++ b/support/xspec-dev/mvn-saxon-xspec-batch.sh @@ -50,7 +50,11 @@ report-to (folder or HTML filename): if not given, no report is written junit-to (filename with suffix): write a JUnit test report (XML) to this file produces a warning if provided value is not a suffixed file name and a valid URI -stop-on-error (yes|no): hard stop on any failure, or keep going - defaults to 'no' +error-on-fail (yes|no): if a failing test is detected in the results, return an error - defaults to 'no' + warning: an optimizing processor may not write complete reports, or any, with this setting - ymmv + +stop-on-error (yes|no): hard stop on any processing error, or keep going - defaults to 'no' + this also switches to 'yes' if error-on-failure=yes baseURI: a URI indicating runtime context relative to which XSpecs are found defaults to repository root From f7f4834b21f45ce7ce8d6810e41d2c011af038f0 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Thu, 8 Feb 2024 11:44:10 -0500 Subject: [PATCH 06/27] Still polishing help msg --- support/xspec-dev/mvn-saxon-xspec-batch.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/support/xspec-dev/mvn-saxon-xspec-batch.sh b/support/xspec-dev/mvn-saxon-xspec-batch.sh index 44371b41..b02c9c2e 100755 --- a/support/xspec-dev/mvn-saxon-xspec-batch.sh +++ b/support/xspec-dev/mvn-saxon-xspec-batch.sh @@ -26,7 +26,8 @@ Runtime parameters (use param=value syntax): folder: XSpec inputs are found in the targeted folder, relative to baseURI - defaults to 'src' (baseURI being set by default to the repository, this gives its /src directory) - + to process outside the repository, pass in an absolute URL or reset baseURI +Yeah e.g. folder=src (sets folder to repository /src) pattern: glob-like syntax for file name matching @@ -54,7 +55,7 @@ error-on-fail (yes|no): if a failing test is detected in the results, return an warning: an optimizing processor may not write complete reports, or any, with this setting - ymmv stop-on-error (yes|no): hard stop on any processing error, or keep going - defaults to 'no' - this also switches to 'yes' if error-on-failure=yes + this also switches to 'yes' if error-on-fail=yes baseURI: a URI indicating runtime context relative to which XSpecs are found defaults to repository root From f87fb3b65f090894f1f710d4d6f9343f8265ef8e Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Fri, 9 Feb 2024 17:51:43 -0500 Subject: [PATCH 07/27] Now with some more nicer XProc and scripting supporting ongoing maintenance / dev --- src/schema-gen/InspectorXSLT/Makefile | 9 ++- src/schema-gen/InspectorXSLT/TESTING.md | 26 +++++++++ src/schema-gen/InspectorXSLT/mx-reporter.xsl | 8 ++- .../testing/COMPUTER-INSPECTOR-PRODUCE.xpl | 39 +++++++++++++ .../testing/CURRENT-MODELS-REFRESH.xpl | 55 +++++++++++++++++++ .../testing/VALIDATORS-REFRESH.xpl | 51 +++++++++++++++++ .../testing/current/computer_inspector.xsl | 52 +++++++++--------- .../InspectorXSLT/testing/planning.md | 12 ++++ .../testing/refresh-computer-inspector.sh | 52 ++++++------------ support/xspec-dev/XSPEC-BATCH.xsl | 7 ++- 10 files changed, 245 insertions(+), 66 deletions(-) create mode 100644 src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl create mode 100644 src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl create mode 100644 src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 7f0e43d7..caa7a8d4 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -12,6 +12,7 @@ include ../../testing/make_common.mk module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) output_folder:=$(module_path)/test_output xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) +smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh) .PHONY: test test: unit-test spec-test smoke-test ## Run all tests @@ -25,8 +26,8 @@ spec-test: ## Run all specification tests "stop-on-error=yes" \ "recurse=yes" -.PHONY: smoke-test -smoke-test: ## Run all smoke tests +.PHONY: unit-test +unit-test: ## Run all unit tests LOGFILE="$(output_folder)/integration-tests.log" $(xspec_script) \ "folder=$(module_path)/testing/tests/inspector-generation-xspec" \ "report-to=$(output_folder)/integration-tests_report.html" \ @@ -34,6 +35,10 @@ smoke-test: ## Run all smoke tests "stop-on-error=yes" \ "recurse=yes" +.PHONY: smoke-test +smoke-test: ## Run smoke test + $(smoketest_script) + .PHONY: clean clean: ## Remove test output rm -fr $(output_folder)/* \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/TESTING.md b/src/schema-gen/InspectorXSLT/TESTING.md index 8a50282c..79e589ff 100644 --- a/src/schema-gen/InspectorXSLT/TESTING.md +++ b/src/schema-gen/InspectorXSLT/TESTING.md @@ -14,6 +14,32 @@ Currently we focus on the *first two* of these, producing functional results in Find resources for testing the XSLT Inspector and its production in the [testing](testing) subdirectory. + +## Testing under `make` + +Documented in the Makefile - + +For example + +``` +> make test +``` + +Runs all tests using provided scripting. + +Don't commit unless this passes. + +``` +> make smoke-test +``` + +Runs only the 'smoke tests' (end to end production pipeline testing - are viable artifacts produced irrespective of their functionality or correctness) + +Available: + - `smoke-test` + - `spec-test` + - `unit-test` + ## Model metaschemas for testing ### `current` resource cache diff --git a/src/schema-gen/InspectorXSLT/mx-reporter.xsl b/src/schema-gen/InspectorXSLT/mx-reporter.xsl index 8a12530f..f63487b6 100644 --- a/src/schema-gen/InspectorXSLT/mx-reporter.xsl +++ b/src/schema-gen/InspectorXSLT/mx-reporter.xsl @@ -26,9 +26,7 @@

Validation { @src ! replace(.,'.*/','') }

{ @src }

{ @elements } elements and { @attributes } attributes found in the document.

-
- -
+ @@ -40,6 +38,10 @@ + +

Metaschema: { . } (namespace '{ @namespace }')

+ +

Good news - nothing to report - the instance is valid.

diff --git a/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl b/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl new file mode 100644 index 00000000..4c75ec5d --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl b/src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl new file mode 100644 index 00000000..25f39d2e --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl b/src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl new file mode 100644 index 00000000..3edfddcc --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/current/computer_inspector.xsl b/src/schema-gen/InspectorXSLT/testing/current/computer_inspector.xsl index ee68e076..b138e41f 100644 --- a/src/schema-gen/InspectorXSLT/testing/current/computer_inspector.xsl +++ b/src/schema-gen/InspectorXSLT/testing/current/computer_inspector.xsl @@ -1,11 +1,11 @@ - + + exclude-result-prefixes="#all"> @@ -26,6 +26,7 @@ match="root()"> + @@ -335,7 +336,7 @@ - + { $validating-filename } - { $checked } - { mx:metaschema/@shortname } validation @@ -382,14 +383,14 @@ details p { margin: 0.2em 0em }

{ count(.//mx:report) } { mx:pluralize(count(.//mx:report),'issue') } reported.

- 💥 + 💥

-

Good news - nothing to report - the instance is valid. 🚀 +

Good news - nothing to report - the instance is valid. 🚀

@@ -552,7 +553,7 @@ details p { margin: 0.2em 0em } - + @@ -562,15 +563,15 @@ details p { margin: 0.2em 0em } computer/(usb-device) UNIQ uniqueness-violation - not(count(mx:key-matches-among-items(.,$selected,'UNQ_3',(@uuid),$within))=1) + not(count(mx:key-matches-among-items(.,$selected,'UNQ_5',((@uuid)),$within))=1) - With respect to its assigned (@uuid), this {name(.)} instance of computer/(usb-device) is expected to be unique within its {$within/name(.)}. {count(mx:key-matches-among-items(.,$selected,'UNQ_3',(@uuid),$within))} items are found with the value {string-join(((@uuid)),',')}. + select="not(count(mx:key-matches-among-items(.,$selected,'UNQ_5',((@uuid)),$within))=1)"/> + With respect to its assigned (@uuid), this {name(.)} instance of computer/(usb-device) is expected to be unique within its {$within/name(.)}. {count(mx:key-matches-among-items(.,$selected,'UNQ_5',((@uuid)),$within))} items are found with the value {string-join(((@uuid)),',')}. - @@ -582,10 +583,10 @@ details p { margin: 0.2em 0em } computer/(descendant::*[exists(@serial-number)]) UNIQ uniqueness-violation - not(count(mx:key-matches-among-items(.,$selected,'UNQ_4',(@serial-number),$within))=1) + not(count(mx:key-matches-among-items(.,$selected,'UNQ_6',((@serial-number)),$within))=1) - With respect to its assigned (@serial-number), this {name(.)} instance of computer/(descendant::*[exists(@serial-number)]) is expected to be unique within its {$within/name(.)}. {count(mx:key-matches-among-items(.,$selected,'UNQ_4',(@serial-number),$within))} items are found with the value {string-join(((@serial-number)),',')}. + select="not(count(mx:key-matches-among-items(.,$selected,'UNQ_6',((@serial-number)),$within))=1)"/> + With respect to its assigned (@serial-number), this {name(.)} instance of computer/(descendant::*[exists(@serial-number)]) is expected to be unique within its {$within/name(.)}. {count(mx:key-matches-among-items(.,$selected,'UNQ_6',((@serial-number)),$within))} items are found with the value {string-join(((@serial-number)),',')}. @@ -948,7 +949,7 @@ details p { margin: 0.2em 0em } - /computer/motherboard/(ata-socket) UNIQ uniqueness-violation - not(count(mx:key-matches-among-items(.,$selected,'UNQ_1',(@slot-no),$within))=1) + not(count(mx:key-matches-among-items(.,$selected,'UNQ_3',((@slot-no)),$within))=1) - With respect to its assigned (@slot-no), this {name(.)} instance of /computer/motherboard/(ata-socket) is expected to be unique within its {$within/name(.)}. {count(mx:key-matches-among-items(.,$selected,'UNQ_1',(@slot-no),$within))} items are found with the value {string-join(((@slot-no)),',')}. + select="not(count(mx:key-matches-among-items(.,$selected,'UNQ_3',((@slot-no)),$within))=1)"/> + With respect to its assigned (@slot-no), this {name(.)} instance of /computer/motherboard/(ata-socket) is expected to be unique within its {$within/name(.)}. {count(mx:key-matches-among-items(.,$selected,'UNQ_3',((@slot-no)),$within))} items are found with the value {string-join(((@slot-no)),',')}. @@ -1797,6 +1798,11 @@ details p { margin: 0.2em 0em } + + + + @@ -1865,7 +1871,7 @@ details p { margin: 0.2em 0em } + match="@id | @ref | @serial-number | @slot-no | @socket-no | @uuid | @currency | @name | @ns | @value | @class | @group | @illuminated"> gix.117 AOOP attribute-out-of-place @@ -1876,13 +1882,6 @@ details p { margin: 0.2em 0em } - - gix.748 - MRQA missing-required-attribute - empty(@id) - - Element { name() } requires attribute @id. - gix.445 MRQC missing-required-contents @@ -1908,6 +1907,9 @@ details p { margin: 0.2em 0em } + + + diff --git a/src/schema-gen/InspectorXSLT/testing/planning.md b/src/schema-gen/InspectorXSLT/testing/planning.md index b464beac..cd01373d 100644 --- a/src/schema-gen/InspectorXSLT/testing/planning.md +++ b/src/schema-gen/InspectorXSLT/testing/planning.md @@ -1,5 +1,17 @@ # Planning +## Testing + +- [ ] [Run CI/CD on forks?]( https://github.com/marketplace/actions/publish-test-results#support-fork-repositories-and-dependabot-branches) +- [ ] If not, then find a graceful way to error on failures, in forks +- [ ] Refactor testing in this directory + - [ ] smoke-test: is a functional XSLT produced from a valid Metaschema + - [ ] unit-test: + - current production tests (build out a little) + - report (mx) format to HTML presentation XSLT + - [ ] spec-test: current functional tests + +## Inspector XSLT - [ ] Revisit allowed-values semantics? - [ ] isolate an example (OSCAL?) diff --git a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh index 2ba89f3a..55d9efe5 100644 --- a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh +++ b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh @@ -1,48 +1,32 @@ #!/usr/bin/env bash -# Fail early if an error occurs -set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common/subcommand_common.bash -usage() { - cat <&2 - exit 1 -fi +usage() { + cat < Date: Fri, 9 Feb 2024 17:54:05 -0500 Subject: [PATCH 08/27] adjustments to test runtime --- ...OR-RUNTIME.xpl => INSPECTOR-XSLT-TEST.xpl} | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) rename src/schema-gen/InspectorXSLT/testing/{TEST-INSPECTOR-RUNTIME.xpl => INSPECTOR-XSLT-TEST.xpl} (79%) diff --git a/src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl b/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl similarity index 79% rename from src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl rename to src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl index 1c5fb2ea..911cdab7 100644 --- a/src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl +++ b/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl @@ -2,20 +2,24 @@ + type="metaschema:INSPECTOR-XSLT-TEST" name="INSPECTOR-XSLT-TEST"> - - - + + + + + + + - - - + @@ -80,9 +84,10 @@ + - + From 662cc4633228c7475167eb3343845d097b5924cb Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Mon, 12 Feb 2024 12:41:16 -0500 Subject: [PATCH 09/27] Further refinements to testing & scripts --- src/schema-gen/InspectorXSLT/Makefile | 2 +- .../testing/COMPUTER-INSPECTOR-PRODUCE.xpl | 13 ++--- .../testing/CURRENT-MODELS-REFRESH.xpl | 55 ------------------- .../testing/INSPECTOR-XSLT-TEST.xpl | 2 +- .../testing/VALIDATORS-REFRESH.xpl | 51 ----------------- .../testing/computer-testing.sch | 2 +- .../generate-oscal-catalog-inspector.sh | 30 ---------- .../InspectorXSLT/testing/inspect-computer.sh | 2 +- .../testing/refresh-computer-inspector.sh | 7 ++- 9 files changed, 15 insertions(+), 149 deletions(-) delete mode 100644 src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl delete mode 100644 src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl delete mode 100644 src/schema-gen/InspectorXSLT/testing/generate-oscal-catalog-inspector.sh diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index caa7a8d4..4d1bef3c 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -37,7 +37,7 @@ unit-test: ## Run all unit tests .PHONY: smoke-test smoke-test: ## Run smoke test - $(smoketest_script) + $(smoketest_script) -oINSPECTOR-XSLT=/dev/null .PHONY: clean clean: ## Remove test output diff --git a/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl b/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl index 4c75ec5d..81947288 100644 --- a/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl +++ b/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl @@ -1,25 +1,25 @@ + xmlns:metaschema="http://csrc.nist.gov/ns/metaschema/1.0" type="metaschema:COMPUTER-INSPECTOR-PRODUCE" + name="COMPUTER-INSPECTOR-PRODUCE"> - + + - - - + + @@ -35,5 +35,4 @@ - \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl b/src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl deleted file mode 100644 index 25f39d2e..00000000 --- a/src/schema-gen/InspectorXSLT/testing/CURRENT-MODELS-REFRESH.xpl +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl b/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl index 911cdab7..16c46525 100644 --- a/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl +++ b/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl @@ -49,7 +49,7 @@ --> - + diff --git a/src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl b/src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl deleted file mode 100644 index 3edfddcc..00000000 --- a/src/schema-gen/InspectorXSLT/testing/VALIDATORS-REFRESH.xpl +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/computer-testing.sch b/src/schema-gen/InspectorXSLT/testing/computer-testing.sch index c0cd58eb..baf4bc2a 100644 --- a/src/schema-gen/InspectorXSLT/testing/computer-testing.sch +++ b/src/schema-gen/InspectorXSLT/testing/computer-testing.sch @@ -5,7 +5,7 @@ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - + &2 - exit 1 -fi - -ADDITIONAL_ARGS=$(echo "${*// /\\ }") - -PIPELINE="${SCRIPT_DIR}/OSCAL-INSPECTOR-XSLT.xpl" - -CALABASH_ARGS="$ADDITIONAL_ARGS \"$PIPELINE\"" - -invoke_calabash "${CALABASH_ARGS}" diff --git a/src/schema-gen/InspectorXSLT/testing/inspect-computer.sh b/src/schema-gen/InspectorXSLT/testing/inspect-computer.sh index 583b1e01..eedf276b 100644 --- a/src/schema-gen/InspectorXSLT/testing/inspect-computer.sh +++ b/src/schema-gen/InspectorXSLT/testing/inspect-computer.sh @@ -7,7 +7,7 @@ usage() { cat < Date: Mon, 12 Feb 2024 15:03:39 -0500 Subject: [PATCH 10/27] Rationalizing including file name regularization --- .../testing/CURRENT-TEST-MODELS-REFRESH.xpl | 72 ++++++ ....xsl => computer_metaschema-inspector.xsl} | 0 ...ema.xsd => computer_metaschema-schema.xsd} | 0 ...ctor.xsl => tiny_metaschema-inspector.xsl} | 0 ...lschema.xsd => tiny_metaschema-schema.xsd} | 0 .../tiny_metaschema_COMPOSED-metaschema.xml | 242 ------------------ .../refresh-computer-inspector-saxon.sh | 52 ++++ .../testing/refresh-test-inspectors.sh | 33 +++ .../computer-constraints.xspec | 12 +- .../computer-modeling.xspec | 2 +- .../tiny-markupdatatypes.xspec | 2 +- .../validations-in-batch.xspec | 4 +- .../xspec-patterns.xspec | 4 +- .../produce-inspector.xspec | 2 +- src/schema-gen/METASCHEMA-XSD-TEST.xpl | 4 +- src/schema-gen/METASCHEMA-XSD.xpl | 42 +++ src/testing/xspec-assurance.sch | 10 +- 17 files changed, 221 insertions(+), 260 deletions(-) create mode 100644 src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl rename src/schema-gen/InspectorXSLT/testing/current/{computer_inspector.xsl => computer_metaschema-inspector.xsl} (100%) rename src/schema-gen/InspectorXSLT/testing/current/{computer_metaschema-xmlschema.xsd => computer_metaschema-schema.xsd} (100%) rename src/schema-gen/InspectorXSLT/testing/current/{tiny_inspector.xsl => tiny_metaschema-inspector.xsl} (100%) rename src/schema-gen/InspectorXSLT/testing/current/{tiny_metaschema-xmlschema.xsd => tiny_metaschema-schema.xsd} (100%) delete mode 100644 src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema_COMPOSED-metaschema.xml create mode 100644 src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh create mode 100644 src/schema-gen/InspectorXSLT/testing/refresh-test-inspectors.sh create mode 100644 src/schema-gen/METASCHEMA-XSD.xpl diff --git a/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl b/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl new file mode 100644 index 00000000..565d95cf --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/current/computer_inspector.xsl b/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl similarity index 100% rename from src/schema-gen/InspectorXSLT/testing/current/computer_inspector.xsl rename to src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl diff --git a/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-xmlschema.xsd b/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-schema.xsd similarity index 100% rename from src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-xmlschema.xsd rename to src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-schema.xsd diff --git a/src/schema-gen/InspectorXSLT/testing/current/tiny_inspector.xsl b/src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema-inspector.xsl similarity index 100% rename from src/schema-gen/InspectorXSLT/testing/current/tiny_inspector.xsl rename to src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema-inspector.xsl diff --git a/src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema-xmlschema.xsd b/src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema-schema.xsd similarity index 100% rename from src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema-xmlschema.xsd rename to src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema-schema.xsd diff --git a/src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema_COMPOSED-metaschema.xml b/src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema_COMPOSED-metaschema.xml deleted file mode 100644 index 7a0e9648..00000000 --- a/src/schema-gen/InspectorXSLT/testing/current/tiny_metaschema_COMPOSED-metaschema.xml +++ /dev/null @@ -1,242 +0,0 @@ - - - Seeing assembly#computer:TINY, field#computer:title, field#computer:term, assembly#computer:part, field#computer:note - Computer Model - 0.0.5 - computer - http://example.com/ns/tinydata - http://example.com/ns/tinydata - -

Tiny Metaschema

-

A metaschema can be small and yet deep.

- - - Tiny Data - A minimally described data set. - TINY - - Tiny ID - An identifier. - - - - - - Stuff - Some amount of tiny stuff. - - - - - - - - - - - Title - markup-line example. - - - Term - Simple string value example. - - - - - Note - Wrapped markup-multiline example. - - - Tiny part - A bit of tiny data. - - - - - Stuff - Some amount of tiny stuff. - - - - - - - - diff --git a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh new file mode 100644 index 00000000..8beba7bc --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# This script shows how to use Saxon with only a Maven dependency. +# Unlike other scripts in this folder it does not rely on an external (bash scripting) dependency +# You do need Maven installed.. + +# Fail early if an error occurs +set -Eeuo pipefail + +usage() { + cat <&2 + exit 1 +fi + +METASCHEMA_SOURCE=computer_metaschema.xml + +XSLT_RESULT=current/computer_metaschema-inspector.xsl + +ADDITIONAL_ARGS=$(echo ${*// /\\ }) + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" +POM_FILE="${SCRIPT_DIR}/../../../../support/pom.xml" + +MAIN_CLASS="net.sf.saxon.Transform" # Saxon defined in pom.xml + +if [ -e "$XSLT_RESULT" ] +then + echo "Deleting prior $XSLT_RESULT ..." + rm -f ./$XSLT_RESULT +fi + +mvn \ + -quiet \ + -f "$POM_FILE" \ + exec:java \ + -Dexec.mainClass="$MAIN_CLASS" \ + -Dexec.args="-xsl:${SCRIPT_DIR}/../../nist-metaschema-MAKE-INSPECTOR-XSLT.xsl -s:\"$METASCHEMA_SOURCE\" -o:\"$XSLT_RESULT\" $ADDITIONAL_ARGS" + +if [ -e "$XSLT_RESULT" ] +then + echo "XSLT written to file $XSLT_RESULT" +fi diff --git a/src/schema-gen/InspectorXSLT/testing/refresh-test-inspectors.sh b/src/schema-gen/InspectorXSLT/testing/refresh-test-inspectors.sh new file mode 100644 index 00000000..ac4841f1 --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/refresh-test-inspectors.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common/subcommand_common.bash + +source "$SCRIPT_DIR/../../../common/subcommand_common.bash" + +# XProc produces Inspector XSLT with a fail-safe check by compiling and running it +XPROC_FILE="file:${SCRIPT_DIR}/CURRENT-TEST-MODELS-REFRESH.xpl" + +usage() { + cat < - + @@ -308,10 +308,10 @@ - With respect to its assigned (@slot-no), this ata-socket instance of /computer/motherboard/(ata-socket) is expected to be unique within its motherboard. 2 items are found with the value 0. - With respect to its assigned (@slot-no), this ata-socket instance of /computer/motherboard/(ata-socket) is expected to be unique within its motherboard. 2 items are found with the value 0. @@ -339,10 +339,10 @@ - With respect to its assigned (@serial-number), this memory instance of computer/(descendant::*[exists(@serial-number)]) is expected to be unique within its computer. 2 items are found with the value a1. - With respect to its assigned (@serial-number), this expansion-card instance of computer/(descendant::*[exists(@serial-number)]) is expected to be unique within its computer. 2 items are found with the value a1. diff --git a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/computer-modeling.xspec b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/computer-modeling.xspec index 69099341..a6a538c7 100644 --- a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/computer-modeling.xspec +++ b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/computer-modeling.xspec @@ -2,7 +2,7 @@ diff --git a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/tiny-markupdatatypes.xspec b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/tiny-markupdatatypes.xspec index 4bbbd19a..8c71dcba 100644 --- a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/tiny-markupdatatypes.xspec +++ b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/tiny-markupdatatypes.xspec @@ -4,7 +4,7 @@ diff --git a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/validations-in-batch.xspec b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/validations-in-batch.xspec index 15c2365e..bb5540b4 100644 --- a/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/validations-in-batch.xspec +++ b/src/schema-gen/InspectorXSLT/testing/tests/inspector-functional-xspec/validations-in-batch.xspec @@ -2,8 +2,8 @@ + stylesheet="../../current/computer_metaschema-inspector.xsl" + xslt-version="3.0"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/testing/xspec-assurance.sch b/src/testing/xspec-assurance.sch index 6678a3c8..2eb1f0e1 100644 --- a/src/testing/xspec-assurance.sch +++ b/src/testing/xspec-assurance.sch @@ -5,9 +5,13 @@ - - No document found at - + + No resource found at + + + + No document found at + From c95285d438e03c775bf9d3e83eefd752f9e1f8d9 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Mon, 12 Feb 2024 16:24:59 -0500 Subject: [PATCH 11/27] Further configuration enhancements --- src/schema-gen/InspectorXSLT/Makefile | 4 +- .../testing/COMPUTER-INSPECTOR-PRODUCE.xpl | 2 +- .../testing/CURRENT-TEST-MODELS-REFRESH.xpl | 45 +++++++------ .../testing/INSPECTOR-XSLT-TEST.xpl | 63 +++---------------- .../current/computer_metaschema-inspector.xsl | 3 +- .../InspectorXSLT/testing/readme.md | 6 +- 6 files changed, 41 insertions(+), 82 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 4d1bef3c..0708de32 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -12,7 +12,7 @@ include ../../testing/make_common.mk module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) output_folder:=$(module_path)/test_output xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) -smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh) +smoketest_script=$(realpath $(module_path)/testing/refresh-computer-inspector.sh -oINSPECTOR-XSLT=/dev/null) .PHONY: test test: unit-test spec-test smoke-test ## Run all tests @@ -37,7 +37,7 @@ unit-test: ## Run all unit tests .PHONY: smoke-test smoke-test: ## Run smoke test - $(smoketest_script) -oINSPECTOR-XSLT=/dev/null + $(smoketest_script) .PHONY: clean clean: ## Remove test output diff --git a/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl b/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl index 81947288..025806c9 100644 --- a/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl +++ b/src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl @@ -20,7 +20,7 @@ - + diff --git a/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl b/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl index 565d95cf..db371d26 100644 --- a/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl +++ b/src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl @@ -21,52 +21,57 @@ + + + + + + - + - - + + - + - - + - - - - - - - - - - - + + + + + + + + + + + - + - - + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl b/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl index 16c46525..cf097793 100644 --- a/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl +++ b/src/schema-gen/InspectorXSLT/testing/INSPECTOR-XSLT-TEST.xpl @@ -6,8 +6,7 @@ - - + - - - + + - - - - - - - - - - - - - - - + @@ -93,17 +53,8 @@ - - - - - - - - - - - - + + + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl b/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl index b138e41f..0c6fde98 100644 --- a/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl +++ b/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl @@ -5,7 +5,7 @@ xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="3.0" xpath-default-namespace="http://example.com/ns/computer" - exclude-result-prefixes="#all"> + exclude-result-prefixes="#all"> @@ -26,7 +26,6 @@ match="root()"> - diff --git a/src/schema-gen/InspectorXSLT/testing/readme.md b/src/schema-gen/InspectorXSLT/testing/readme.md index 029f5909..9cb97756 100644 --- a/src/schema-gen/InspectorXSLT/testing/readme.md +++ b/src/schema-gen/InspectorXSLT/testing/readme.md @@ -12,7 +12,11 @@ Keep in mind when considering testing that Inspector XSLT currently only support Any valid and correct Metaschema module can be used to produce an Inspector for the XML format defined by that metaschema. Use the included script to produce your Inspector XSLT, or run the generator pipeline directly, not skipping composition, (Even a self-containted schema module in one file needs to be composed to link up definitions with their uses.) -See the top-level [readme](../readme.md) for more information. Please consider reporting what you learn. +In this subdirectory, two models (at time of writing) are used in harnesses for testing Metaschema-based XSLT generation (transpiling): `computer_metaschema.xml` and `tiny_metaschema.xml`. Together these are able to represent a broad range (ultimateley, the broadest range possible) of metaschema-based declarative functionality in modeling and constraint definition. At present they are also useful examples of small metaschema-based applications. + +This means the developer of any metaschema should be able to copy handling found here, especially for `computer_metaschema.xml`, such as scripts for [refreshing an InspectorXSLT for testing](refresh-computer-inspector.sh), or [simply to apply an InspectorXSLT](inspect-computer.sh) to an XML document nominated at the command line. + +See the [InspectorXSLT readme](../readme.md) for more information. Additionally, please consider reporting what you learn and letting project developers know of your needs, as a Metaschema consumer and implementor. ## Working with the trial (model) metaschemas From 903c1570e9ff3bf0e84ea0b58f50f3d611477866 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Mon, 12 Feb 2024 16:30:01 -0500 Subject: [PATCH 12/27] More help in docs --- src/schema-gen/InspectorXSLT/TESTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/TESTING.md b/src/schema-gen/InspectorXSLT/TESTING.md index 79e589ff..bcfeafb0 100644 --- a/src/schema-gen/InspectorXSLT/TESTING.md +++ b/src/schema-gen/InspectorXSLT/TESTING.md @@ -36,9 +36,9 @@ Don't commit unless this passes. Runs only the 'smoke tests' (end to end production pipeline testing - are viable artifacts produced irrespective of their functionality or correctness) Available: - - `smoke-test` - - `spec-test` - - `unit-test` + - `smoke-test` - builds an XSLT from the `computer_metaschema.xml` test example, and attempts to execute the resulting XSLT over stub input. + - `spec-test` - runs specification tests - does the produced XSLT behave as expected when used on the possible range of (XML) inputs? this is a validator: does it validate? + - `unit-test` - runs transformation template- and function-level unit tests regulating the mapping between source (Metaschema) and target (XSLT) expressions. ## Model metaschemas for testing From c9599a928e1724ed98d4488555775553c122c91a Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Mon, 12 Feb 2024 18:03:31 -0500 Subject: [PATCH 13/27] Further testing/tuning/tweaking test runtimes and docs --- src/schema-gen/InspectorXSLT/Makefile | 2 +- src/schema-gen/InspectorXSLT/TESTING.md | 4 +++- src/schema-gen/InspectorXSLT/testing/readme.md | 4 ++-- support/xspec-dev/mvn-saxon-xspec-batch.sh | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 0708de32..2c640a8f 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -15,7 +15,7 @@ xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspe smoketest_script=$(realpath $(module_path)/testing/refresh-computer-inspector.sh -oINSPECTOR-XSLT=/dev/null) .PHONY: test -test: unit-test spec-test smoke-test ## Run all tests +test: smoke-test unit-test spec-test ## Run all tests .PHONY: spec-test spec-test: ## Run all specification tests diff --git a/src/schema-gen/InspectorXSLT/TESTING.md b/src/schema-gen/InspectorXSLT/TESTING.md index bcfeafb0..15a12810 100644 --- a/src/schema-gen/InspectorXSLT/TESTING.md +++ b/src/schema-gen/InspectorXSLT/TESTING.md @@ -36,10 +36,12 @@ Don't commit unless this passes. Runs only the 'smoke tests' (end to end production pipeline testing - are viable artifacts produced irrespective of their functionality or correctness) Available: - - `smoke-test` - builds an XSLT from the `computer_metaschema.xml` test example, and attempts to execute the resulting XSLT over stub input. + - `smoke-test` - builds an XSLT from the `computer_metaschema.xml` test example, and attempts to execute the resulting XSLT over stub input. A failure indicates a problem in the production pipeline - it is either broken or wrong - `spec-test` - runs specification tests - does the produced XSLT behave as expected when used on the possible range of (XML) inputs? this is a validator: does it validate? - `unit-test` - runs transformation template- and function-level unit tests regulating the mapping between source (Metaschema) and target (XSLT) expressions. +Note this is work in progress and may change over time especially as we bring more tests in. + ## Model metaschemas for testing ### `current` resource cache diff --git a/src/schema-gen/InspectorXSLT/testing/readme.md b/src/schema-gen/InspectorXSLT/testing/readme.md index 9cb97756..0d2ca505 100644 --- a/src/schema-gen/InspectorXSLT/testing/readme.md +++ b/src/schema-gen/InspectorXSLT/testing/readme.md @@ -16,7 +16,7 @@ In this subdirectory, two models (at time of writing) are used in harnesses for This means the developer of any metaschema should be able to copy handling found here, especially for `computer_metaschema.xml`, such as scripts for [refreshing an InspectorXSLT for testing](refresh-computer-inspector.sh), or [simply to apply an InspectorXSLT](inspect-computer.sh) to an XML document nominated at the command line. -See the [InspectorXSLT readme](../readme.md) for more information. Additionally, please consider reporting what you learn and letting project developers know of your needs, as a Metaschema consumer and implementor. +See the [InspectorXSLT readme](../readme.md) for more information. Additionally, please consider reporting what you learn and letting project developers know of your needs as a Metaschema consumer and implementor. ## Working with the trial (model) metaschemas @@ -30,6 +30,6 @@ Additionally, `tiny_metaschema.xml` is a small metaschema made specifically for An Inspector can be generated from a metaschema such as `computer_model.xml` and tested against known inputs to demonstrate that the tests performed by the Inspector bring the correct results. -Exercising these tests, a number of XSpec files in this folder calling `current/computer_inspector.xsl` should all complete successfully and report "all green" -- no warnings, no errors, no unexpected 'pending' sections. +Exercising these tests, a number of XSpec files in this folder calling `current/computer_metaschema-inspector.xsl` should all complete successfully and report "all green" -- no warnings, no errors, no unexpected 'pending' sections. See the [TESTING](../TESTING.md) docs for more information. diff --git a/support/xspec-dev/mvn-saxon-xspec-batch.sh b/support/xspec-dev/mvn-saxon-xspec-batch.sh index b02c9c2e..12b4f321 100755 --- a/support/xspec-dev/mvn-saxon-xspec-batch.sh +++ b/support/xspec-dev/mvn-saxon-xspec-batch.sh @@ -27,8 +27,8 @@ Runtime parameters (use param=value syntax): folder: XSpec inputs are found in the targeted folder, relative to baseURI - defaults to 'src' (baseURI being set by default to the repository, this gives its /src directory) to process outside the repository, pass in an absolute URL or reset baseURI -Yeah -e.g. folder=src (sets folder to repository /src) + + e.g. folder=src (sets folder to repository /src) pattern: glob-like syntax for file name matching cf https://www.saxonica.com/html/documentation12/sourcedocs/collections/collection-directories.html '?select' From 9458ee903ad23707a90c06f5e5e5add45081af8a Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Mon, 12 Feb 2024 18:15:49 -0500 Subject: [PATCH 14/27] Further alignment --- src/schema-gen/InspectorXSLT/Makefile | 2 +- .../testing/current/computer_metaschema-inspector.xsl | 2 +- .../InspectorXSLT/testing/refresh-computer-inspector.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 2c640a8f..8dc87ff6 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -12,7 +12,7 @@ include ../../testing/make_common.mk module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) output_folder:=$(module_path)/test_output xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) -smoketest_script=$(realpath $(module_path)/testing/refresh-computer-inspector.sh -oINSPECTOR-XSLT=/dev/null) +smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh) .PHONY: test test: smoke-test unit-test spec-test ## Run all tests diff --git a/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl b/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl index 0c6fde98..4865e3aa 100644 --- a/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl +++ b/src/schema-gen/InspectorXSLT/testing/current/computer_metaschema-inspector.xsl @@ -5,7 +5,7 @@ xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="3.0" xpath-default-namespace="http://example.com/ns/computer" - exclude-result-prefixes="#all"> + exclude-result-prefixes="#all"> diff --git a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh index fbe89375..f010ebbf 100644 --- a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh +++ b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector.sh @@ -8,7 +8,7 @@ source "$SCRIPT_DIR/../../../common/subcommand_common.bash" # XProc produces Inspector XSLT with a fail-safe check by compiling and running it XPROC_FILE="${SCRIPT_DIR}/COMPUTER-INSPECTOR-PRODUCE.xpl" -XSLT_RESULT=current/computer_metaschema-inspector.xsl +XSLT_RESULT=${SCRIPT_DIR}/current/computer_metaschema-inspector.xsl usage() { cat < Date: Tue, 13 Feb 2024 10:23:17 -0500 Subject: [PATCH 15/27] New 'quiet' XSpec batching script, plus adjustment --- .../xspec-dev/mvn-saxon-xspec-batch-quiet.sh | 80 +++++++++++++++++++ support/xspec-dev/mvn-saxon-xspec-batch.sh | 7 -- 2 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh diff --git a/support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh b/support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh new file mode 100644 index 00000000..9c64eb4e --- /dev/null +++ b/support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common/subcommand_common.bash + +source "$SCRIPT_DIR/../../src/common/subcommand_common.bash" + +XSLT_FILE="${SCRIPT_DIR}/XSPEC-BATCH.xsl" +LOGFILE=${LOGFILE:-"xspec_$(date +"%Y%m%d%H%M").log.txt"} + +usage() { + cat </dev/null to drop all runtime messages / progress reports instead of logging +# the process should error out only if stop-on-error=yes, otherwise it will do its best to complete +# invoke_saxon "${SAXON_ARGS}" 2>${LOGFILE} | echo_on_error "Failure running XSpec: see ${LOGFILE}" | tee ${LOGFILE} +invoke_saxon "${SAXON_ARGS}" 2>${LOGFILE} | tee ${LOGFILE} diff --git a/support/xspec-dev/mvn-saxon-xspec-batch.sh b/support/xspec-dev/mvn-saxon-xspec-batch.sh index 12b4f321..16c34ae0 100755 --- a/support/xspec-dev/mvn-saxon-xspec-batch.sh +++ b/support/xspec-dev/mvn-saxon-xspec-batch.sh @@ -65,13 +65,6 @@ theme=(clean|classic|toybox|uswds) defaulting to 'clean' EOF } -echo_on_error() - -{ - echo "Error: $1" - exit 1 -} - ADDITIONAL_ARGS=$(echo "${*// /\\ }") SAXON_ARGS="-it:go -xsl:\"${XSLT_FILE}\" -init:org.nineml.coffeesacks.RegisterCoffeeSacks \ From 551757c2179e3540bbbb29aef633386f00680181 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 13 Feb 2024 10:47:56 -0500 Subject: [PATCH 16/27] New 'make' feature for testing InspectorXSLT --- src/schema-gen/InspectorXSLT/Makefile | 21 ++++- .../testing/TEST-INSPECTOR-RUNTIME.xpl | 91 +++++++++++++++++++ .../testing/smoketest-computer-inspector.sh | 32 +++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl create mode 100644 src/schema-gen/InspectorXSLT/testing/smoketest-computer-inspector.sh diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 8dc87ff6..354640ca 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -11,15 +11,18 @@ include ../../testing/make_common.mk module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) output_folder:=$(module_path)/test_output -xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) +xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh) +xspec_ci_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh) +folder=. .PHONY: test test: smoke-test unit-test spec-test ## Run all tests + .PHONY: spec-test spec-test: ## Run all specification tests - LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_script) \ + LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_ci_script) \ "folder=$(module_path)/testing/tests/inspector-functional-xspec" \ "report-to=$(output_folder)/inspector-functional-tests_report.html" \ "junit-to=$(output_folder)/inspector-functional-tests_junit-report.xml" \ @@ -28,7 +31,7 @@ spec-test: ## Run all specification tests .PHONY: unit-test unit-test: ## Run all unit tests - LOGFILE="$(output_folder)/integration-tests.log" $(xspec_script) \ + LOGFILE="$(output_folder)/integration-tests.log" $(xspec_ci_script) \ "folder=$(module_path)/testing/tests/inspector-generation-xspec" \ "report-to=$(output_folder)/integration-tests_report.html" \ "junit-to=$(output_folder)/integration-tests_junit-report.xml" \ @@ -39,6 +42,18 @@ unit-test: ## Run all unit tests smoke-test: ## Run smoke test $(smoketest_script) + +.PHONY: xspec +xspec: ## Run all *.xspec in folder $(folder) + LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_script) \ + "baseURI=file:$(module_path)/" \ + "folder=$(folder)" \ + "report-to=$(output_folder)/inspector-functional-tests_report.html" \ + "junit-to=$(output_folder)/inspector-functional-tests_junit-report.xml" \ + "stop-on-error=no" \ + "recurse=yes" + + .PHONY: clean clean: ## Remove test output rm -fr $(output_folder)/* \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl b/src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl new file mode 100644 index 00000000..07380b31 --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/TEST-INSPECTOR-RUNTIME.xpl @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/smoketest-computer-inspector.sh b/src/schema-gen/InspectorXSLT/testing/smoketest-computer-inspector.sh new file mode 100644 index 00000000..db1e7d32 --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/smoketest-computer-inspector.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common/subcommand_common.bash + +source "$SCRIPT_DIR/../../../common/subcommand_common.bash" + +# XProc produces Inspector XSLT with a fail-safe check by compiling and running it +XPROC_FILE="${SCRIPT_DIR}/COMPUTER-INSPECTOR-PRODUCE.xpl" + +usage() { + cat < Date: Tue, 13 Feb 2024 12:09:20 -0500 Subject: [PATCH 17/27] More Makefile edits / cleanup --- src/schema-gen/InspectorXSLT/Makefile | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 354640ca..d3dedff4 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -1,19 +1,19 @@ include ../../testing/make_common.mk -# TODO (sort out): -# unit testing - XSLT production templates -# smoke testing - whether an XSLT is produced and compiles in a run -# trying on available samples? -# use testing/TEST-INSPECTOR-RUNTIME.xpl -# spec testing - functional runtime tests of the generated XSLT - -# refresh-inspector - run XSLT production for test metaschemas writing to current (and smoke test) +# INCLUDES: +# unit-test - unit testing - XSLT production templates for InspectorXSLT +# smoke-test - smoke testing - whether an XSLT is produced and compiles in a run +# spec-test - specification conformance testing - functional runtime tests of the generated XSLT +# refresh-testing - update InspectorXSLT and XSD for test metaschemas +# xspec - run XSpec tests in designated folder +# clean - clean up designated output folder module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) output_folder:=$(module_path)/test_output xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh) xspec_ci_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh) smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh) +testing-refresh_script=$(realpath $(module_path)/testing/refresh-test-inspectors.sh) folder=. .PHONY: test @@ -31,10 +31,10 @@ spec-test: ## Run all specification tests .PHONY: unit-test unit-test: ## Run all unit tests - LOGFILE="$(output_folder)/integration-tests.log" $(xspec_ci_script) \ + LOGFILE="$(output_folder)/inspector-generation-tests.log" $(xspec_ci_script) \ "folder=$(module_path)/testing/tests/inspector-generation-xspec" \ - "report-to=$(output_folder)/integration-tests_report.html" \ - "junit-to=$(output_folder)/integration-tests_junit-report.xml" \ + "report-to=$(output_folder)/inspector-generation_report.html" \ + "junit-to=$(output_folder)/inspector-generation_junit-report.xml" \ "stop-on-error=yes" \ "recurse=yes" @@ -42,10 +42,13 @@ unit-test: ## Run all unit tests smoke-test: ## Run smoke test $(smoketest_script) +.PHONY: refresh-testing +refresh-testing: ## Update InspectorXSLT and XSD for test metaschemas (doesn't run tests) + $(testing-refresh_script) .PHONY: xspec xspec: ## Run all *.xspec in folder $(folder) - LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_script) \ + LOGFILE="$(output_folder)/$(folder)-xspec-tests.log" $(xspec_script) \ "baseURI=file:$(module_path)/" \ "folder=$(folder)" \ "report-to=$(output_folder)/inspector-functional-tests_report.html" \ From 7dfc6776631da056fc6a43e88cd941db62c01b09 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 13 Feb 2024 13:15:57 -0500 Subject: [PATCH 18/27] Further docs and readme improvements --- src/schema-gen/InspectorXSLT/Makefile | 3 +++ src/schema-gen/TESTING.md | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index d3dedff4..62b7a6ab 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -38,14 +38,17 @@ unit-test: ## Run all unit tests "stop-on-error=yes" \ "recurse=yes" + .PHONY: smoke-test smoke-test: ## Run smoke test $(smoketest_script) + .PHONY: refresh-testing refresh-testing: ## Update InspectorXSLT and XSD for test metaschemas (doesn't run tests) $(testing-refresh_script) + .PHONY: xspec xspec: ## Run all *.xspec in folder $(folder) LOGFILE="$(output_folder)/$(folder)-xspec-tests.log" $(xspec_script) \ diff --git a/src/schema-gen/TESTING.md b/src/schema-gen/TESTING.md index 807ad508..aad87dd1 100644 --- a/src/schema-gen/TESTING.md +++ b/src/schema-gen/TESTING.md @@ -25,3 +25,22 @@ Use models_metaschema.xml XProc to generate a schema dynamically and then validate a set of documents? +## XSD Schemas + +XSD Schema production was not originally developed using test-driven-development practices. We are now catching up. + +Work to be done includes comprehensive XSpecs for XSD production. A dependency for this is a specification for how to represent Metaschema in XSD, either established by Metaschema itself, or for this application. The current XSD generated provides a place to start. + +## JSON Schemas + +The pipelines generating JSON Schema have some ad-hoc unit testing developed to support bug fixing, buts await comprehensive unit tests. + +A separate work item is to update this generation to support more recent versions of JSON Schema - while that too demands unit tests. + +## InspectorXSLT + +The InspectorXSLT application produces an XSLT representing the constraints defined by a Metaschema, both model-based constraints (thereby in emulation of grammar-based validators such as XML DTD, W3C XSD and ISO RelaxNG) and constraints based on arbitrary assertions. + +It thus combines in one the functional properties of Schematron with XSD used as a validation application. + +Its testing is also the most advanced in this repository at time of writing, including smoke tests, unit tests and specification testing, and growing. See documentation in the folder including [readme](InspectorXSLT/readme.md) and [testing](InspectorXSLT/TESTING.md) documentation. From c4c6c2c9c507192a79c23db1ff61499195aa3651 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 13 Feb 2024 13:16:46 -0500 Subject: [PATCH 19/27] Removed obsolete Schematron implementation, no longer of interest --- .../Schematron/make-metaschema-metatron.xsl | 629 ------------------ .../metatron-datatype-functions.xsl | 107 --- 2 files changed, 736 deletions(-) delete mode 100644 src/schema-gen/Schematron/make-metaschema-metatron.xsl delete mode 100644 src/schema-gen/Schematron/metatron-datatype-functions.xsl diff --git a/src/schema-gen/Schematron/make-metaschema-metatron.xsl b/src/schema-gen/Schematron/make-metaschema-metatron.xsl deleted file mode 100644 index cf73b735..00000000 --- a/src/schema-gen/Schematron/make-metaschema-metatron.xsl +++ /dev/null @@ -1,629 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - no - - - yes - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - INDEX DEFINITIONS AS KEY DECLARATIONS - - - - - RULES - - - - - - - - - - - - - - - - - - - LEXICAL / DATATYPE VALIDATION FUNCTIONS - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - not({ $condition }) or - - - - is expected to have at most of - - - - - - - - - not({ $condition }) or - - - - is expected to have at least of - - - - - - - - - not({ $condition }) or - - count({$target}) eq { xs:integer(@min-occurs) } - - - - is expected to have exactly of - - - - - - - - - - - - - - - not({ m:condition(.) }) or - - exists(key('{@name}',$lookup,ancestor::{$parent-context})) - - - - - - - - is expected to correspond to an entry in the '' index within the containing - - - - - - - - , - ( - - [matches(.,'^{.}$')] ! replace(.,'^{.}$','$1') - ) - - - - - - - not({ m:condition(.) }) or - - exists(key('{@name}',$lookup)) - - - - - - - - is expected to correspond to an entry in the '' index. - - - - - - - - - - not({ m:condition(.) }) or - - exactly-one(key('{@name}',{m:key-value(.)},ancestor::{$parent-context}))) - - - - is expected to be unique within the containing - - - - - - - - not({ m:condition(.) }) or - - count(key('{@name}',{m:key-value(.)}))=1 - - - - is expected to be unique. - - - - - - - not({ m:condition(.) }) or - - exists(self::node()[{ @test }]) - - - - fails to pass evaluation of '' - - - - - - - not({ m:condition(parent::matches) }) or - - m:datatype-validate(., '{.}') - - - - is expected to take the form of datatype ' - - - - - - - not({ m:condition(parent::matches) }) or - - matches(., '^{.}$') - - - - - - is expected to match regular expression '^$' - - - - - - - - - This - - - - - - - - - - - - not({ $this-condition }) or - - - - ({ @allow-other[.='yes']/'$silence-warnings or ' }. = ( { $value-sequence } )) - - - - - warning - - - is expected to be (one of) , not '' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - / - - - - - - - - / - - - - - - - - / - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { $declaration-prefix }:{ $whose/string(.) } - - - - - - - - - - - - - - - - - - - - - - - ( (: ... not liking absolute path - - ... :) ) - - - - (parent|ancestor|ancestor-or-self|preceding-sibling|preceding):: - - - (: not liking the reverse axis - - :) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { if ($count eq 1) then ('one ' || $noun) else ($count || ' ' || $noun || 's' ) } - - - - - [[See {local-name()}#{.}]] - - - - - - - - - - - - - - / - - - - - - - yes - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/schema-gen/Schematron/metatron-datatype-functions.xsl b/src/schema-gen/Schematron/metatron-datatype-functions.xsl deleted file mode 100644 index b70ff7af..00000000 --- a/src/schema-gen/Schematron/metatron-datatype-functions.xsl +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - - - - - - ../../../support/metaschema/ - - {$metaschema-repository}/schema/xml/metaschema-datatypes.xsd - - - - - - - - - - - - - - - - - - - - - - string - string - NCName - { $nominal-type } - - - - {$value} - - - - - - - - - - - - - - - - - - - - true() - - - - - - - - - - - - - - - - - - - - From 7e7bf1f7cb9fd95ce9dce895581a8ab47a0d4f49 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 13 Feb 2024 16:04:40 -0500 Subject: [PATCH 20/27] More polishing --- src/schema-gen/InspectorXSLT/Makefile | 3 +-- support/xspec-dev/xspec-summary-reduce.xsl | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 62b7a6ab..9182c800 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -54,8 +54,7 @@ xspec: ## Run all *.xspec in folder $(folder) LOGFILE="$(output_folder)/$(folder)-xspec-tests.log" $(xspec_script) \ "baseURI=file:$(module_path)/" \ "folder=$(folder)" \ - "report-to=$(output_folder)/inspector-functional-tests_report.html" \ - "junit-to=$(output_folder)/inspector-functional-tests_junit-report.xml" \ + "report-to=$(output_folder)/inspector-$(folder)-tests_report.html" \ "stop-on-error=no" \ "recurse=yes" diff --git a/support/xspec-dev/xspec-summary-reduce.xsl b/support/xspec-dev/xspec-summary-reduce.xsl index 351c588b..39ba50ce 100644 --- a/support/xspec-dev/xspec-summary-reduce.xsl +++ b/support/xspec-dev/xspec-summary-reduce.xsl @@ -15,6 +15,7 @@ + { (1 to 14) ! ' ...' } XSpec summary report: { mx:enumerate('XSpec',count(report)) } { (1 to 14) ! ' ...' } @@ -22,7 +23,6 @@ - SUCCESS - { mx:give-report-counts(.) } - NO FAILURES REPORTED From 9abc41046ea2dcc6e74d111637602daa927b5448 Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 13 Feb 2024 17:09:01 -0500 Subject: [PATCH 21/27] Further small but useful improvements and cleanup --- src/schema-gen/InspectorXSLT/Makefile | 4 +- src/schema-gen/InspectorXSLT/readme.md | 8 +++ .../testing/OSCAL-INSPECTOR-XSLT.xpl | 65 ------------------- .../refresh-computer-inspector-saxon.sh | 5 +- 4 files changed, 13 insertions(+), 69 deletions(-) delete mode 100644 src/schema-gen/InspectorXSLT/testing/OSCAL-INSPECTOR-XSLT.xpl diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile index 9182c800..5a9e87bb 100644 --- a/src/schema-gen/InspectorXSLT/Makefile +++ b/src/schema-gen/InspectorXSLT/Makefile @@ -40,7 +40,7 @@ unit-test: ## Run all unit tests .PHONY: smoke-test -smoke-test: ## Run smoke test +smoke-test: ## Run InspectorXSLT production smoke test $(smoketest_script) @@ -50,7 +50,7 @@ refresh-testing: ## Update InspectorXSLT and XSD for test metaschemas (doesn't r .PHONY: xspec -xspec: ## Run all *.xspec in folder $(folder) +xspec: ## Run all *.xspec in a designated folder, quietly - use folder=[folder] LOGFILE="$(output_folder)/$(folder)-xspec-tests.log" $(xspec_script) \ "baseURI=file:$(module_path)/" \ "folder=$(folder)" \ diff --git a/src/schema-gen/InspectorXSLT/readme.md b/src/schema-gen/InspectorXSLT/readme.md index 37e9a563..902b2d03 100644 --- a/src/schema-gen/InspectorXSLT/readme.md +++ b/src/schema-gen/InspectorXSLT/readme.md @@ -19,6 +19,14 @@ The XSLT that is created this way can be used to test XML instances for errors i That is, it combines the effective functionality of XML schema and Schematron (XPath-based) validation. +## Limitations + +Aiming at a complete and conformant implementation for XML of the [Metaschema](https://pages.nist.gov/metaschema) language version 1.0, there are a few known gaps, with a testing framework and protocols now being established both for filling those gaps and for demonstrating the tool's full functionality. Please use the processor with this in mind, that developers and sponsors need your feedback and contributions to enable us to complete the work. + +Currently we plan to support only XML-based formats as defined by Metaschema, not their semantically-equivalent JSON or YAML 'cousins'. + +Users of Metaschema-defined JSON can try reformatting their data as XML using automated means such as scripts produced by the [Metaschema XSLT Converter Generators](../../converter-gen). Successful conversion will be valid on the other side. But failures will be indicated not typically be invalid results, and instead results missing those parts of the invalid that went unrecognized by the converter because they are invalid. + ## Demo The [testing/current](testing/current) directory shows such an XSLT, which can be applied to an instance or set of instances (documents) to be tested against the rules defined by its metaschema - in this case the Computer Model metaschema example provided. diff --git a/src/schema-gen/InspectorXSLT/testing/OSCAL-INSPECTOR-XSLT.xpl b/src/schema-gen/InspectorXSLT/testing/OSCAL-INSPECTOR-XSLT.xpl deleted file mode 100644 index c135ff3d..00000000 --- a/src/schema-gen/InspectorXSLT/testing/OSCAL-INSPECTOR-XSLT.xpl +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh index 8beba7bc..0331ed90 100644 --- a/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh +++ b/src/schema-gen/InspectorXSLT/testing/refresh-computer-inspector-saxon.sh @@ -2,7 +2,8 @@ # This script shows how to use Saxon with only a Maven dependency. # Unlike other scripts in this folder it does not rely on an external (bash scripting) dependency -# You do need Maven installed.. +# Another difference is that it bypasses the XSLT-checking, so GIGO - this can be useful (for producing an XSLT for debugging) +# Maven is required on your system PATH. # Fail early if an error occurs set -Eeuo pipefail @@ -11,7 +12,7 @@ usage() { cat < Date: Tue, 13 Feb 2024 17:42:49 -0500 Subject: [PATCH 22/27] Added another utility script --- .../testing/refresh-inspector-anyway.sh | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/schema-gen/InspectorXSLT/testing/refresh-inspector-anyway.sh diff --git a/src/schema-gen/InspectorXSLT/testing/refresh-inspector-anyway.sh b/src/schema-gen/InspectorXSLT/testing/refresh-inspector-anyway.sh new file mode 100644 index 00000000..d057cca2 --- /dev/null +++ b/src/schema-gen/InspectorXSLT/testing/refresh-inspector-anyway.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# This script produces an Inspector XSLT, except for a metaschema designated at runtime. +# Like refresh-computer-inspector-saxon.sh, it bypasses the XSLT-checking, so GIGO (useful for diagnostics). +# The resulting XSLT is suffixed 'X' in the file (base) name to indicate its untested status. +# Maven is required on your system PATH. + +# Fail early if an error occurs +set -Eeuo pipefail + +usage() { + cat <&2 + exit 1 +fi + +[[ -z "${1-}" ]] && { echo "Error: METASCHEMA_SOURCE not specified"; usage; exit 1; } +METASCHEMA_SOURCE=$1 + +if [ ! -f "${METASCHEMA_SOURCE}" ]; then + echo "${METASCHEMA_SOURCE} not found." +fi + +XSLT_RESULT=current/$(basename "$METASCHEMA_SOURCE" .xml)-inspectorX.xsl + +ADDITIONAL_ARGS=$(shift 1; echo ${*// /\\ }) + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" +POM_FILE="${SCRIPT_DIR}/../../../../support/pom.xml" + +MAIN_CLASS="net.sf.saxon.Transform" # Saxon defined in pom.xml + +if [ -e "$XSLT_RESULT" ] +then + echo "Deleting prior $XSLT_RESULT ..." + rm -f ./$XSLT_RESULT +fi + +mvn \ + -quiet \ + -f "$POM_FILE" \ + exec:java \ + -Dexec.mainClass="$MAIN_CLASS" \ + -Dexec.args="-xsl:${SCRIPT_DIR}/../../nist-metaschema-MAKE-INSPECTOR-XSLT.xsl -s:\"$METASCHEMA_SOURCE\" -o:\"$XSLT_RESULT\" $ADDITIONAL_ARGS" + +if [ -e "$XSLT_RESULT" ] +then + echo "XSLT written to file $XSLT_RESULT" +fi From 8f412b914a8a5b5f62500cca1cad7a06c0c7e28c Mon Sep 17 00:00:00 2001 From: Wendell Piez Date: Tue, 13 Feb 2024 17:57:20 -0500 Subject: [PATCH 23/27] Continuing to touch up --- src/schema-gen/InspectorXSLT/testing/planning.md | 13 ++++++------- support/xspec-dev/xspec-summary-reduce.xsl | 5 ++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/schema-gen/InspectorXSLT/testing/planning.md b/src/schema-gen/InspectorXSLT/testing/planning.md index cd01373d..0eca38cc 100644 --- a/src/schema-gen/InspectorXSLT/testing/planning.md +++ b/src/schema-gen/InspectorXSLT/testing/planning.md @@ -4,12 +4,12 @@ - [ ] [Run CI/CD on forks?]( https://github.com/marketplace/actions/publish-test-results#support-fork-repositories-and-dependabot-branches) - [ ] If not, then find a graceful way to error on failures, in forks -- [ ] Refactor testing in this directory - - [ ] smoke-test: is a functional XSLT produced from a valid Metaschema - - [ ] unit-test: +- [x] Refactor testing in this directory + - [x] smoke-test: is a functional XSLT produced from a valid Metaschema + - [x] unit-test: - current production tests (build out a little) - report (mx) format to HTML presentation XSLT - - [ ] spec-test: current functional tests + - [x] spec-test: current functional tests ## Inspector XSLT @@ -17,6 +17,8 @@ - [ ] isolate an example (OSCAL?) - [ ] XSpec a retrieval function returning (rule/value)* - [ ] check back, anything missed? +- [ ] Address OSCAL functional gaps + - [ ] OSCAL implementation in oscal-xslt repository - [ ] More assertively test all constraints - [ ] More assertively test markup-line and markup-multiline - [ ] Test out other aspects @@ -24,9 +26,6 @@ - [ ] XSpec MX to HTML to Markdown? functions? - [ ] Build out TINY to include constraints on insert (note-ref and term-ref) - [ ] Review specs with eye to contributions -- [ ] OSCAL implementation - - [ ] Publishing considerations - - [ ] Pages / platform - [ ] CSX version - [ ] CI/CD support for deployment from updated metaschema sources? - [ ] Release plans and considerations diff --git a/support/xspec-dev/xspec-summary-reduce.xsl b/support/xspec-dev/xspec-summary-reduce.xsl index 39ba50ce..fad32c0f 100644 --- a/support/xspec-dev/xspec-summary-reduce.xsl +++ b/support/xspec-dev/xspec-summary-reduce.xsl @@ -24,14 +24,13 @@ - + SUCCESS - { mx:give-report-counts(.) } - NO FAILURES REPORTED - - + FAILURE - { mx:give-report-counts(.) } - { count(descendant::fail) } { mx:pluralize('failure',count(descendant::fail)) => upper-case() } REPORTED From cf2ce9ec256e491e0fa063f385b5a984c94ce186 Mon Sep 17 00:00:00 2001 From: "A.J. Stein" Date: Fri, 16 Feb 2024 09:04:07 -0500 Subject: [PATCH 24/27] Add permissions for EnricoMi/publish-unit-test-result-action Using the guidance from the action's instructions: https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56dc13a5..402a94c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,9 @@ on: - main - dev pull_request: {} +permissions: + checks: write + pull-requests: write env: JAVA_VERSION: "17" JAVA_DISTRIBUTION: "temurin" @@ -32,7 +35,6 @@ jobs: run: | make -C src spec-test id: spec-tests - # Publish the test summary as comment on the PR - name: Publish XSpec Test Results Summary uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 From ca2c5388b5bd2e66a133968c40cf95df5099000f Mon Sep 17 00:00:00 2001 From: "A.J. Stein" Date: Fri, 16 Feb 2024 09:15:11 -0500 Subject: [PATCH 25/27] Add artifact event pull for EnricoMi/publish-unit-test-result-action Using the guidance from the action's instructions: https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches --- .github/workflows/publish.yml | 32 ++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 19 ++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..3d68a896 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,32 @@ +# Rationale and instructions for this GitHub Actions workflow: +# https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches + +name: Test Results +on: + workflow_run: + workflows: ["CI"] + types: + - completed +permissions: {} +jobs: + test-results: + name: Test Results + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + permissions: + checks: write + pull-requests: write + actions: read + steps: + - name: Download and Extract Artifacts + uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 + with: + run_id: ${{ github.event.workflow_run.id }} + path: artifacts + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 + with: + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Test Results Event/event.json + event_name: ${{ github.event.workflow_run.event }} + files: "**/*_junit-report.xml" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 402a94c0..9c75720a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,8 @@ env: JAVA_VERSION: "17" JAVA_DISTRIBUTION: "temurin" jobs: - deploy: + test: + name: "Test and Collect Results" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 @@ -45,3 +46,19 @@ jobs: files: "**/*_junit-report.xml" report_individual_runs: true deduplicate_classes_by_file_name: false + - name: Upload test results + if: always() + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + name: Test Results + path: | + **/*_junit-report.xml + event_file: + name: "Upload Results to Event File" + runs-on: ubuntu-20.04 + steps: + - name: Upload + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + name: Test Results Event + path: ${{ github.event_path }} From 1e838c9883c3a344f3c217e7104eae9b1e1e0cf5 Mon Sep 17 00:00:00 2001 From: "A.J. Stein" Date: Fri, 16 Feb 2024 10:11:48 -0500 Subject: [PATCH 26/27] Ensure XSpec test results comment always made on PR --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c75720a..c668d69e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,7 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} check_name: XSpec Test Results + comment_mode: always files: "**/*_junit-report.xml" report_individual_runs: true deduplicate_classes_by_file_name: false From 35629a11b4c47d229cb3f4a090cfaf078245cb0d Mon Sep 17 00:00:00 2001 From: "A.J. Stein" Date: Fri, 16 Feb 2024 10:31:23 -0500 Subject: [PATCH 27/27] [WIP] Dedupe event trigger and inline JUnit XML processing in CI workflow --- .github/workflows/publish.yml | 9 ++++++++- .github/workflows/test.yml | 14 ++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3d68a896..6c2a5984 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,7 +1,7 @@ # Rationale and instructions for this GitHub Actions workflow: # https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches -name: Test Results +name: Publish Test Results on: workflow_run: workflows: ["CI"] @@ -27,6 +27,13 @@ jobs: uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 with: commit: ${{ github.event.workflow_run.head_sha }} + action_fail: true + fail_on: "test failures" event_file: artifacts/Test Results Event/event.json event_name: ${{ github.event.workflow_run.event }} + github_token: ${{ secrets.GITHUB_TOKEN }} + check_name: XSpec Test Results + comment_mode: always files: "**/*_junit-report.xml" + report_individual_runs: true + deduplicate_classes_by_file_name: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c668d69e..ac6e0ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ on: push: branches: - main - - dev + - develop pull_request: {} permissions: checks: write @@ -36,17 +36,6 @@ jobs: run: | make -C src spec-test id: spec-tests - # Publish the test summary as comment on the PR - - name: Publish XSpec Test Results Summary - uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 - if: runner.os == 'Linux' - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - check_name: XSpec Test Results - comment_mode: always - files: "**/*_junit-report.xml" - report_individual_runs: true - deduplicate_classes_by_file_name: false - name: Upload test results if: always() uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 @@ -57,6 +46,7 @@ jobs: event_file: name: "Upload Results to Event File" runs-on: ubuntu-20.04 + needs: test steps: - name: Upload uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3