Adjustments to XSpec reporting under CI

.github/workflows/publish.yml

@@ -0,0 +1,39 @@
+# Rationale and instructions for this GitHub Actions workflow:
+# https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches
+
+name: Publish Test Results
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+permissions: {}
+jobs:
+  test-results:
+    name: Test Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Download and Extract Artifacts
+        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          path: artifacts
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          action_fail: true
+          fail_on: "test failures"
+          event_file: artifacts/Test Results Event/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          check_name: XSpec Test Results
+          comment_mode: always
+          files: "**/*_junit-report.xml"
+          report_individual_runs: true
+          deduplicate_classes_by_file_name: false

.github/workflows/test.yml

@@ -3,13 +3,17 @@ on:
   push:
     branches:
       - main
-      - dev
+      - develop
   pull_request: {}
+permissions:
+  checks: write
+  pull-requests: write  
 env:
   JAVA_VERSION: "17"
   JAVA_DISTRIBUTION: "temurin"
 jobs:
-  deploy:
+  test:
+    name: "Test and Collect Results"
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
@@ -32,14 +36,20 @@ jobs:
         run: |
           make -C src spec-test
         id: spec-tests
-
-      # Publish the test summary as comment on the PR
-      - name: Publish XSpec Test Results Summary
-        uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98
-        if: runner.os == 'Linux'
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          check_name: XSpec Test Results
-          files: "**/*_junit-report.xml"
-          report_individual_runs: true
-          deduplicate_classes_by_file_name: false
+          name: Test Results
+          path: |
+            **/*_junit-report.xml
+  event_file:
+    name: "Upload Results to Event File"
+    runs-on: ubuntu-20.04
+    needs: test
+    steps:
+    - name: Upload
+      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+      with:
+        name: Test Results Event
+        path: ${{ github.event_path }}

src/schema-gen/InspectorXSLT/Makefile

@@ -1,28 +1,64 @@
 include ../../testing/make_common.mk
 
+# INCLUDES:
+# unit-test - unit testing - XSLT production templates for InspectorXSLT
+# smoke-test - smoke testing - whether an XSLT is produced and compiles in a run
+# spec-test - specification conformance testing - functional runtime tests of the generated XSLT
+# refresh-testing - update InspectorXSLT and XSD for test metaschemas
+# xspec - run XSpec tests in designated folder
+# clean - clean up designated output folder
+
 module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 output_folder:=$(module_path)/test_output
-xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh)
+xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh)
+xspec_ci_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh)
+smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh)
+testing-refresh_script=$(realpath $(module_path)/testing/refresh-test-inspectors.sh)
+folder=.
 
 .PHONY: test
-test: unit-test smoke-test ## Run all tests
+test: smoke-test unit-test spec-test ## Run all tests
+
 
 .PHONY: spec-test
-spec-test: ## Run all specification-tests
-	LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_script) \
+spec-test: ## Run all specification tests
+	LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_ci_script) \
 		"folder=$(module_path)/testing/tests/inspector-functional-xspec" \
 		"report-to=$(output_folder)/inspector-functional-tests_report.html" \
 		"junit-to=$(output_folder)/inspector-functional-tests_junit-report.xml" \
+		"stop-on-error=yes" \
 		"recurse=yes"
 
-.PHONY: smoke-test
-smoke-test: ## Run all smoke-tests
-	LOGFILE="$(output_folder)/integration-tests.log" $(xspec_script) \
+.PHONY: unit-test
+unit-test: ## Run all unit tests
+	LOGFILE="$(output_folder)/inspector-generation-tests.log" $(xspec_ci_script) \
 		"folder=$(module_path)/testing/tests/inspector-generation-xspec" \
-		"report-to=$(output_folder)/integration-tests_report.html" \
-		"junit-to=$(output_folder)/integration-tests_junit-report.xml" \
+		"report-to=$(output_folder)/inspector-generation_report.html" \
+		"junit-to=$(output_folder)/inspector-generation_junit-report.xml" \
+		"stop-on-error=yes" \
+		"recurse=yes"
+
+
+.PHONY: smoke-test
+smoke-test: ## Run InspectorXSLT production smoke test
+	$(smoketest_script)
+
+
+.PHONY: refresh-testing
+refresh-testing: ## Update InspectorXSLT and XSD for test metaschemas (doesn't run tests)
+	$(testing-refresh_script)
+
+
+.PHONY: xspec
+xspec: ## Run all *.xspec in a designated folder, quietly - use folder=[folder]
+	LOGFILE="$(output_folder)/$(folder)-xspec-tests.log" $(xspec_script) \
+		"baseURI=file:$(module_path)/" \
+		"folder=$(folder)" \
+		"report-to=$(output_folder)/inspector-$(folder)-tests_report.html" \
+		"stop-on-error=no" \
 		"recurse=yes"
 
+
 .PHONY: clean
 clean: ## Remove test output
 	rm -fr $(output_folder)/*

src/schema-gen/InspectorXSLT/TESTING.md

@@ -14,6 +14,34 @@ Currently we focus on the *first two* of these, producing functional results in
 
 Find resources for testing the XSLT Inspector and its production in the [testing](testing) subdirectory.
 
+
+## Testing under `make`
+
+Documented in the Makefile -
+
+For example
+
+```
+> make test
+```
+
+Runs all tests using provided scripting.
+
+Don't commit unless this passes.
+
+```
+> make smoke-test
+```
+
+Runs only the 'smoke tests' (end to end production pipeline testing - are viable artifacts produced irrespective of their functionality or correctness)
+
+Available:
+ - `smoke-test` - builds an XSLT from the `computer_metaschema.xml` test example, and attempts to execute the resulting XSLT over stub input. A failure indicates a problem in the production pipeline - it is either broken or wrong
+ - `spec-test` - runs specification tests - does the produced XSLT behave as expected when used on the possible range of (XML) inputs? this is a validator: does it validate?
+ - `unit-test` - runs transformation template- and function-level unit tests regulating the mapping between source (Metaschema) and target (XSLT) expressions.
+
+Note this is work in progress and may change over time especially as we bring more tests in.
+
 ## Model metaschemas for testing
 
 ### `current` resource cache

src/schema-gen/InspectorXSLT/mx-reporter.xsl

@@ -26,9 +26,7 @@
                 <h1>Validation { @src ! replace(.,'.*/','') }</h1>
                 <p class="source"><a href="{ @src }">{ @src }</a></p>
                 <p>{ @elements } elements and { @attributes } attributes found in the document.</p>
-                <div>
-                    <xsl:apply-templates select="." mode="summary"/>
-                </div>
+                <xsl:apply-templates select="." mode="summary"/>
                 <xsl:apply-templates/>
             </body>
         </html>
@@ -40,6 +38,10 @@
         </div>
     </xsl:template>
 
+    <xsl:template match="mx:metaschema">
+       <h2>Metaschema: { . } (namespace '{ @namespace }')</h2>
+    </xsl:template>
+
     <xsl:template match="mx:validation[empty(descendant::mx:report)]" mode="summary">
         <div class="summary valid">
             <p>Good news - nothing to report - the instance is valid.</p>

src/schema-gen/InspectorXSLT/readme.md

@@ -19,6 +19,14 @@ The XSLT that is created this way can be used to test XML instances for errors i
 
 That is, it combines the effective functionality of XML schema and Schematron (XPath-based) validation.
 
+## Limitations
+
+Aiming at a complete and conformant implementation for XML of the [Metaschema](https://pages.nist.gov/metaschema) language version 1.0, there are a few known gaps, with a testing framework and protocols now being established both for filling those gaps and for demonstrating the tool's full functionality. Please use the processor with this in mind, that developers and sponsors need your feedback and contributions to enable us to complete the work.
+
+Currently we plan to support only XML-based formats as defined by Metaschema, not their semantically-equivalent JSON or YAML 'cousins'.
+
+Users of Metaschema-defined JSON can try reformatting their data as XML using automated means such as scripts produced by the  [Metaschema XSLT Converter Generators](../../converter-gen). Successful conversion will be valid on the other side. But failures will be indicated not typically be invalid results, and instead results missing those parts of the invalid that went unrecognized by the converter because they are invalid.
+
 ## Demo
 
 The [testing/current](testing/current) directory shows such an XSLT, which can be applied to an instance or set of instances (documents) to be tested against the rules defined by its metaschema - in this case the Computer Model metaschema example provided.

src/schema-gen/InspectorXSLT/testing/COMPUTER-INSPECTOR-PRODUCE.xpl

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" xmlns:c="http://www.w3.org/ns/xproc-step" version="1.0"
+   xmlns:metaschema="http://csrc.nist.gov/ns/metaschema/1.0" type="metaschema:COMPUTER-INSPECTOR-PRODUCE"
+   name="COMPUTER-INSPECTOR-PRODUCE">
+
+   <!-- Purpose: Builds an XSLT dynamically from the computer metaschema and tries running the XSLT on a workable instance -->
+   <!-- Input: `computer_metaschema.xml` (hard-wired) -->
+   <!-- Input: `computer-invalid/invalid1.xml` (hard-wired) -->
+
+   <!-- Output: on port 'inspector-xslt', see the InspectorXSLT  -->
+   <!-- Note: if no Inspector XSLT is wanted, bind output -oinspector-xslt=/dev/null   -->
+   <!-- Assumptions: Both inputs are current and viable:
+       - Metaschema: correct, complete and valid
+       - Instance: well-formed, not necessarily valid -->
+   <!-- &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& -->
+   <!-- Import (subpipeline) -->
+
+
+   <p:input port="parameters" kind="parameter"/>
+
+   <p:serialization port="INSPECTOR-XSLT" indent="true" method="xml" encoding="us-ascii" omit-xml-declaration="false"/>
+   <p:output port="INSPECTOR-XSLT">
+      <p:pipe port="OUT_INSPECTOR-XSLT" step="produce-inspector"/>
+   </p:output>
+
+   <p:import href="INSPECTOR-XSLT-TEST.xpl"/>
+
+   <metaschema:INSPECTOR-XSLT-TEST name="produce-inspector">
+      <p:input port="METASCHEMA">
+         <p:document href="computer_metaschema.xml"/>
+      </p:input>
+      <p:input port="instance">
+         <p:document href="computer-invalid/invalid1.xml"/>
+      </p:input>
+      <p:input port="parameters"/>
+   </metaschema:INSPECTOR-XSLT-TEST>
+
+</p:declare-step>

src/schema-gen/InspectorXSLT/testing/CURRENT-TEST-MODELS-REFRESH.xpl

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" xmlns:c="http://www.w3.org/ns/xproc-step" version="1.0"
+   xmlns:metaschema="http://csrc.nist.gov/ns/metaschema/1.0" type="metaschema:CURRENT-TEST-MODELS-REFRESH"
+   name="CURRENT-TEST-MODELS-REFRESH">
+
+   <!-- Purpose: For a given metaschema instance, produces both a viable InspectorXSLT and a viable XSD -->
+   <!-- Use to refresh artifacts for testing metaschemas -->
+   <!-- Input: A valid, properly-linked metaschema -->
+
+   <!-- Output: For a metaschema input, produces both an XSD and an InspectorXSLT, together -->
+
+   <!-- &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& -->
+   <p:input port="parameters" kind="parameter"/>
+
+<!-- For some reason this errors when run on a sequence from Maven (but not in oXygen's older XML Calabash)  -->
+
+   <p:option name="targetDir" select="'current'"/>
+
+   <p:input port="METASCHEMA" primary="true" sequence="true">
+      <p:document href="tiny_metaschema.xml"/>
+      <p:document href="computer_metaschema.xml"/>
+   </p:input>
+
+
+   <!-- &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& -->
+   <!-- Import (subpipelines) -->
+
+   <p:import href="../../METASCHEMA-XSD.xpl"/>
+
+   <p:import href="INSPECTOR-XSLT-TEST.xpl"/>
+
+
+   <!-- &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& &__& -->
+   <!-- It beginneth -->
+
+   <p:for-each name="iterate">
+      <p:iteration-source>
+         <p:pipe port="METASCHEMA" step="CURRENT-TEST-MODELS-REFRESH"/>
+      </p:iteration-source>
+
+      <p:variable name="baseURI" select="base-uri(.)"/>
+      <p:variable name="filepath"
+         select="(replace($baseURI,'[^/]+$',''), $targetDir, replace($baseURI,'(.*/|\.xml$)','') ) => string-join('/')"/>
+
+      <p:identity name="this-metaschema"/>
+
+      <metaschema:METASCHEMA-XSD name="produce-xsd"/>
+
+      <p:store name="save-xsd" indent="true" encoding="ASCII">
+         <p:with-option name="href" select="$filepath || '-schema.xsd'"/>
+         <p:input port="source">
+            <p:pipe port="XSD" step="produce-xsd"/>
+         </p:input>
+      </p:store>
+
+      <metaschema:INSPECTOR-XSLT-TEST name="produce-inspector">
+         <p:input port="METASCHEMA">
+            <p:pipe port="result" step="this-metaschema"/>
+         </p:input>
+         <p:input port="instance">
+            <p:inline>
+               <X/>
+            </p:inline>
+         </p:input>
+         <p:input port="parameters"/>
+      </metaschema:INSPECTOR-XSLT-TEST>
+
+      <p:store name="save-inspector" indent="true" encoding="ASCII">
+         <p:with-option name="href" select="$filepath || '-inspector.xsl'"/>
+         <p:input port="source">
+            <p:pipe port="OUT_INSPECTOR-XSLT" step="produce-inspector"/>
+         </p:input>
+      </p:store>
+
+   </p:for-each>
+
+</p:declare-step>