Metaschema XSLT Inspector draft release in progress (#73) (#98)

* Metaschema XSLT Inspector (#73) Several months of work building InspectorXSLT using test-driven-development. Also includes considerable work on XSpec support including XProc and Saxon runtimes, scripts and CI/CD support. --------- Co-authored-by: A.J. Stein <[email protected]> * Update XSPEC-BATCH.xsl On-the-fly modification of JUnit XML results * Adjustments to XSpec reporting under CI (#99) * Fine adjustments in prep for 'nearly good enough' XSpec support under ci/cd (in feature branch) * Adjustments for legibility * Another touchup * Buffs in view of CI/CD and local/CL use cases for testing under make including new error-on-fail option * Added to script documentation * Still polishing help msg * Now with some more nicer XProc and scripting supporting ongoing maintenance / dev * adjustments to test runtime * Further refinements to testing & scripts * Rationalizing including file name regularization * Further configuration enhancements * More help in docs * Further testing/tuning/tweaking test runtimes and docs * Further alignment * New 'quiet' XSpec batching script, plus adjustment * New 'make' feature for testing InspectorXSLT * More Makefile edits / cleanup * Further docs and readme improvements * Removed obsolete Schematron implementation, no longer of interest * More polishing * Further small but useful improvements and cleanup * Added another utility script * Continuing to touch up * Add permissions for EnricoMi/publish-unit-test-result-action Using the guidance from the action's instructions: https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches * Add artifact event pull for EnricoMi/publish-unit-test-result-action Using the guidance from the action's instructions: https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches * Ensure XSpec test results comment always made on PR * [WIP] Dedupe event trigger and inline JUnit XML processing in CI workflow --------- Co-authored-by: A.J. Stein <[email protected]> --------- Co-authored-by: A.J. Stein <[email protected]>
usnistgov · Feb 16, 2024 · 0010340 · 0010340
1 parent 2515000
commit 0010340
Show file tree

Hide file tree

Showing 89 changed files with 11,521 additions and 880 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,39 @@
+# Rationale and instructions for this GitHub Actions workflow:
+# https://github.com/EnricoMi/publish-unit-test-result-action?tab=readme-ov-file#support-fork-repositories-and-dependabot-branches
+
+name: Publish Test Results
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+permissions: {}
+jobs:
+  test-results:
+    name: Test Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Download and Extract Artifacts
+        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          path: artifacts
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          action_fail: true
+          fail_on: "test failures"
+          event_file: artifacts/Test Results Event/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          check_name: XSpec Test Results
+          comment_mode: always
+          files: "**/*_junit-report.xml"
+          report_individual_runs: true
+          deduplicate_classes_by_file_name: false
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,55 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request: {}
+permissions:
+  checks: write
+  pull-requests: write  
+env:
+  JAVA_VERSION: "17"
+  JAVA_DISTRIBUTION: "temurin"
+jobs:
+  test:
+    name: "Test and Collect Results"
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "${{ env.JAVA_DISTRIBUTION }}"
+          java-version: "${{ env.JAVA_VERSION }}"
+      - name: Run unit tests
+        run: |
+          make -C src unit-test
+        id: unit-tests
+      - name: Run integration tests
+        run: |
+            make -C src smoke-test
+        id: integration-tests
+      - name: Run specification tests
+        run: |
+          make -C src spec-test
+        id: spec-tests
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        with:
+          name: Test Results
+          path: |
+            **/*_junit-report.xml
+  event_file:
+    name: "Upload Results to Event File"
+    runs-on: ubuntu-20.04
+    needs: test
+    steps:
+    - name: Upload
+      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+      with:
+        name: Test Results Event
+        path: ${{ github.event_path }}
diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,9 @@
 *.xpr
 
 # other generated files
-xspec/*-result.html
+**/*-xspec/*-result.html
+**/xspec/*-result.html
+**/*xspec*-result.html
 
 # test outputs
 src/**/test_output/
diff --git a/README.md b/README.md
@@ -175,6 +175,10 @@ See the [Metaschema Repository](https://github.com/usnistgov/metaschema) and its
 
 See the Project [Wiki](https://github.com/usnistgov/metaschema-xslt/wiki) for documentation maintained on this site.
 
+## Acknowledgements
+
+This work and especially work on testing XSLT represented by this project would have been impossible without examples and leadership provided by persons including AG; AJS; DW; NW.
+
 ## Required outline
 
 This page includes all the following, as described by guidelines at https://raw.githubusercontent.com/usnistgov/opensource-repo/main/README.md

diff --git a/TESTING.md b/TESTING.md
@@ -16,47 +16,65 @@ If code legibility and consistency become an issue, we can consider more stringe
 
 ## *Test Everything*
 
-TBD - a TEST EVERYTHING subroutine.
+With `make`, `bash`, and Maven installed, `make test -C src` runs all the tests (tested under Ubuntu) in the `src` directory relative to the current working directory.
 
-## Testing technologies
+The top-level Makefile in this directory collects commands from Makefiles distributed throughout the repo.
+
+You can also use `make` in isolation from the top-level testing, to focus on your application. As a developer you only need to worry about the folder containing your application, binding test tasks to the targets 'smoke-test', 'spec-test' and 'unit-test' as need be.
+
+Model such a Makefile, which calls XSpec for testing XSLT, after the example `src/schema-gen/InspectorXSLT/Makefile`.
+
+Experiment using different Makefile targets as configured in the various directories.
+
+For any directory, `make` with no arguments should offer tips.
+
+### Testing technologies
 
 [XSpec](https://github.com/xspec/xspec/) is the preferred testing harness for XSLT in this initiative. XSpec test suites can be either self-contained, or can reference testing artifacts maintained externally. The repository contains XSpec examples demonstrating a range of usage patterns that can be applied.
 
 Script-driven testing should rely on the same dependencies as the runtimes they test, as documented.
 
-## Global functional testing
-
-`src/testing` includes resources for global-level testing. This folder or its contents should not to be moved or edited without fully testing *all* test runtimes, as resources inside this directory are sometimes dependencies.
+### Extensions to XSpec
 
-Do not commit anything to this folder that you do not wish to stay there indefinitely; instead, copy into a sibling (temporary) directory that can be deleted freely.
+Currently we are emulating and re-engineering some specific XSpec capabilities in the [support/xspec-dev](support/xspec-dev) folder.
 
-## Application component-level (functional) unit testing
+These efforts are focused on producing and refining XSpec runtimes for various use cases and scenarios with specialized requirements faced by this project, such as arbitrary batching and iXML support. Tools we develop here are released under the same terms as Metaschema-XSLT (as open-source software).
 
-`src/**/testing` includes (functional) testing for utilities supported in a given folder.
 
-When developing applications, feel free to add and modify any `testing` folder or its contents within the scope of work.
+## Test-driven development
 
-Unit tests are expected to run successfully when committed - both completing, and passing all applicable tests. Keep in mind that most testing frameworks support marking tests as not applicable (in XSpec, [flag a scenario or `expect` as `pending`](https://github.com/xspec/xspec/wiki/Focusing-Your-Efforts#marking-scenario-or-expectation-as-pending)), so it is possible to write tests ahead of an implementation and still pass.
+Almost all testing in this repository falls into the category of either XSLT transformations, or runtimes that embed transformations.
 
-### Test-driven development
+### The approach
 
 While this project began as an experimental proof of concept, it now aims for higher levels of assurance and confidence than are necessary or appropriate for applications intended only to produce findings regarding feasibility and levels of effort. Accordingly, our development approach has shifted from rapid prototyping to a more explicit and traceable process of design, specification and implementation.
 
 If you touch a particular unit of code that doesn't have tests, write tests for it in the same PR as your change. If you touch a particular unit of code that has tests, update or augment them to test the change you are making. In general, push the tests ahead of the code, not the other way around, aligning the tests with [the Metaschema specification(s)](https://pages.nist.gov/metaschema/specification/) first.
 
 This expenditure of effort prevents bugs (easier than repairing them) and guards against regression, opening opportunities to do more interesting things. So it is not so much "extra" as an investment in future stability and sustainability.
 
-## Test applications
+The approach can require changing some habits. Looking for inspiration and "striking while the iron is hot" no longer works as well (since the forge must be warmed up first). Sometimes immediate gratification has to be set aside. Yet the payoffs are substantial, and come early.
 
-`examples` (tbd) includes top-level independent metaschema examples made for testing and demonstration.
+## Global functional testing
 
-This location is available for lightweight and <q>toy</q> applications, useful for evaluation, demonstration and learning. Fully built-out applications of Metaschema can also call this repository in as a submodule (like [OSCAL](https://github.com/usnistgov/oscal)).
+`src/testing` includes resources for global-level testing. This folder or its contents should not to be moved or edited without fully testing *all* test runtimes, as resources inside this directory are sometimes dependencies.
+
+Do not commit anything to this folder that you do not wish to stay there indefinitely; instead, copy into a sibling (temporary) directory that can be deleted freely.
+
+## Application component-level (functional) unit testing
+
+`src/**/testing` includes (functional) testing for utilities supported in a given folder.
+
+When developing applications, feel free to add and modify any `testing` folder or its contents within the scope of work.
+
+Unit tests are expected to run successfully when committed - both completing, and passing all applicable tests. Keep in mind that most testing frameworks support marking tests as not applicable (in XSpec, [flag a scenario or `expect` as `pending`](https://github.com/xspec/xspec/wiki/Focusing-Your-Efforts#marking-scenario-or-expectation-as-pending)), so it is possible to write tests ahead of an implementation and still pass.
 
 ## Testing under CI/CD
 
-Also tbd
+Github Actions is configured in the file [.github/workflows/test.yml](.github/workflows/test.yml)
+
+Note that since this logic enters the `Makefile` logic from the top, make executes the specified subroutines recursively.
+
+Accordingly, adding a test subroutine to a `spec-tests` Makefile configuration anywhere in the repository has the effect of enabling it (turning it on) for CI/CD as well.
 
-Links of interest: 
 
-- https://github.com/nkutsche/xspec-maven-plugin
-- https://github.com/galtm/xslt-accumulator-tools/blob/db1c6b2a/pom.xml#L68
diff --git a/src/Makefile b/src/Makefile
@@ -1,7 +1,10 @@
 include testing/make_common.mk
 
-# Each subdirectory that has a makefile
-dirs:=$(dir $(wildcard ./*/Makefile))
+# Each subdirectory (recursively) that has a makefile
+# Makefile wildcard function does not support that, so we use the shell
+# function with the find utility and look ever Makefile in a child dir
+# relative to this one, but exclude this one to use with the FOREACH macro.
+dirs:=$(shell find '.' ! -wholename ./Makefile -name 'Makefile' -printf "%h\n")
 
 .PHONY: test
 test: ## Run all tests

diff --git a/src/schema-gen/InspectorXSLT/Makefile b/src/schema-gen/InspectorXSLT/Makefile
@@ -0,0 +1,64 @@
+include ../../testing/make_common.mk
+
+# INCLUDES:
+# unit-test - unit testing - XSLT production templates for InspectorXSLT
+# smoke-test - smoke testing - whether an XSLT is produced and compiles in a run
+# spec-test - specification conformance testing - functional runtime tests of the generated XSLT
+# refresh-testing - update InspectorXSLT and XSD for test metaschemas
+# xspec - run XSpec tests in designated folder
+# clean - clean up designated output folder
+
+module_path:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+output_folder:=$(module_path)/test_output
+xspec_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch-quiet.sh)
+xspec_ci_script=$(realpath $(module_path)/../../../support/xspec-dev/mvn-saxon-xspec-batch.sh)
+smoketest_script=$(realpath $(module_path)/testing/smoketest-computer-inspector.sh)
+testing-refresh_script=$(realpath $(module_path)/testing/refresh-test-inspectors.sh)
+folder=.
+
+.PHONY: test
+test: smoke-test unit-test spec-test ## Run all tests
+
+
+.PHONY: spec-test
+spec-test: ## Run all specification tests
+	LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_ci_script) \
+		"folder=$(module_path)/testing/tests/inspector-functional-xspec" \
+		"report-to=$(output_folder)/inspector-functional-tests_report.html" \
+		"junit-to=$(output_folder)/inspector-functional-tests_junit-report.xml" \
+		"stop-on-error=yes" \
+		"recurse=yes"
+
+.PHONY: unit-test
+unit-test: ## Run all unit tests
+	LOGFILE="$(output_folder)/inspector-generation-tests.log" $(xspec_ci_script) \
+		"folder=$(module_path)/testing/tests/inspector-generation-xspec" \
+		"report-to=$(output_folder)/inspector-generation_report.html" \
+		"junit-to=$(output_folder)/inspector-generation_junit-report.xml" \
+		"stop-on-error=yes" \
+		"recurse=yes"
+
+
+.PHONY: smoke-test
+smoke-test: ## Run InspectorXSLT production smoke test
+	$(smoketest_script)
+
+
+.PHONY: refresh-testing
+refresh-testing: ## Update InspectorXSLT and XSD for test metaschemas (doesn't run tests)
+	$(testing-refresh_script)
+
+
+.PHONY: xspec
+xspec: ## Run all *.xspec in a designated folder, quietly - use folder=[folder]
+	LOGFILE="$(output_folder)/$(folder)-xspec-tests.log" $(xspec_script) \
+		"baseURI=file:$(module_path)/" \
+		"folder=$(folder)" \
+		"report-to=$(output_folder)/inspector-$(folder)-tests_report.html" \
+		"stop-on-error=no" \
+		"recurse=yes"
+
+
+.PHONY: clean
+clean: ## Remove test output
+	rm -fr $(output_folder)/*