Feature profile resolution unittesting a (usnistgov#1770)

* Profile resolution unit tests and specification updates (usnistgov#1219) * Updates to tests including whitespace scrub test * Adding oXygen project * Integration work on specs and unit tests / XSLT and Schematron * Profile resolution testing infrastructure, plus some tests * Some test files with expected results * Updated and improved XSpec generation and tests - now driven directly from SpecML source * Adjusted gitignore; extended spec with more bindings; illustrator XSLT * Added XSLT producing Markdown summary of requirements w/ examples; updates * Added a small utility for converting JSON "prop" fields to "props" fields (emended 1.0.2 syntax) * Updated to Profile Resolution spec (with example files now tagged) and readme.md * Updated profile resolution (spec) readme with clarification regarding unit testing * Adding XSLT provisionally to produce a unit testing harness after @david-waltermire-nist model, with example and schema; also cleanup * Removed working file included by accident * More unit tests with edits to spec for tracking. * Catching up various outstanding profile resolution work * Small update to oXygen setup * Update metaschema submodule to existing commit. * Move up Metaschema submodule Prior work on pushing up the XSpec dependency and pulling in other file relocation improvements were merged and it appears I pushed the wrong commit into this branch and inadvertently led to more confusion where builds failed in current version of usnistgov#1754. * XSpec test suites script & CI (usnistgov#1754) * XSpec test suites script & CI * Helper script to build and tag the OSCAL env * Move up Metaschema submodule Prior work on pushing up the XSpec dependency and pulling in other file relocation improvements were merged and it appears I pushed the wrong commit into this branch and inadvertently led to more confusion where builds failed in current version of usnistgov#1754. * Save XSpec test summary + pull appropriate tag * Escape TEST_SUITE in XSpec runner Co-authored-by: A.J. Stein <[email protected]> * Update test script to address shellcheck guidance * Grab branch from GHA * Fix for builds on the M1 --------- Co-authored-by: A.J. Stein <[email protected]> * Fixed logic for when a tag does not exist --------- Co-authored-by: Wendell Piez <[email protected]> Co-authored-by: A.J. Stein <[email protected]>
aj-stein-nist · Jun 29, 2023 · 77e2699 · 77e2699
1 parent 0905b90
commit 77e2699
Show file tree

Hide file tree

Showing 124 changed files with 11,985 additions and 239 deletions.
diff --git a/.github/workflows/xspec-tests.yml b/.github/workflows/xspec-tests.yml
@@ -0,0 +1,65 @@
+name: OSCAL XSpec Test Suite
+on:
+  push:
+    branches:
+      - main
+      - develop
+      - "feature-*"
+      - "release-*"
+    paths:
+      - /src
+      - "**.xsl"
+      - "**.xpl"
+      - "**.xspec"
+  pull_request:
+    branches:
+      - main
+      - develop
+      - "feature-*"
+      - "release-*"
+    paths:
+      - /src
+      - "**.xsl"
+      - "**.xpl"
+      - "**.xspec"
+  workflow_call: {}
+jobs:
+  xspec-tests:
+    name: Run XSpec tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Pull the correct image
+        shell: bash
+        # Produces a tagged image oscal-common-env:selected
+        run: ./build/pull-oscal-env-dockerfile.sh "${{ github.head_ref || github.ref_name }}"
+      - name: Run XSpec tests
+        shell: bash
+        run: |
+          set -o pipefail # propagate return code
+          docker run \
+            -v $(pwd):/oscal \
+            -e TEST_DIR=/oscal/xspec \
+            oscal-common-env:selected \
+            /oscal/src/utils/util/resolver-pipeline/testing/test.sh \
+            | tee summary.csv || {
+            if [ "$?" = 83 ]; then
+              # For now we only fail when tests fail to compile
+              echo "Some test suites failed to compile, failing..."
+              exit 1
+            fi
+          }
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: xspec-output
+          path: |
+            xspec/*.html
+            summary.csv
diff --git a/.gitignore b/.gitignore
@@ -55,3 +55,5 @@ node_modules/
 /docs/scratch-dir
 /.skipbuild
 /.runbuild
+
+/summary.csv
diff --git a/build/build-oscal-env-dockerfile.sh b/build/build-oscal-env-dockerfile.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Build and tag the oscal-common-env dockerfile
+#
+# By default the tag is the sanitized branch name, but can be overidden
+#     by an argument.
+
+set -Eeuo pipefail
+
+IMAGE="csd773/oscal-common-env"
+BRANCH=$(git branch --show-current)
+BRANCH_SANITIZED=${BRANCH/\//_}
+
+TAG="${1:-$BRANCH_SANITIZED}"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+docker build \
+    --label "branch=${BRANCH}" \
+    --label "commit_sha=$(git rev-parse HEAD)" \
+    --label "dirty=$(git diff --quiet && echo 'false' || echo 'true')" \
+    --label "[email protected]" \
+    --label "author=$(git config user.email)" \
+    --platform linux/amd64 \
+    -f "$SCRIPT_DIR/Dockerfile" \
+    -t "$IMAGE:$TAG" \
+    "$SCRIPT_DIR"
+
+echo "Built and tagged $IMAGE:$TAG, to push run:"
+echo "    docker push $IMAGE:$TAG"
diff --git a/build/pull-oscal-env-dockerfile.sh b/build/pull-oscal-env-dockerfile.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Pull the oscal-common-env from the registry using a defined set of tags, and
+#   retag the first working image with a new tag.
+# This is intended to be used in CI/CD environments where rebuilding the 
+#   oscal-common-env would be expensive, but an escape hatch to allow for a
+#   "special" image for a run is preferred.
+
+set -Eeuo pipefail
+
+# Allow the user to override the "branch" name (note that this still sanitizes the input)
+if [ "${1-}" ]; then
+    BRANCH=${1}
+else
+    BRANCH=$(git branch --show-current)
+fi
+
+# Docker tags cannot have "/" in them
+SANITIZED_BRANCH=${BRANCH/\//_}
+
+TAGS=("${SANITIZED_BRANCH}" "develop")
+
+SOURCE_IMAGE="csd773/oscal-common-env"
+
+# the output image and tag to write to
+OUTPUT_IMAGE="oscal-common-env"
+OUTPUT_TAG="selected"
+OUTPUT_REF="${OUTPUT_IMAGE}:${OUTPUT_TAG}"
+
+for TAG in "${TAGS[@]}"; do
+    REF="${SOURCE_IMAGE}:${TAG}"
+    docker pull "${REF}" && {
+        docker tag "${REF}" "${OUTPUT_REF}"
+        echo "Successsfully pulled ${REF} and retagged it as ${OUTPUT_REF}"
+        exit 0
+    } || echo "Pulling tag ${REF} failed..."
+done
+
+echo "Failed to pull any images in"
+exit 1
diff --git a/src/release/content-upgrade/json-prop-to-props.xsl b/src/release/content-upgrade/json-prop-to-props.xsl
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="3.0"
+    xpath-default-namespace="http://www.w3.org/2005/xpath-functions"
+    xmlns="http://www.w3.org/2005/xpath-functions">
+
+<!-- 
+    Command line invocation:
+    
+    $ java -jar $SAXON_HOME/saxon-he-10.2.jar -xsl:json-prop-to-props.xsl -it json-input=YOUR.json
+
+    where YOUR.json is your JSON file and the jar file is Saxon 10 or later.
+    
+    The process reads the file at this URL, parses it as JSON, maps it through a filter, and reports it back.
+    
+    -->
+
+    <xsl:output method="text"/>
+
+    <xsl:mode on-no-match="shallow-copy"/>
+
+    <!-- The input JSON file -->
+    <xsl:param name="json-input" select="'url to json file'"/>
+
+    <!-- The initial template that process the JSON -->
+    <xsl:template name="xsl:initial-template">
+        <xsl:variable name="mapped-result">
+            <xsl:apply-templates select="json-to-xml(unparsed-text($json-input))"/>
+        </xsl:variable>
+        <!--<xsl:copy-of select="$mapped-result"/>-->
+        <xsl:value-of select="xml-to-json($mapped-result)"/>
+    </xsl:template>
+
+    <xsl:template match="array[@key='prop']">
+        <array key="props">
+            <xsl:apply-templates/>
+        </array>
+    </xsl:template>
+
+</xsl:stylesheet>
diff --git a/src/specifications/profile-resolution/.gitignore b/src/specifications/profile-resolution/.gitignore
@@ -0,0 +1,2 @@
+!oscal-specs_share.xpr
+requirement-tests/output-actual/
diff --git a/src/specifications/profile-resolution/example-set.xspec b/src/specifications/profile-resolution/example-set.xspec
@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- autogenerated 2021-12-14T16:02:57.305-05:00 following model in example-set.xspec-->
+<x:description xmlns:x="http://www.jenitennison.com/xslt/xspec"
+               xmlns:opr="http://csrc.nist.gov/ns/oscal/profile-resolution"
+               xmlns:o="http://csrc.nist.gov/ns/oscal/1.0"
+               stylesheet="../../utils/util/resolver-pipeline/oscal-profile-RESOLVE.xsl"
+               run-as="external"
+               xmlns:ov="http://csrc.nist.gov/ns/oscal/test/variable">
+   <x:helper stylesheet="lib/oscal-profile-test-helper.xsl"/>
+   <!-- old tests are out of whack need update to current functionality /spec  -->
+   <!-- but the XSpec provides a functional model -->
+   <x:scenario label="Base test">
+      <!-- Alternative model for test scrubbing both the result and the expected
+           result for comparison -->
+      <x:context href="profile-resolution-examples/base-test_profile.xml"/>
+      <x:variable name="ov:expected-output"
+         href="profile-resolution-examples/output-expected/base-test_profile_RESOLVED.xml"/>
+      <x:expect label="Base test - fully resolved"
+         test="opr:scrub($x:result)" select="opr:scrub($ov:expected-output)"/>
+   </x:scenario>
+
+   <x:scenario label="Testing base-test_profile.xml">
+
+      <x:context href="profile-resolution-examples/base-test_profile.xml"/>
+      <x:expect label="Resolution of base-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/base-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing base2-test_profile.xml">
+      <x:context href="profile-resolution-examples/base2-test_profile.xml"/>
+      <x:expect label="Resolution of base2-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/base2-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing broken_profile.xml" pending="galtm">
+      <x:context href="profile-resolution-examples/broken_profile.xml"/>
+      <x:expect label="Resolution of broken_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/broken_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing circular_profile.xml">
+      <x:context href="profile-resolution-examples/circular_profile.xml"/>
+      <x:expect label="Resolution of circular_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/circular_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing exclude-call-test_profile.xml">
+      <x:context href="profile-resolution-examples/exclude-call-test_profile.xml"/>
+      <x:expect label="Resolution of exclude-call-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/exclude-call-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing full-test_profile.xml">
+      <x:context href="profile-resolution-examples/full-test_profile.xml"/>
+      <x:expect label="Resolution of full-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/full-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing home_profile.xml">
+      <x:context href="profile-resolution-examples/home_profile.xml"/>
+      <x:expect label="Resolution of home_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/home_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing import-twice_profile.xml">
+      <x:context href="profile-resolution-examples/import-twice_profile.xml"/>
+      <x:expect label="Resolution of import-twice_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/import-twice_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing include-all-no-children-test_profile.xml">
+      <x:context href="profile-resolution-examples/include-all-no-children-test_profile.xml"/>
+      <x:expect label="Resolution of include-all-no-children-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/include-all-no-children-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing include-all-test_profile.xml">
+      <x:context href="profile-resolution-examples/include-all-test_profile.xml"/>
+      <x:expect label="Resolution of include-all-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/include-all-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing include-call-with-children-test_profile.xml">
+      <x:context href="profile-resolution-examples/include-call-with-children-test_profile.xml"/>
+      <x:expect label="Resolution of include-call-with-children-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/include-call-with-children-test_profile_RESOLVED.xml"/>
+      <x:expect label="Includes grandparent in result" test="$x:result//o:control/@id = 'c3'"/>
+      <x:expect label="Includes grandchild in result"  test="$x:result//o:control/@id = 'c3.a-1'"/>
+      <x:expect label="Is missing a control not included"   test="not( $x:result//o:control/@id = 'c2' )"/>
+   </x:scenario>
+   <x:scenario label="Testing include-loose-param-test_profile.xml">
+      <x:context href="profile-resolution-examples/include-loose-param-test_profile.xml"/>
+      <x:expect label="Resolution of include-loose-param-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/include-loose-param-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing include-match-test_profile.xml">
+      <x:context href="profile-resolution-examples/include-match-test_profile.xml"/>
+      <x:expect label="Resolution of include-match-test_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/include-match-test_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing merge-implicit-keep_profile.xml">
+      <x:context href="profile-resolution-examples/merge-implicit-keep_profile.xml"/>
+      <x:expect label="Resolution of merge-implicit-keep_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/merge-implicit-keep_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing merge-keep-resources_profile.xml">
+      <x:context href="profile-resolution-examples/merge-keep-resources_profile.xml"/>
+      <x:expect label="Resolution of merge-keep-resources_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/merge-keep-resources_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing merge-keep_profile.xml">
+      <x:context href="profile-resolution-examples/merge-keep_profile.xml"/>
+      <x:expect label="Resolution of merge-keep_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/merge-keep_profile_RESOLVED.xml"/>
+   </x:scenario>
+   <x:scenario label="Testing modify-adds_profile.xml">
+      <x:context href="profile-resolution-examples/modify-adds_profile.xml"/>
+      <x:expect label="Resolution of modify-adds_profile.xml"
+                select="opr:scrub(.)"
+                href="profile-resolution-examples/output-expected/modify-adds_profile_RESOLVED.xml"/>
+   </x:scenario>
+</x:description>