XSpec test suites script & CI

.github/workflows/xspec-tests.yml

@@ -0,0 +1,65 @@
+name: OSCAL XSpec Test Suite
+on:
+  push:
+    branches:
+      - main
+      - develop
+      - "feature-*"
+      - "release-*"
+    paths:
+      - /src
+      - "**.xsl"
+      - "**.xpl"
+      - "**.xspec"
+  pull_request:
+    branches:
+      - main
+      - develop
+      - "feature-*"
+      - "release-*"
+    paths:
+      - /src
+      - "**.xsl"
+      - "**.xpl"
+      - "**.xspec"
+  workflow_call: {}
+jobs:
+  xspec-tests:
+    name: Run XSpec tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Pull the correct image
+        shell: bash
+        # Produces a tagged image oscal-common-env:selected
+        run: ./build/pull-oscal-env-dockerfile.sh
+      - name: Run XSpec tests
+        shell: bash
+        run: |
+          set -o pipefail # propagate return code
+          docker run \
+            -v $(pwd):/oscal \
+            -e TEST_DIR=/oscal/xspec \
+            oscal-common-env:selected \
+            /oscal/src/utils/util/resolver-pipeline/testing/test.sh \
+            | tee summary.csv || {
+            if [ "$?" = 83 ]; then
+              # For now we only fail when tests fail to compile
+              echo "Some test suites failed to compile, failing..."
+              exit 1
+            fi
+          }
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: xspec-output
+          path: |
+            xspec/*.html
+            summary.csv

.gitignore

@@ -55,3 +55,5 @@ node_modules/
 /docs/scratch-dir
 /.skipbuild
 /.runbuild
+
+/summary.csv

build/build-oscal-env-dockerfile.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Build and tag the oscal-common-env dockerfile
+#
+# By default the tag is the sanitized branch name, but can be overidden
+#     by an argument.
+
+set -Eeuo pipefail
+
+IMAGE="csd773/oscal-common-env"
+BRANCH=$(git branch --show-current)
+BRANCH_SANITIZED=${BRANCH/\//_}
+
+TAG="${1:-$BRANCH_SANITIZED}"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+docker build \
+    -f "$SCRIPT_DIR/Dockerfile" \
+    -t "$IMAGE:$TAG" \
+    "$SCRIPT_DIR"
+
+echo "Built and tagged $IMAGE:$TAG, to push run:"
+echo "    docker push $IMAGE:$TAG"

build/pull-oscal-env-dockerfile.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Pull the oscal-common-env from the registry using a defined set of tags, and
+#   retag the first working image with a new tag.
+# This is intended to be used in CI/CD environments where rebuilding the 
+#   oscal-common-env would be expensive, but an escape hatch to allow for a
+#   "special" image for a run is preferred.
+
+set -Eeuo pipefail
+
+if [ "${GITHUB_REF-}" ]; then
+    BRANCH=${GITHUB_REF#refs/heads/}
+else
+    BRANCH=$(git branch --show-current)
+fi
+
+# Docker tags cannot have "/" in them
+SANITIZED_BRANCH=${BRANCH/\//_}
+
+TAGS=("${SANITIZED_BRANCH}" "develop")
+
+SOURCE_IMAGE="csd773/oscal-common-env"
+
+# the output image and tag to write to
+OUTPUT_IMAGE="oscal-common-env"
+OUTPUT_TAG="selected"
+OUTPUT_REF="${OUTPUT_IMAGE}:${OUTPUT_TAG}"
+
+for REF in "${SOURCE_IMAGE}:${TAGS[@]}"; do
+    docker pull "${REF}" && {
+        docker tag "${REF}" "${OUTPUT_REF}"
+        echo "Successsfully pulled ${REF} and retagged it as ${OUTPUT_REF}"
+        exit 0
+    } || echo "Pulling tag ${REF} failed..."
+done
+
+echo "Failed to pull any images in"
+exit 1

src/utils/util/resolver-pipeline/testing/test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# OSCAL XSLT resolver pipeline test suite helper
+# Runs all XSpec suites in this folder.
+# Inputs:
+#   - The $XSPEC_COMMAND env var can override the XSpec command.
+#     If not set, xspec/bin must be in the $PATH.
+#   - The $TEST_DIR env var can override the XSpec output directory.
+# Outputs:
+#   - All XSpec output is redirected to STDERR.
+#   - The final test status is pretty-printed to STDERR.
+#   - The suite path and status (passed, failed, or compile_failed) is printed to STDOUT as a CSV.
+#     ex: /oscal/src/utils/util/resolver-pipeline/testing/1_selected/select.xspec,passed
+#   - Return code:
+#     - 0 if all tests pass
+#     - 1 if one or more tests fail
+#     - 83 if one or more tests fail to compile (preferred over 1)
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+# Default to running xspec.sh (xspec/bin must be in PATH)
+#   with option to override using the XSPEC_COMMAND env var
+XSPEC_COMMAND="${XSPEC_COMMAND:-xspec.sh}"
+# The output directory to write to
+#   with option to override using the TEST_DIR env var
+export TEST_DIR="${TEST_DIR:-${SCRIPT_DIR}/xspec}"
+
+# All .xspec files in the "testing" directory
+TEST_SUITES="$(find ${SCRIPT_DIR} -type f -name "*.xspec")"
+
+# Setup an "alias" fd for use in subshells,
+#    which is used to capture the STDERR of the XSpec output
+# (subshells capture STDOUT and STDERR, but not other descriptors)
+exec 6>&2
+
+# True if one or more test suites fail to run
+SUITES_FAILURE=false
+# True if one or more test suite fail to compile
+SUITES_COMPILATION_FAILURE=false
+
+# CSV header
+printf "xspec_suite_path,status\n"
+
+for TEST_SUITE in ${TEST_SUITES}; do
+    # Run XSpec on the test suite, while:
+    #   1) Redirecting STDOUT to STDERR (make XSpec less noisy)
+    #   2) Capturing the output to a variable "stderr_output"
+    # Then, if the suite failed, check "stderr_output" for compilation failures
+    #   setting "SUITES_FAILURE" and "SUITES_COMPILATION_FAILURE" as appropriate
+
+    printf "\n=== Testing Suite ${TEST_SUITE} ===\n" 1>&2
+
+    suite_passed=true
+    stderr_output=$(${XSPEC_COMMAND} -e $TEST_SUITE 2>&1 | tee /dev/fd/6) || suite_passed=false
+
+    if [ "$suite_passed" = true ]; then
+        printf "${TEST_SUITE},passed\n"
+    elif [[ $stderr_output == *"*** Error compiling the test suite"* ]]; then
+        printf "${TEST_SUITE},compile_failed\n"
+        SUITES_COMPILATION_FAILURE=true
+    else
+        printf "${TEST_SUITE},failed\n"
+        SUITES_FAILURE=true
+    fi
+done
+
+if [ "$SUITES_COMPILATION_FAILURE" = true ]; then
+    printf "\nOne or more test suites failed to compile 🆘\n" 1>&2
+    exit 83 # special status code to differentiate compilation failure
+elif [ "$SUITES_FAILURE" = true ] ; then
+    printf "\nOne or more test suites failed to run ❌\n" 1>&2
+    exit 1
+else
+    printf "\nAll test suites passed ✅\n" 1>&2
+fi