diff --git a/.github/workflows/xspec-tests.yml b/.github/workflows/xspec-tests.yml new file mode 100644 index 0000000000..dc3cffd2c2 --- /dev/null +++ b/.github/workflows/xspec-tests.yml @@ -0,0 +1,65 @@ +name: OSCAL XSpec Test Suite +on: + push: + branches: + - main + - develop + - "feature-*" + - "release-*" + paths: + - /src + - "**.xsl" + - "**.xpl" + - "**.xspec" + pull_request: + branches: + - main + - develop + - "feature-*" + - "release-*" + paths: + - /src + - "**.xsl" + - "**.xpl" + - "**.xspec" + workflow_call: {} +jobs: + xspec-tests: + name: Run XSpec tests + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Pull the correct image + shell: bash + # Produces a tagged image oscal-common-env:selected + run: ./build/pull-oscal-env-dockerfile.sh "${{ github.head_ref || github.ref_name }}" + - name: Run XSpec tests + shell: bash + run: | + set -o pipefail # propagate return code + docker run \ + -v $(pwd):/oscal \ + -e TEST_DIR=/oscal/xspec \ + oscal-common-env:selected \ + /oscal/src/utils/util/resolver-pipeline/testing/test.sh \ + | tee summary.csv || { + if [ "$?" = 83 ]; then + # For now we only fail when tests fail to compile + echo "Some test suites failed to compile, failing..." + exit 1 + fi + } + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: xspec-output + path: | + xspec/*.html + summary.csv diff --git a/.gitignore b/.gitignore index d12a8f4df0..bc48dcc7c9 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ node_modules/ /docs/scratch-dir /.skipbuild /.runbuild + +/summary.csv diff --git a/build/build-oscal-env-dockerfile.sh b/build/build-oscal-env-dockerfile.sh new file mode 100755 index 0000000000..9b3b94034d --- /dev/null +++ b/build/build-oscal-env-dockerfile.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Build and tag the oscal-common-env dockerfile +# +# By default the tag is the sanitized branch name, but can be overidden +# by an argument. + +set -Eeuo pipefail + +IMAGE="csd773/oscal-common-env" +BRANCH=$(git branch --show-current) +BRANCH_SANITIZED=${BRANCH/\//_} + +TAG="${1:-$BRANCH_SANITIZED}" + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" + +docker build \ + --label "branch=${BRANCH}" \ + --label "commit_sha=$(git rev-parse HEAD)" \ + --label "dirty=$(git diff --quiet && echo 'false' || echo 'true')" \ + --label "maintainer=oscal@nist.gov" \ + --label "author=$(git config user.email)" \ + --platform linux/amd64 \ + -f "$SCRIPT_DIR/Dockerfile" \ + -t "$IMAGE:$TAG" \ + "$SCRIPT_DIR" + +echo "Built and tagged $IMAGE:$TAG, to push run:" +echo " docker push $IMAGE:$TAG" diff --git a/build/metaschema b/build/metaschema index a466a38793..70085548fe 160000 --- a/build/metaschema +++ b/build/metaschema @@ -1 +1 @@ -Subproject commit a466a38793a5d09b9e5cf3015c0afbced861b826 +Subproject commit 70085548fec75de999eebb480a0be7d7b1abdada diff --git a/build/pull-oscal-env-dockerfile.sh b/build/pull-oscal-env-dockerfile.sh new file mode 100755 index 0000000000..d110b12a8f --- /dev/null +++ b/build/pull-oscal-env-dockerfile.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Pull the oscal-common-env from the registry using a defined set of tags, and +# retag the first working image with a new tag. +# This is intended to be used in CI/CD environments where rebuilding the +# oscal-common-env would be expensive, but an escape hatch to allow for a +# "special" image for a run is preferred. + +set -Eeuo pipefail + +# Allow the user to override the "branch" name (note that this still sanitizes the input) +if [ "${1-}" ]; then + BRANCH=${1} +else + BRANCH=$(git branch --show-current) +fi + +# Docker tags cannot have "/" in them +SANITIZED_BRANCH=${BRANCH/\//_} + +TAGS=("${SANITIZED_BRANCH}" "develop") + +SOURCE_IMAGE="csd773/oscal-common-env" + +# the output image and tag to write to +OUTPUT_IMAGE="oscal-common-env" +OUTPUT_TAG="selected" +OUTPUT_REF="${OUTPUT_IMAGE}:${OUTPUT_TAG}" + +for REF in "${SOURCE_IMAGE}:${TAGS[@]}"; do + docker pull "${REF}" && { + docker tag "${REF}" "${OUTPUT_REF}" + echo "Successsfully pulled ${REF} and retagged it as ${OUTPUT_REF}" + exit 0 + } || echo "Pulling tag ${REF} failed..." +done + +echo "Failed to pull any images in" +exit 1 diff --git a/src/utils/util/resolver-pipeline/testing/test.sh b/src/utils/util/resolver-pipeline/testing/test.sh new file mode 100755 index 0000000000..c0f7e1e229 --- /dev/null +++ b/src/utils/util/resolver-pipeline/testing/test.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# OSCAL XSLT resolver pipeline test suite helper +# Runs all XSpec suites in this folder. +# Inputs: +# - The $XSPEC_COMMAND env var can override the XSpec command. +# If not set, xspec/bin must be in the $PATH. +# - The $TEST_DIR env var can override the XSpec output directory. +# Outputs: +# - All XSpec output is redirected to STDERR. +# - The final test status is pretty-printed to STDERR. +# - The suite path and status (passed, failed, or compile_failed) is printed to STDOUT as a CSV. +# ex: /oscal/src/utils/util/resolver-pipeline/testing/1_selected/select.xspec,passed +# - Return code: +# - 0 if all tests pass +# - 1 if one or more tests fail +# - 83 if one or more tests fail to compile (preferred over 1) + +set -Eeuo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" + +# Default to running xspec.sh (xspec/bin must be in PATH) +# with option to override using the XSPEC_COMMAND env var +XSPEC_COMMAND="${XSPEC_COMMAND:-xspec.sh}" +# The output directory to write to +# with option to override using the TEST_DIR env var +export TEST_DIR="${TEST_DIR:-${SCRIPT_DIR}/xspec}" + +# All .xspec files in the "testing" directory +TEST_SUITES=$(find "${SCRIPT_DIR}" -type f -name "*.xspec") + +# Setup an "alias" fd for use in subshells, +# which is used to capture the STDERR of the XSpec output +# (subshells capture STDOUT and STDERR, but not other descriptors) +exec 6>&2 + +# True if one or more test suites fail to run +SUITES_FAILURE=false +# True if one or more test suite fail to compile +SUITES_COMPILATION_FAILURE=false + +# CSV header +printf "xspec_suite_path,status\n" + +for TEST_SUITE in ${TEST_SUITES}; do + # Run XSpec on the test suite, while: + # 1) Redirecting STDOUT to STDERR (make XSpec less noisy) + # 2) Capturing the output to a variable "stderr_output" + # Then, if the suite failed, check "stderr_output" for compilation failures + # setting "SUITES_FAILURE" and "SUITES_COMPILATION_FAILURE" as appropriate + + printf "\n=== Testing Suite %s ===\n" "${TEST_SUITE}" 1>&2 + + suite_passed=true + stderr_output=$(${XSPEC_COMMAND} -e "${TEST_SUITE}" 2>&1 | tee /dev/fd/6) || suite_passed=false + + if [ "$suite_passed" = true ]; then + printf "%s,passed\n" "${TEST_SUITE}" + elif [[ $stderr_output == *"*** Error compiling the test suite"* ]]; then + printf "%s,compile_failed\n" "${TEST_SUITE}" + SUITES_COMPILATION_FAILURE=true + else + printf "%s,failed\n" "${TEST_SUITE}" + SUITES_FAILURE=true + fi +done + +if [ "$SUITES_COMPILATION_FAILURE" = true ]; then + printf "\nOne or more test suites failed to compile 🆘\n" 1>&2 + exit 83 # special status code to differentiate compilation failure +elif [ "$SUITES_FAILURE" = true ] ; then + printf "\nOne or more test suites failed to run ❌\n" 1>&2 + exit 1 +else + printf "\nAll test suites passed ✅\n" 1>&2 +fi