Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed JSON Validation #5

63 changes: 41 additions & 22 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ references:
common_environment: &common_environment
environment:
SAXON_VERSION: 9.9.0-1
JSON_CLI_VERSION: 0.0.1-SNAPSHOT
CICD_DIR: git-oscal/build/ci-cd
OSCAL_REPO_DIR: git-oscal
OSCAL_BUILD_DIR: build_artifacts
OSCAL_TOOLS_DIR: oscal_tools
SCHEMATRON_HOME: git-schematron
TERM: xterm
working_directory: ~/oscal
Expand All @@ -28,19 +30,6 @@ references:
paths:
- ~/jekyll-bundle
commands:
install-saxon:
description: "Install Saxon-HE"
steps:
- restore_cache:
key: maven-m2
- run:
name: Get Saxon
command: |
mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
- save_cache:
key: maven-m2
paths:
- ~/.m2
install-schematron:
description: "Install the schematron skeleton"
steps:
Expand All @@ -50,6 +39,37 @@ commands:
git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "$SCHEMATRON_HOME"
cd "$SCHEMATRON_HOME"
git checkout master -- trunk/schematron/code
install-maven-dependencies:
description: "Install Maven Dependencies"
steps:
- run:
name: Get OSCAL Tools
command: |
git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}"
- run:
name: Generate cache checksum
command: |
# Ensure we cache the necessary dependencies
cat "${OSCAL_TOOLS_DIR}/json-cli/pom.xml" > .maven-cache-checksum
echo "$SAXON_VERSION" >> .maven-cache-checksum
- restore_cache:
keys:
- mvn-m2-cache-V2-{{ checksum ".maven-cache-checksum" }}
- mvn-m2-cache-V2
- run:
name: Get Saxon-HE
command: |
mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
- run:
name: Build JSON Validation CLI
command: |
cd "${OSCAL_TOOLS_DIR}/json-cli"
mvn dependency:go-offline
mvn install
- save_cache:
paths:
- ~/.m2
key: mvn-m2-cache-V2-{{ checksum ".maven-cache-checksum" }}
install-maven:
description: "Installs Maven"
steps:
Expand Down Expand Up @@ -148,6 +168,7 @@ jobs: # a collection of
steps:
- checkout
- run:
name: Check if skip build
command: |
mkdir -p "$OSCAL_BUILD_DIR/state"
COMMIT_MSG=$(git log --format=oneline -n 1 $CIRCLE_SHA1)
Expand All @@ -169,7 +190,7 @@ jobs: # a collection of
- *attach_build_workspace
- check-skip-build
- install-xmllint
- install-saxon
- install-maven-dependencies
- install-schematron
- run:
name: Validate metaschema instances
Expand All @@ -186,8 +207,7 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-ajv
- install-maven-dependencies
- run:
name: Generate OSCAL schemas
command: |
Expand All @@ -204,7 +224,7 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-maven-dependencies
- run:
name: Generate OSCAL schemas
command: |
Expand All @@ -221,10 +241,9 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-maven-dependencies
- install-jq
- install-prettyjson
- install-ajv
- run:
name: Generate OSCAL converters
command: |
Expand Down Expand Up @@ -252,7 +271,7 @@ jobs: # a collection of
- *attach_build_workspace
- check-skip-build
- install-xmllint
- install-ajv
- install-maven-dependencies
- run:
name: Validate content instances
command: |
Expand All @@ -264,7 +283,7 @@ jobs: # a collection of
- *attach_build_workspace
- check-skip-build
- install-maven
- install-saxon
- install-maven-dependencies
- install-lxml
- install-xmllint
- run:
Expand All @@ -278,7 +297,7 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-maven-dependencies
- run:
name: Generate schema documentation
command: |
Expand Down
3 changes: 2 additions & 1 deletion build/ci-cd/copy-and-convert-content.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ if [[ -z "$OSCALDIR" ]]; then
fi

source $OSCALDIR/build/ci-cd/saxon-init.sh
source $OSCALDIR/build/ci-cd/init-validate-json.sh

if [ -z "$1" ]; then
working_dir="$OSCALDIR"
Expand Down Expand Up @@ -63,7 +64,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do

# validate generated file
schema="$working_dir/json/schema/oscal_${model}_schema.json"
ajv validate -s "$schema" -d "$dest" --extend-refs=true --verbose
validate_json "$schema" "$dest"

# produce pretty JSON
dest_pretty="$working_dir/${newpath}.${altformat}"
Expand Down
4 changes: 2 additions & 2 deletions build/ci-cd/generate-schema.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ if [[ -z "$OSCALDIR" ]]; then
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "$DIR/common-environment.sh"
fi

source $OSCALDIR/build/ci-cd/saxon-init.sh
source $OSCALDIR/build/ci-cd/init-validate-json.sh

if [ -z "$1" ]; then
working_dir="$OSCALDIR"
Expand Down Expand Up @@ -75,7 +75,7 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
# xml)
# TODO: Add support for XML schema validation
json)
ajv compile -s "$schema" --extend-refs=true --verbose
validate_json "$OSCALDIR/build/ci-cd/json-schema-schema.json" "$schema"
cmd_exitcode=$?
;;
*)
Expand Down
45 changes: 45 additions & 0 deletions build/ci-cd/init-validate-json.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/bin/bash

if [[ -z "$OSCALDIR" ]]; then
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "$DIR/common-environment.sh"
fi

if [[ -z "$JSON_CLI_HOME" ]]; then
if [[ -z "$JSON_CLI_VERSION" ]]; then
echo "${P_ERROR}JSON_CLI_VERSION is not set or is empty.${P_END} ${P_INFO}Please set JSON_CLI_VERSION to indicate the library version${P_END}"
fi
JSON_CLI_HOME=~/.m2/repository/gov/nist/oscal/json-cli/${JSON_CLI_VERSION}
fi

# ( set -o posix ; set )

validate_json() {
local json_schema="$1"; shift
local json_file="$1"; shift
local extra_params=($@)

local classpath=$(JARS=("$JSON_CLI_HOME"/*.jar); IFS=:; echo "${JARS[*]}")

set --

if [ -z "$json_schema" ]; then
echo "${P_ERROR}The JSON schema must be provided as the first argument.${P_END}"
else
set -- "$@" "-s" "${json_schema}"
fi

if [ -z "$json_file" ]; then
echo "${P_ERROR}The JSON file must be provided as the second argument.${P_END}"
else
set -- "$@" "-v" "${json_file}"
fi

java -cp "$classpath" gov.nist.oscal.json.JsonCLI "$@" "${extra_params[@]}"

if [ "$?" -ne 0 ]; then
echo "${P_ERROR}Error running JsonCLI.${P_END}"
return 3
fi
return 0
}
168 changes: 168 additions & 0 deletions build/ci-cd/json-schema-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "http://json-schema.org/draft-07/schema#",
"title": "Core schema meta-schema",
"definitions": {
"schemaArray": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#" }
},
"nonNegativeInteger": {
"type": "integer",
"minimum": 0
},
"nonNegativeIntegerDefault0": {
"allOf": [
{ "$ref": "#/definitions/nonNegativeInteger" },
{ "default": 0 }
]
},
"simpleTypes": {
"enum": [
"array",
"boolean",
"integer",
"null",
"number",
"object",
"string"
]
},
"stringArray": {
"type": "array",
"items": { "type": "string" },
"uniqueItems": true,
"default": []
}
},
"type": ["object", "boolean"],
"properties": {
"$id": {
"type": "string",
"format": "uri-reference"
},
"$schema": {
"type": "string",
"format": "uri"
},
"$ref": {
"type": "string",
"format": "uri-reference"
},
"$comment": {
"type": "string"
},
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"default": true,
"readOnly": {
"type": "boolean",
"default": false
},
"examples": {
"type": "array",
"items": true
},
"multipleOf": {
"type": "number",
"exclusiveMinimum": 0
},
"maximum": {
"type": "number"
},
"exclusiveMaximum": {
"type": "number"
},
"minimum": {
"type": "number"
},
"exclusiveMinimum": {
"type": "number"
},
"maxLength": { "$ref": "#/definitions/nonNegativeInteger" },
"minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
"pattern": {
"type": "string",
"format": "regex"
},
"additionalItems": { "$ref": "#" },
"items": {
"anyOf": [
{ "$ref": "#" },
{ "$ref": "#/definitions/schemaArray" }
],
"default": true
},
"maxItems": { "$ref": "#/definitions/nonNegativeInteger" },
"minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
"uniqueItems": {
"type": "boolean",
"default": false
},
"contains": { "$ref": "#" },
"maxProperties": { "$ref": "#/definitions/nonNegativeInteger" },
"minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
"required": { "$ref": "#/definitions/stringArray" },
"additionalProperties": { "$ref": "#" },
"definitions": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"properties": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"patternProperties": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"propertyNames": { "format": "regex" },
"default": {}
},
"dependencies": {
"type": "object",
"additionalProperties": {
"anyOf": [
{ "$ref": "#" },
{ "$ref": "#/definitions/stringArray" }
]
}
},
"propertyNames": { "$ref": "#" },
"const": true,
"enum": {
"type": "array",
"items": true,
"minItems": 1,
"uniqueItems": true
},
"type": {
"anyOf": [
{ "$ref": "#/definitions/simpleTypes" },
{
"type": "array",
"items": { "$ref": "#/definitions/simpleTypes" },
"minItems": 1,
"uniqueItems": true
}
]
},
"format": { "type": "string" },
"contentMediaType": { "type": "string" },
"contentEncoding": { "type": "string" },
"if": { "$ref": "#" },
"then": { "$ref": "#" },
"else": { "$ref": "#" },
"allOf": { "$ref": "#/definitions/schemaArray" },
"anyOf": { "$ref": "#/definitions/schemaArray" },
"oneOf": { "$ref": "#/definitions/schemaArray" },
"not": { "$ref": "#" }
},
"default": true
}
Loading