Skip to content

Commit

Permalink
Merge pull request #5 from david-waltermire-nist/wendell-json-validation
Browse files Browse the repository at this point in the history
Fixed JSON Validation
  • Loading branch information
wendellpiez authored Jun 15, 2019
2 parents 4851e64 + 8dd2f1d commit 361634f
Show file tree
Hide file tree
Showing 18 changed files with 272 additions and 38 deletions.
63 changes: 41 additions & 22 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ references:
common_environment: &common_environment
environment:
SAXON_VERSION: 9.9.0-1
JSON_CLI_VERSION: 0.0.1-SNAPSHOT
CICD_DIR: git-oscal/build/ci-cd
OSCAL_REPO_DIR: git-oscal
OSCAL_BUILD_DIR: build_artifacts
OSCAL_TOOLS_DIR: oscal_tools
SCHEMATRON_HOME: git-schematron
TERM: xterm
working_directory: ~/oscal
Expand All @@ -28,19 +30,6 @@ references:
paths:
- ~/jekyll-bundle
commands:
install-saxon:
description: "Install Saxon-HE"
steps:
- restore_cache:
key: maven-m2
- run:
name: Get Saxon
command: |
mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
- save_cache:
key: maven-m2
paths:
- ~/.m2
install-schematron:
description: "Install the schematron skeleton"
steps:
Expand All @@ -50,6 +39,37 @@ commands:
git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "$SCHEMATRON_HOME"
cd "$SCHEMATRON_HOME"
git checkout master -- trunk/schematron/code
install-maven-dependencies:
description: "Install Maven Dependencies"
steps:
- run:
name: Get OSCAL Tools
command: |
git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}"
- run:
name: Generate cache checksum
command: |
# Ensure we cache the necessary dependencies
cat "${OSCAL_TOOLS_DIR}/json-cli/pom.xml" > .maven-cache-checksum
echo "$SAXON_VERSION" >> .maven-cache-checksum
- restore_cache:
keys:
- mvn-m2-cache-V2-{{ checksum ".maven-cache-checksum" }}
- mvn-m2-cache-V2
- run:
name: Get Saxon-HE
command: |
mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
- run:
name: Build JSON Validation CLI
command: |
cd "${OSCAL_TOOLS_DIR}/json-cli"
mvn dependency:go-offline
mvn install
- save_cache:
paths:
- ~/.m2
key: mvn-m2-cache-V2-{{ checksum ".maven-cache-checksum" }}
install-maven:
description: "Installs Maven"
steps:
Expand Down Expand Up @@ -148,6 +168,7 @@ jobs: # a collection of
steps:
- checkout
- run:
name: Check if skip build
command: |
mkdir -p "$OSCAL_BUILD_DIR/state"
COMMIT_MSG=$(git log --format=oneline -n 1 $CIRCLE_SHA1)
Expand All @@ -169,7 +190,7 @@ jobs: # a collection of
- *attach_build_workspace
- check-skip-build
- install-xmllint
- install-saxon
- install-maven-dependencies
- install-schematron
- run:
name: Validate metaschema instances
Expand All @@ -186,8 +207,7 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-ajv
- install-maven-dependencies
- run:
name: Generate OSCAL schemas
command: |
Expand All @@ -204,7 +224,7 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-maven-dependencies
- run:
name: Generate OSCAL schemas
command: |
Expand All @@ -221,10 +241,9 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-maven-dependencies
- install-jq
- install-prettyjson
- install-ajv
- run:
name: Generate OSCAL converters
command: |
Expand Down Expand Up @@ -252,7 +271,7 @@ jobs: # a collection of
- *attach_build_workspace
- check-skip-build
- install-xmllint
- install-ajv
- install-maven-dependencies
- run:
name: Validate content instances
command: |
Expand All @@ -264,7 +283,7 @@ jobs: # a collection of
- *attach_build_workspace
- check-skip-build
- install-maven
- install-saxon
- install-maven-dependencies
- install-lxml
- install-xmllint
- run:
Expand All @@ -278,7 +297,7 @@ jobs: # a collection of
- *checkout
- *attach_build_workspace
- check-skip-build
- install-saxon
- install-maven-dependencies
- run:
name: Generate schema documentation
command: |
Expand Down
3 changes: 2 additions & 1 deletion build/ci-cd/copy-and-convert-content.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ if [[ -z "$OSCALDIR" ]]; then
fi

source $OSCALDIR/build/ci-cd/saxon-init.sh
source $OSCALDIR/build/ci-cd/init-validate-json.sh

if [ -z "$1" ]; then
working_dir="$OSCALDIR"
Expand Down Expand Up @@ -63,7 +64,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do

# validate generated file
schema="$working_dir/json/schema/oscal_${model}_schema.json"
ajv validate -s "$schema" -d "$dest" --extend-refs=true --verbose
validate_json "$schema" "$dest"

# produce pretty JSON
dest_pretty="$working_dir/${newpath}.${altformat}"
Expand Down
4 changes: 2 additions & 2 deletions build/ci-cd/generate-schema.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ if [[ -z "$OSCALDIR" ]]; then
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "$DIR/common-environment.sh"
fi

source $OSCALDIR/build/ci-cd/saxon-init.sh
source $OSCALDIR/build/ci-cd/init-validate-json.sh

if [ -z "$1" ]; then
working_dir="$OSCALDIR"
Expand Down Expand Up @@ -75,7 +75,7 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d
# xml)
# TODO: Add support for XML schema validation
json)
ajv compile -s "$schema" --extend-refs=true --verbose
validate_json "$OSCALDIR/build/ci-cd/json-schema-schema.json" "$schema"
cmd_exitcode=$?
;;
*)
Expand Down
45 changes: 45 additions & 0 deletions build/ci-cd/init-validate-json.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/bin/bash

if [[ -z "$OSCALDIR" ]]; then
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "$DIR/common-environment.sh"
fi

if [[ -z "$JSON_CLI_HOME" ]]; then
if [[ -z "$JSON_CLI_VERSION" ]]; then
echo "${P_ERROR}JSON_CLI_VERSION is not set or is empty.${P_END} ${P_INFO}Please set JSON_CLI_VERSION to indicate the library version${P_END}"
fi
JSON_CLI_HOME=~/.m2/repository/gov/nist/oscal/json-cli/${JSON_CLI_VERSION}
fi

# ( set -o posix ; set )

validate_json() {
local json_schema="$1"; shift
local json_file="$1"; shift
local extra_params=($@)

local classpath=$(JARS=("$JSON_CLI_HOME"/*.jar); IFS=:; echo "${JARS[*]}")

set --

if [ -z "$json_schema" ]; then
echo "${P_ERROR}The JSON schema must be provided as the first argument.${P_END}"
else
set -- "$@" "-s" "${json_schema}"
fi

if [ -z "$json_file" ]; then
echo "${P_ERROR}The JSON file must be provided as the second argument.${P_END}"
else
set -- "$@" "-v" "${json_file}"
fi

java -cp "$classpath" gov.nist.oscal.json.JsonCLI "$@" "${extra_params[@]}"

if [ "$?" -ne 0 ]; then
echo "${P_ERROR}Error running JsonCLI.${P_END}"
return 3
fi
return 0
}
168 changes: 168 additions & 0 deletions build/ci-cd/json-schema-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "http://json-schema.org/draft-07/schema#",
"title": "Core schema meta-schema",
"definitions": {
"schemaArray": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#" }
},
"nonNegativeInteger": {
"type": "integer",
"minimum": 0
},
"nonNegativeIntegerDefault0": {
"allOf": [
{ "$ref": "#/definitions/nonNegativeInteger" },
{ "default": 0 }
]
},
"simpleTypes": {
"enum": [
"array",
"boolean",
"integer",
"null",
"number",
"object",
"string"
]
},
"stringArray": {
"type": "array",
"items": { "type": "string" },
"uniqueItems": true,
"default": []
}
},
"type": ["object", "boolean"],
"properties": {
"$id": {
"type": "string",
"format": "uri-reference"
},
"$schema": {
"type": "string",
"format": "uri"
},
"$ref": {
"type": "string",
"format": "uri-reference"
},
"$comment": {
"type": "string"
},
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"default": true,
"readOnly": {
"type": "boolean",
"default": false
},
"examples": {
"type": "array",
"items": true
},
"multipleOf": {
"type": "number",
"exclusiveMinimum": 0
},
"maximum": {
"type": "number"
},
"exclusiveMaximum": {
"type": "number"
},
"minimum": {
"type": "number"
},
"exclusiveMinimum": {
"type": "number"
},
"maxLength": { "$ref": "#/definitions/nonNegativeInteger" },
"minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
"pattern": {
"type": "string",
"format": "regex"
},
"additionalItems": { "$ref": "#" },
"items": {
"anyOf": [
{ "$ref": "#" },
{ "$ref": "#/definitions/schemaArray" }
],
"default": true
},
"maxItems": { "$ref": "#/definitions/nonNegativeInteger" },
"minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
"uniqueItems": {
"type": "boolean",
"default": false
},
"contains": { "$ref": "#" },
"maxProperties": { "$ref": "#/definitions/nonNegativeInteger" },
"minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" },
"required": { "$ref": "#/definitions/stringArray" },
"additionalProperties": { "$ref": "#" },
"definitions": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"properties": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"default": {}
},
"patternProperties": {
"type": "object",
"additionalProperties": { "$ref": "#" },
"propertyNames": { "format": "regex" },
"default": {}
},
"dependencies": {
"type": "object",
"additionalProperties": {
"anyOf": [
{ "$ref": "#" },
{ "$ref": "#/definitions/stringArray" }
]
}
},
"propertyNames": { "$ref": "#" },
"const": true,
"enum": {
"type": "array",
"items": true,
"minItems": 1,
"uniqueItems": true
},
"type": {
"anyOf": [
{ "$ref": "#/definitions/simpleTypes" },
{
"type": "array",
"items": { "$ref": "#/definitions/simpleTypes" },
"minItems": 1,
"uniqueItems": true
}
]
},
"format": { "type": "string" },
"contentMediaType": { "type": "string" },
"contentEncoding": { "type": "string" },
"if": { "$ref": "#" },
"then": { "$ref": "#" },
"else": { "$ref": "#" },
"allOf": { "$ref": "#/definitions/schemaArray" },
"anyOf": { "$ref": "#/definitions/schemaArray" },
"oneOf": { "$ref": "#/definitions/schemaArray" },
"not": { "$ref": "#" }
},
"default": true
}
Loading

0 comments on commit 361634f

Please sign in to comment.