Redefined lexical constraint on 'token' datatype (#183)

* Redefined lexical constraint on 'token' datatype - #181 - removing references to upper-Unicode characters that break at least one processor - this is not actually relaxed, since the exclusion tests out as inoperative (the offending characters are not in the sets from which they are being excluded) * Rolling back #165 addressing usnistgov/OSCAL#956. We now have no upper-Unicode characters in the schema to be concerned about. * Adding schema-generation unit tests for 'token' datatype * Adjusted JSON Schema generation to capture latest datatype definitions #195 * Debugging path in datatype integration * Adding XSD datatype production logic - reads and rewrites JSON definitions in XSD syntax. * Adjustment to JSON -> XSD rough casting * XSD datatypes adjusted to align with JSON #195 * Adding new datatypes as aliases for old names #1186 * Touchups to inline documentation (for propagation to tools) * Replaced with updated mapping plus adjustments to XSD production * Now doing a better job excluding unneeded datatype (simpleType) definitions * Reconciled datatype merge to emit functional JSON schemas * Removing safety backups; small correction to Metaschema Schematron to avoid a runtime error.
usnistgov · Apr 15, 2022 · 47d6472 · 47d6472
1 parent db13180
commit 47d6472
Show file tree

Hide file tree

Showing 16 changed files with 967 additions and 136 deletions.
diff --git a/test-suite/schema-generation/token/datatype-token-test-FAIL.xml b/test-suite/schema-generation/token/datatype-token-test-FAIL.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- tokens can't start with digits or contain : -->
+<parent xmlns="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri" token-flag="7b5d9713">
+    <token-field>7b5d9713-4488-4d71-8e77-85b0b185d71a</token-field>
+    <token-field>:xyz</token-field>
+</parent>
diff --git a/test-suite/schema-generation/token/datatypes-token_metaschema.xml b/test-suite/schema-generation/token/datatypes-token_metaschema.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- OSCAL CATALOG METASCHEMA -->
+<!-- validate with XSD and Schematron (linked) -->
+<?xml-stylesheet type="text/css" href="../../../toolchains/oscal-m2/lib/metaschema-author.css"?>
+<?xml-model href="../../../toolchains/xslt-M4/validate/metaschema-check.sch" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<METASCHEMA xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://csrc.nist.gov/ns/oscal/metaschema/1.0 ../../../toolchains/xslt-M4/validate/metaschema.xsd"
+ xmlns="http://csrc.nist.gov/ns/oscal/metaschema/1.0">
+  <schema-name>Metaschema Unit Test: Token Datatype</schema-name>
+  <schema-version>1.0-milestone2</schema-version>
+  <short-name>metaschema-datatypes-token</short-name>
+  <namespace>http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri</namespace>
+  <json-base-uri>http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri</json-base-uri>
+
+  <define-assembly name="parent">
+    <formal-name>parent</formal-name>
+    <description>parent assembly</description>
+    <root-name>parent</root-name>
+    <define-flag name="token-flag" as-type="token" required="yes">
+      <formal-name>A metaschema 'token', a string without spaces suitable for use as a key value)</formal-name>
+      <description>Corresponds to XML NCName (no-colon name).</description>
+    </define-flag>
+    <model>
+      <field ref="token-field" max-occurs="unbounded">
+        <group-as name="token-fields"/>
+      </field>
+    </model>
+  </define-assembly>
+
+  <define-field name="token-field" as-type="token">
+    <formal-name>A metaschema 'token', a string without spaces for use as a key value</formal-name>
+    <description>Corresponds to XML NCName (no-colon name).</description>
+  </define-field>
+
+</METASCHEMA>
diff --git a/test-suite/schema-generation/token/datatypes-token_metaschema_JSON-SCHEMA.json b/test-suite/schema-generation/token/datatypes-token_metaschema_JSON-SCHEMA.json
@@ -0,0 +1,42 @@
+
+  { "$schema" : "http://json-schema.org/draft-07/schema#",
+    "$id" : "http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri/1.0-milestone2/metaschema-datatypes-token-schema.json",
+    "$comment" : "Metaschema Unit Test: Token Datatype: JSON Schema",
+    "type" : "object",
+    "definitions" : 
+    { "metaschema-datatypes-token-metaschema-datatypes-token:parent" : 
+      { "title" : "parent",
+        "description" : "parent assembly",
+        "$id" : "#assembly_metaschema-datatypes-token_parent",
+        "type" : "object",
+        "properties" : 
+        { "token-flag" : 
+          { "title" : "A metaschema 'token', a string without spaces suitable for use as a key value)",
+            "description" : "Corresponds to XML NCName (no-colon name).",
+            "type" : "string",
+            "pattern" : "^[_A-Za-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD][_A-Za-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD\\-\\.0-9\\u00B7\\u0300-\\u036F\\u203F-\\u2040]*$" },
+          "token-fields" : 
+          { "anyOf" : 
+            [ 
+              { "$ref" : "#field_metaschema-datatypes-token_token-field" },
+
+              { "type" : "array",
+                "minItems" : 1,
+                "items" : 
+                { "$ref" : "#field_metaschema-datatypes-token_token-field" } } ] } },
+        "required" : 
+        [ "token-flag" ],
+        "additionalProperties" : false },
+      "metaschema-datatypes-token-metaschema-datatypes-token:token-field" : 
+      { "title" : "A metaschema 'token', a string without spaces for use as a key value",
+        "description" : "Corresponds to XML NCName (no-colon name).",
+        "$id" : "#field_metaschema-datatypes-token_token-field",
+        "type" : "string",
+        "pattern" : "^[_A-Za-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD][_A-Za-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD\\-\\.0-9\\u00B7\\u0300-\\u036F\\u203F-\\u2040]*$" } },
+    "properties" : 
+    { "parent" : 
+      { "$ref" : "#assembly_metaschema-datatypes-token_parent" } },
+    "required" : 
+    [ "parent" ],
+    "additionalProperties" : false,
+    "maxProperties" : 1 }
diff --git a/test-suite/schema-generation/token/datatypes-token_metaschema_XML-SCHEMA.xsd b/test-suite/schema-generation/token/datatypes-token_metaschema_XML-SCHEMA.xsd
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+           xmlns:m="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
+           xmlns:metaschema-datatypes-token="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri"
+           xmlns:oscal-prose="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri"
+           elementFormDefault="qualified"
+           targetNamespace="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-uri"
+           version="1.0-milestone2">
+   <xs:annotation>
+      <xs:appinfo>
+         <m:schema-name>Metaschema Unit Test: Token Datatype</m:schema-name>
+         <m:schema-version>1.0-milestone2</m:schema-version>
+         <m:short-name>metaschema-datatypes-token</m:short-name>
+         <m:root>parent</m:root>
+      </xs:appinfo>
+   </xs:annotation>
+   <xs:element name="parent"
+               type="metaschema-datatypes-token:metaschema-datatypes-token-parent-ASSEMBLY"/>
+   <xs:complexType name="metaschema-datatypes-token-parent-ASSEMBLY">
+      <xs:annotation>
+         <xs:appinfo>
+            <m:formal-name>parent</m:formal-name>
+            <m:description>parent assembly</m:description>
+         </xs:appinfo>
+         <xs:documentation>
+            <b>parent</b>: parent assembly</xs:documentation>
+      </xs:annotation>
+      <xs:sequence>
+         <xs:element name="token-field"
+                     type="metaschema-datatypes-token:metaschema-datatypes-token-token-field-FIELD"
+                     minOccurs="0"
+                     maxOccurs="unbounded"/>
+      </xs:sequence>
+      <xs:attribute name="token-flag"
+                    use="required"
+                    type="metaschema-datatypes-token:token">
+         <xs:annotation>
+            <xs:appinfo>
+               <m:formal-name>A metaschema 'token', a string without spaces suitable for use as a key value)</m:formal-name>
+               <m:description>Corresponds to XML NCName (no-colon name).</m:description>
+            </xs:appinfo>
+            <xs:documentation>
+               <b>A metaschema 'token', a string without spaces suitable for use as a key value)</b>: Corresponds to XML NCName (no-colon name).</xs:documentation>
+         </xs:annotation>
+      </xs:attribute>
+   </xs:complexType>
+   <xs:simpleType name="metaschema-datatypes-token-token-field-FIELD">
+      <xs:annotation>
+         <xs:appinfo>
+            <m:formal-name>A metaschema 'token', a string without spaces for use as a key value</m:formal-name>
+            <m:description>Corresponds to XML NCName (no-colon name).</m:description>
+         </xs:appinfo>
+         <xs:documentation>
+            <b>A metaschema 'token', a string without spaces for use as a key value</b>: Corresponds to XML NCName (no-colon name).</xs:documentation>
+      </xs:annotation>
+      <xs:restriction base="metaschema-datatypes-token:token"/>
+   </xs:simpleType>
+   <xs:simpleType name="token">
+      <xs:annotation><!--<xs:documentation>Matching XSD NCName, except whitespace is not collapsed.</xs:documentation>-->
+         <xs:documentation>A string token following the rules of XML "no colon" names, with no whitespace. (XML names are single alphabetic characters
+      followed by alphanumeric characters, periods, underscores or dashes.)</xs:documentation>
+      </xs:annotation>
+      <xs:restriction base="xs:string">
+         <xs:whiteSpace value="preserve"/>
+         <xs:pattern value="[\i-[:]][\c-[:]]*"><!-- matches NCName (XML name without colons) -->
+            <xs:annotation><!--<xs:documentation>An XML initial character (but not colon), followed by any XML name character (but not colon).</xs:documentation>-->
+               <xs:documentation>A single token may not contain whitespace.</xs:documentation>
+            </xs:annotation>
+         </xs:pattern>
+      </xs:restriction>
+   </xs:simpleType>
+</xs:schema>
diff --git a/test-suite/schema-generation/token/datatypes-token_test_invalid_FAIL.json b/test-suite/schema-generation/token/datatypes-token_test_invalid_FAIL.json
@@ -0,0 +1,5 @@
+{
+    "parent": {
+        "token-flag": ":::"
+    }
+}
diff --git a/test-suite/schema-generation/token/datatypes-token_test_valid_PASS.json b/test-suite/schema-generation/token/datatypes-token_test_valid_PASS.json
@@ -0,0 +1,11 @@
+{
+    "parent": {
+        "token-flag": "F22",
+        "token-fields": [
+            "Af70e7c83",
+            "B4b1344c3",
+            "C398f9ba5",
+            "Ω"
+        ]
+    }
+}
diff --git a/toolchains/xslt-M4/make-metaschema-xsd.xpl b/toolchains/xslt-M4/make-metaschema-xsd.xpl
@@ -31,7 +31,7 @@
   </p:output>
 
   <!-- us-ascii encoding to provide escaping on upper-ASCII characters in the schema -->
-  <p:serialization port="f.final" indent="true" method="xml" omit-xml-declaration="false" encoding="us-ascii"/>
+  <p:serialization port="f.final" indent="true" method="xml" omit-xml-declaration="false"/>
   <p:output        port="f.final" primary="true">
     <p:pipe        port="result" step="final"/>
   </p:output>

diff --git a/toolchains/xslt-M4/nist-metaschema-MAKE-JSON-SCHEMA-debug.xsl b/toolchains/xslt-M4/nist-metaschema-MAKE-JSON-SCHEMA-debug.xsl
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="3.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:nm="http://csrc.nist.gov/ns/metaschema"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:xs="http://www.w3.org/2001/XMLSchema"
+    exclude-result-prefixes="#all">
+
+    <!-- Purpose: Produce a JSON Schema reflecting constraints defined in a metaschema -->
+    <!-- Dependencies: This is a 'shell' XSLT and calls several steps in sequence, each implemented as an XSLT -->
+    <!-- Input: A top-level metaschema; this XSLT also composes metaschema input so composition is not necessary -->
+    <!-- Output: A JSON Schema (v7) describing a JSON format consistent with definitions given in the input metaschema -->
+    <!-- Note: This XSLT uses the transform() function to execute a series of transformations (referenced out of line) over its input -->
+
+    <!-- Debug variant writes out JSON Schema as XPath XML subject to validation -->
+    <!-- Check its fitness for serialization by validating against XSD ../../support/xml/xpath-json.xsd -->
+
+    <xsl:output method="xml" indent="yes"/>
+
+    <!-- Turning $trace to 'on' will
+         - emit runtime messages with each transformation, and
+         - retain nm:ERROR and nm:WARNING messages in results. -->
+
+    <xsl:param name="trace" as="xs:string">off</xsl:param>
+    <xsl:variable name="louder" select="$trace = 'on'"/>
+
+    <xsl:variable name="xslt-base" select="document('')/document-uri()"/>
+
+    <xsl:import href="nist-metaschema-metaprocess.xsl"/>
+
+    <!-- The $transformation-sequence declares transformations to be applied in order. -->
+    <xsl:variable name="transformation-sequence">
+        <nm:transform version="3.0">compose/metaschema-collect.xsl</nm:transform>
+        <nm:transform version="3.0">compose/metaschema-build-refs.xsl</nm:transform>
+        <nm:transform version="3.0">compose/metaschema-trim-extra-modules.xsl</nm:transform>
+        <nm:transform version="3.0">compose/metaschema-prune-unused-definitions.xsl</nm:transform>
+        <nm:transform version="3.0">compose/metaschema-resolve-use-names.xsl</nm:transform>
+        <nm:transform version="3.0">compose/metaschema-resolve-sibling-names.xsl</nm:transform>
+        <nm:transform version="3.0">compose/metaschema-digest.xsl</nm:transform>
+        <nm:transform version="3.0">compose/annotate-composition.xsl</nm:transform>
+
+        <nm:transform version="3.0">schema-gen/make-json-schema-metamap.xsl</nm:transform>
+        <!--<nm:transform version="3.0">util/xpath-json-to-json.xsl</nm:transform>-->
+    </xsl:variable>
+
+</xsl:stylesheet>
diff --git a/toolchains/xslt-M4/nist-metaschema-MAKE-XSD.xsl b/toolchains/xslt-M4/nist-metaschema-MAKE-XSD.xsl
@@ -11,7 +11,7 @@
     <!-- Output: An XSD describing an XML format consistent with definitions given in the input metaschema -->
     <!-- Note: This XSLT uses the transform() function to execute a series of transformations (referenced out of line) over its input -->
 
-    <xsl:output method="xml" indent="yes" encoding="us-ascii"/>
+    <xsl:output method="xml" indent="yes"/>
 
     <!-- Turning $trace to 'on' will
          - emit runtime messages with each transformation, and

diff --git a/toolchains/xslt-M4/schema-gen/configure-namespaces.xsl b/toolchains/xslt-M4/schema-gen/configure-namespaces.xsl
@@ -5,7 +5,7 @@
     xmlns:m="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
     xpath-default-namespace="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
     exclude-result-prefixes="xs math m"
-    version="2.0"
+    version="3.0"
 
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
 
@@ -49,13 +49,27 @@
     <xsl:template name="namespace-fixup">
         <xsl:namespace name="m">http://csrc.nist.gov/ns/oscal/metaschema/1.0</xsl:namespace>
         <xsl:namespace name="{$declaration-prefix}" select="$target-namespace"/>
-        <xsl:namespace name="oscal-prose" select="$target-namespace"/>
+        <xsl:namespace name="metaschema-datatypes" select="$target-namespace"/>
     </xsl:template>
 
     <xsl:template match="xs:documentation//text() | m:*//text()" mode="wire-ns">
         <xsl:copy-of select="."/>
     </xsl:template>
 
+    <xsl:key name="datatype-invocation" use="substring-after(@base,':')"
+        match="xs:extension | xs:restriction"/>
+
+    <xsl:key name="datatype-invocation" use="substring-after(@type,':')"
+        match="xs:element[exists(@type)]
+        | xs:attribute[exists(@type)]"/>
+
+    <xsl:template match="/*/xs:simpleType" mode="wire-ns">
+        <xsl:if test="exists(key('datatype-invocation',@name))">
+          <xsl:next-match/>
+        </xsl:if>
+    </xsl:template>
+
+<!-- matching xs:restriction/@base rewrite m: as {$declaration-prefix}: -->
     <xsl:template match="text()" mode="wire-ns"/>
 
 </xsl:stylesheet>