Produce Metaschemas without XXEs (#1665)

build/Makefile

@@ -14,7 +14,7 @@ help: ## Show this help message
 all: artifacts checks archives ## Run all pipelines
 
 .PHONY: artifacts
-artifacts: schemas converters ## Generate all artifacts
+artifacts: schemas converters resolved-metaschemas ## Generate all artifacts
 
 .PHONY: checks
 checks: linkcheck validate test-profile-resolution ## Run all tests and checks
@@ -26,11 +26,13 @@ node_modules: package.json package-lock.json
 	npm ci
 
 .PHONY: clean
-clean: clean-schemas clean-linkcheck clean-converters clean-archives ## Remove all generated content
+clean: clean-schemas clean-linkcheck clean-converters clean-archives clean-resolved-metaschemas ## Remove all generated content
 
 METASCHEMA_XSLT_COMMAND:=metaschema-xslt/bin/metaschema-xslt
 SRC_DIR:=../src/metaschema
-# Contains the metaschemas that do not contain "common" or "metadata"
+# Contains all OSCAL metaschema modules, including those without exported roots
+ALL_METASCHEMAS:=$(shell find $(SRC_DIR) -name '*_metaschema.xml')
+# Contains the OSCAL metaschema modules that contain root assemblies
 METASCHEMAS:=$(shell find $(SRC_DIR) -name '*_metaschema.xml' -a ! -name '*common*' -a ! -name '*metadata*')
 GENERATED_DIR:=generated
 
@@ -71,6 +73,33 @@ clean-converters: ## Remove generated converters
 	rm -fr $(GENERATED_DIR)/*_xml-to-json-converter.xsl
 	rm -fr $(GENERATED_DIR)/*_json-to-xml-converter.xsl
 
+###################################
+# Resolved Metaschemas Generation #
+###################################
+
+POM_PATH:=./pom.xml
+define EXEC_SAXON
+	mvn --quiet -f "$(POM_PATH)" exec:java \
+    	-Dexec.mainClass="net.sf.saxon.Transform" \
+		-Dexec.args="$1"
+endef
+
+RESOLVER_STYLESHEET:=./resolve-entities.xsl
+
+RESOLVED_METASCHEMA_SUFFIX:=RESOLVED
+RESOLVED_METASCHEMAS:=$(patsubst $(SRC_DIR)/%_metaschema.xml,$(GENERATED_DIR)/%_metaschema_$(RESOLVED_METASCHEMA_SUFFIX).xml,$(ALL_METASCHEMAS))
+
+$(GENERATED_DIR)/%_metaschema_$(RESOLVED_METASCHEMA_SUFFIX).xml: $(SRC_DIR)/%_metaschema.xml
+	@mkdir -p $(GENERATED_DIR)
+	$(call EXEC_SAXON,-s:$(SRC_DIR)/$*_metaschema.xml -xsl:$(RESOLVER_STYLESHEET) -o:$@ importHrefSuffix=$(RESOLVED_METASCHEMA_SUFFIX))
+
+.PHONY: resolved-metaschemas
+resolved-metaschemas: $(RESOLVED_METASCHEMAS) ## Generate the resolved metaschema modules
+
+.PHONY: clean-resolved-metaschemas
+clean-resolved-metaschemas: ## Remove generated resolvd metaschema modules
+	rm -f $(RESOLVED_METASCHEMAS)
+
 ######################
 # Archive Generation #
 ######################
@@ -83,14 +112,15 @@ ARCHIVE_TEMP_DIR:=$(GENERATED_DIR)/archive_temp
 .PHONY: archives
 archives: $(ZIP_ARCHIVE) ## Archive converters and schemas
 
-$(ZIP_ARCHIVE) $(TARBZ2_ARCHIVE): converters schemas
+$(ZIP_ARCHIVE) $(TARBZ2_ARCHIVE): converters schemas resolved-metaschemas
 	@echo Generating archive
-	mkdir -p $(ARCHIVE_TEMP_DIR)/{json,xml}/{convert,schema}
-	cp ../src/release/README.txt "$(ARCHIVE_TEMP_DIR)/README.md"
+	mkdir -p $(ARCHIVE_TEMP_DIR)/{{json,xml}/{convert,schema},metaschema}
+	cp ../src/release/release-readme.md "$(ARCHIVE_TEMP_DIR)/README.md"
 	cp $(XSD_OUTPUTS) "$(ARCHIVE_TEMP_DIR)/xml/schema"
 	cp $(JSONSCHEMA_OUTPUTS) "$(ARCHIVE_TEMP_DIR)/json/schema"
 	cp $(XML2JSON_CONVERTERS) "$(ARCHIVE_TEMP_DIR)/xml/convert"
 	cp $(JSON2XML_CONVERTERS) "$(ARCHIVE_TEMP_DIR)/json/convert"
+	cp $(RESOLVED_METASCHEMAS) "$(ARCHIVE_TEMP_DIR)/metaschema"
 
 	(cd "$(ARCHIVE_TEMP_DIR)" && zip -r $(abspath $(ZIP_ARCHIVE)) .)
 	tar -jcvf "$(TARBZ2_ARCHIVE)" -C "$(ARCHIVE_TEMP_DIR)" .
@@ -161,3 +191,17 @@ validate-composition-%:
 .PHONY: test-profile-resolution
 test-profile-resolution: ## Unit test the profile resolver
 	$(MAKE) -C ../src/utils/resolver-pipeline test
+
+###################
+# Utility Targets #
+###################
+
+# These targets may be used by consumers of the OSCAL repository
+
+# All artifacts typically included in a release
+RELEASE_ARTIFACTS:=$(XSD_OUTPUTS) $(JSONSCHEMA_OUTPUTS) $(XML2JSON_CONVERTERS) $(JSON2XML_CONVERTERS) $(RESOLVED_METASCHEMAS) $(ZIP_ARCHIVE) $(TARBZ2_ARCHIVE)
+
+# This target is used by OSCAL-Reference to generate meta-redirects for release assets
+.PHONY: list-release-artifacts
+list-release-artifacts: ## Print out a list of all artifacts typically included in an OSCAL release
+	@echo $(RELEASE_ARTIFACTS)

build/README.md

@@ -34,7 +34,8 @@ Developers can generate schemas locally using the `make artifacts` command.
 Developers can also generate individual artifacts using the following commands:
 
 * `make schemas`: Generates the JSON Schemas and XSDs off of the source Metaschemas;
-* `make converters`: Generates the XSLT stylesheets for JSON<->XML conversion off of the source Metaschemas.
+* `make converters`: Generates the XSLT stylesheets for JSON<->XML conversion off of the source Metaschemas;
+* `make resolved-metaschemas`: Resolves external entities (XXE) present in the source OSCAL Metaschema modules for use with tools that do not support XXEs.
 
 ### Checks
 

build/resolve-entities.xsl

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+     Entity resolver for OSCAL Metaschemas
+
+     This stylesheet:
+     1. Copies an input metaschema module, resolving all external entities
+     2. Replace all import/@href's to match a given $importHrefSuffix parameter
+-->
+<xsl:stylesheet version="1.0"
+                xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:metaschema="http://csrc.nist.gov/ns/oscal/metaschema/1.0">
+    <xsl:output omit-xml-declaration="no" indent="yes"/>
+
+    <xsl:param name="importHrefSuffix" select="'RESOLVED'"/>
+
+    <!--
+         XSLT1.0 compatible string replacement
+
+         Via https://gist.github.com/ijy/6572481
+    -->
+    <xsl:template name="string-replace">
+        <xsl:param name="string" />
+        <xsl:param name="replace" />
+        <xsl:param name="with" />
+
+        <xsl:choose>
+            <xsl:when test="contains($string, $replace)">
+                <xsl:value-of select="substring-before($string, $replace)" />
+                <xsl:value-of select="$with" />
+                <xsl:call-template name="string-replace">
+                    <xsl:with-param name="string" select="substring-after($string,$replace)" />
+                    <xsl:with-param name="replace" select="$replace" />
+                    <xsl:with-param name="with" select="$with" />
+                </xsl:call-template>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:value-of select="$string" />
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+
+    <!-- Simple identity transform, resolving entities implicitly -->
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()"/>
+        </xsl:copy>
+    </xsl:template>
+
+    <!-- Transform import/@href using $importHrefSuffix -->
+    <xsl:template match="metaschema:import/@href">
+        <xsl:attribute name="href">
+            <!-- oscal_poam_metaschema.xml => oscal_poam_metaschema_$importHrefSuffix.xml -->
+            <xsl:call-template name="string-replace">
+                <xsl:with-param name="string" select="." />
+                <xsl:with-param name="replace" select="'.xml'" />
+                <xsl:with-param name="with" select="concat('_', $importHrefSuffix, '.xml')" />
+            </xsl:call-template>
+        </xsl:attribute>
+    </xsl:template>
+</xsl:stylesheet>

decisions/0005-repository-reorganization.md

@@ -5,7 +5,7 @@ Date: 2023/07/06
 
 ## Status
 
-Proposed
+Approved
 
 ## Context
 

decisions/0006-source-metaschema-xxes.md

@@ -0,0 +1,32 @@
+# Production of transformed source Metaschema modules without external entities (XXEs)
+
+Date: 08/25/2023
+
+## Status
+
+Proposed
+
+## Context
+
+We wish to remove or mitigate points friction encountered by NIST or community OSCAL developers producing tooling that consumes the OSCAL Metaschema module sources.
+
+The OSCAL Metaschema modules currently use external entities to prevent duplication of constraint data.
+These external entities are important for modeling ergonomics and cannot be removed until a Metaschema-native approach is stabilized, however external entities have a storied history of abuse.
+So called "XML External Entity (XXE) Attacks", along with the additional complexity needed to support external entity resolution, have led to a situation where many XML parsers do not ship with XXE functionality.
+This has put additional burden on OSCAL tool developers seeking to consume the source Metaschemas, who have either had to choose from the small subset of XML parsers that support external entities (if one exists for their target language at all) and inherit all additional risks that come with XXEs, or perform transformation of the source Metaschema modules before consuming them.
+
+- Related to Issue [#1665](https://github.com/usnistgov/OSCAL/issues/1665)
+
+## Decision
+
+The NIST OSCAL Team should include the "resolved" Metaschema module sources as an artifact generated upon release.
+Additionally, the NIST OSCAL Team should document the process for obtaining a resolved Metaschema module as part of the [streamlined build process](./0005-repository-reorganization.md#streamlined-build-process).
+
+In the event that Metaschema stabilizes constraint imports, the NIST OSCAL team will deprecate the generated artifacts for removal in the next major release.
+
+## Consequences
+
+This decision will not have any breaking changes to our process, however:
+
+1. The NIST team will be responsible for reviewing the additional artifacts before performing a release.
+2. In the event that Metaschema stabilizes constraint imports, the NIST OSCAL team will have to maintain deprecated artifacts until the next major version.

src/release/README.txt → src/release/release-readme.md

@@ -9,8 +9,8 @@ Documentation for the OSCAL models can be found at: https://pages.nist.gov/OSCAL
 
 This release provides 2 types of resources, each located in a different subdirectory:
 
-- xml: Provides the XML schemas and content converters that are needed to support the OSCAL model XML-based formats. Instructions for using this information can be found at: https://github.com/usnistgov/OSCAL/tree/master/xml.
-- json: Provides the JSON schemas and content converters that are needed to support the OSCAL model JSON-based formats. Instructions for using this information can be found at: https://github.com/usnistgov/OSCAL/tree/master/json.
+- `xml/` and `json/`: Provides the XML and JSON schemas and content converters that are needed to support the OSCAL model. Instructions for using these artifacts can be found at https://github.com/usnistgov/OSCAL/blob/develop/build/README.md#artifact-usage
+- `metaschema/`: Provides the source OSCAL Metaschema modules with all external entities (XXE) resolved for tools that do not support XXEs.
 
 These directories provide stable, released versions of the resources provided on the OSCAL GitHub repository: https://github.com/usnistgov/OSCAL.
 
@@ -32,6 +32,6 @@ OSCAL is being developed in a public GitHub repository, in collaboration with in
 - Help with developing OSCAL models and associated content.
 - Assistance with developing documentation, tutorials, and other informational resources.
 
-If you are interested in helping, please visit or contributing page for more information at: https://github.com/usnistgov/OSCAL/blob/master/CONTRIBUTING.md.
+If you are interested in helping, please visit or contributing page for more information at: https://github.com/usnistgov/OSCAL/blob/main/CONTRIBUTING.md.
 
 Please direct any questions, comments, concerns, or kudos by email to: [email protected].