Add schema generation support (#25)

* Implemented JSON schema generation. Resolves usnistgov/OSCAL#1145. Resolves usnistgov/OSCAL#1132. Resolves usnistgov/OSCAL#1131. Resolves usnistgov/OSCAL#1003. * Improved JSON schema generation. Worked out unicode support for JSON and XSD. Resolves usnistgov/OSCAL#1127. Resolves usnistgov/OSCAL#956. * Fixed bugs in DateAdapter causing dates without timezones to not parse or write properly. * Refactored and moved validation API to metaschema-model-common. * Fixed a bug in write operations causing some file contents to not be truncated when overwriting files. Added StandardOpenOption.TRUNCATE_EXISTING to write operations. * Fixed a bug in array writing for properties using in-json=ARRAY or SINGLETON_OR_ARRAY, that caused an error related to the closing array syntax. * Refactored Java class generation to provide more information to the caller about generated classes. * Added dynamic test support for Metaschema-based testing in a new module metaschema-testing. * Updated JSON parsing code to be more resilient. * Updated to new metaschema feature branch for major refactor. * Schema generation refactor, focusing on XML and JSON alignment. * Fixed some compiler, PMD, and checkstyle warnings. Making incremental progress towards getting these cleaned up. * Simplified and removed some unneeded interfaces and abstract classes. Reducing public/protected classes and methods. * Fixed bugs around inline type handling in schema generation. * Migrated the metaschema-java-binding-annotations module sources to metaschema-java-binding. * Completed full support for Metaschema information in bound Java classes. * Refactored function library, adding functions for abs, boolean, ceiling, compare, data, floor, and round. Identified all functions that need to be implemented eventually. * Added some Javadocs. * Fixing relative path in POM causing build errors in CI.
usnistgov · Apr 15, 2022 · ee9dcc4 · ee9dcc4
1 parent 14ad5b0
commit ee9dcc4
Show file tree

Hide file tree

Showing 527 changed files with 31,236 additions and 10,792 deletions.
diff --git a/.gitignore b/.gitignore
@@ -22,3 +22,4 @@ release.properties
 node_modules/
 package.json
 package-lock.json
+/metaschema-schema-generator/test-schemagen
diff --git a/README.md b/README.md
@@ -1,3 +1,33 @@
 # Metaschema Java Tools and Libraries
 
-Documentation for and implementations of the [Metaschema modeling language](https://github.com/usnistgov/metaschema) in Java. This Java metaschema toolchain provides support for Java code generation based on one or more Metaschema.
+Documentation for and implementations of the [Metaschema modeling language](https://github.com/usnistgov/metaschema) in Java. This Java metaschema toolchain provides support for:
+
+- Java objects for loading and working with XML-based Metaschema constructs. This functionality is provided by the [Metaschema XML model](metaschema-model/).
+- Java bean code generation based on one or more Metaschema using Maven. This functionality is provided by the [Metaschema Maven plugin](metaschema-maven-plugin/).
+- A Java parser for reading and writing XML, JSON, or YAML into Java beans generated by this framework. This functionality is provided by the [Metaschema Java Binding Parser](metaschema-java-binding/).
+
+## Using as a Maven dependency
+
+This projects modules are published to [Maven Central](https://search.maven.org/search?q=g:gov.nist.secauto.metaschema).
+
+You can include these artifacts in your Maven POM as a dependency.
+
+## Building
+
+This project can be built with [Apache Maven](https://maven.apache.org/) version 3.8.4 or greater.
+
+The following instructions can be used to clone and build this project.
+
+1. Clone the GitHub repository.
+
+```bash
+git clone --recurse-submodules https://github.com/usnistgov/metaschema-java.git 
+```
+
+2. Build the project with Maven
+
+```bash
+mvn install
+```
+
+
diff --git a/metaschema-documentation-generator/pom.xml b/metaschema-documentation-generator/pom.xml
@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>gov.nist.secauto.metaschema</groupId>
 		<artifactId>metaschema-framework</artifactId>
-		<version>0.5.1-SNAPSHOT</version>
+		<version>0.7.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>metaschema-documentation-generator</artifactId>
 
@@ -42,7 +42,6 @@
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>
-			<version>2.8.0</version>
 			<scope>test</scope>
 		</dependency>
 	</dependencies>

diff --git a/...tor/src/main/java/gov/nist/secauto/metaschema/docsgen/AbstractDocumentationGenerator.java b/...tor/src/main/java/gov/nist/secauto/metaschema/docsgen/AbstractDocumentationGenerator.java
@@ -32,4 +32,5 @@
 public abstract class AbstractDocumentationGenerator
     extends AbstractFreemarkerGenerator
     implements DocumentationGenerator {
+
 }
diff --git a/...n-generator/src/main/java/gov/nist/secauto/metaschema/docsgen/DocumentationGenerator.java b/...n-generator/src/main/java/gov/nist/secauto/metaschema/docsgen/DocumentationGenerator.java
@@ -26,7 +26,7 @@
 
 package gov.nist.secauto.metaschema.docsgen;
 
-import gov.nist.secauto.metaschema.freemarker.support.FreemarkerGenerator;
+import gov.nist.secauto.metaschema.freemarker.support.IFreemarkerGenerator;
 
-public interface DocumentationGenerator extends FreemarkerGenerator {
+public interface DocumentationGenerator extends IFreemarkerGenerator {
 }
diff --git a/...r/src/main/java/gov/nist/secauto/metaschema/docsgen/XmlOutlineDocumentationGenerator.java b/...r/src/main/java/gov/nist/secauto/metaschema/docsgen/XmlOutlineDocumentationGenerator.java
@@ -27,11 +27,13 @@
 package gov.nist.secauto.metaschema.docsgen;
 
 import java.io.IOException;
+import java.util.Map;
 
 import freemarker.core.ParseException;
 import freemarker.template.Configuration;
 import freemarker.template.MalformedTemplateNameException;
 import freemarker.template.Template;
+import freemarker.template.TemplateException;
 import freemarker.template.TemplateNotFoundException;
 
 public class XmlOutlineDocumentationGenerator
@@ -43,4 +45,8 @@ protected Template getTemplate(Configuration cfg)
     return cfg.getTemplate("xml-outline.ftlx");
   }
 
+  @Override
+  protected void buildModel(Configuration cfg, Map<String, Object> root) throws IOException, TemplateException {
+    // nothing to add
+  }
 }
diff --git a/...c/test/java/gov/nist/secauto/metaschema/docsgen/XmlOutlineDocumentationGeneratorTest.java b/...c/test/java/gov/nist/secauto/metaschema/docsgen/XmlOutlineDocumentationGeneratorTest.java
@@ -27,10 +27,8 @@
 package gov.nist.secauto.metaschema.docsgen;
 
 import gov.nist.secauto.metaschema.model.MetaschemaLoader;
-import gov.nist.secauto.metaschema.model.UsedDefinitionModelWalker;
 import gov.nist.secauto.metaschema.model.common.IMetaschema;
 import gov.nist.secauto.metaschema.model.common.MetaschemaException;
-import gov.nist.secauto.metaschema.model.common.definition.IDefinition;
 
 import org.apache.commons.io.output.TeeOutputStream;
 import org.jetbrains.annotations.NotNull;
@@ -40,7 +38,6 @@
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.net.URL;
-import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -63,14 +60,12 @@ void test() throws TemplateNotFoundException, MalformedTemplateNameException, Pa
     IMetaschema metaschema
         = loader.loadXmlMetaschema(new URL(
             "https://raw.githubusercontent.com/usnistgov/OSCAL/v1.0.0/src/metaschema/oscal_complete_metaschema.xml"));
+
     metaschemas.add(metaschema);
 
-    Collection<@NotNull ? extends IDefinition> definitions
-        = UsedDefinitionModelWalker.collectUsedDefinitionsFromMetaschema(metaschemas);
-
     try (FileOutputStream fos = new FileOutputStream("xml-outline.html")) {
       TeeOutputStream out = new TeeOutputStream(System.out, fos);
-      generator.generateFromDefinitions(definitions, new OutputStreamWriter(out));
+      generator.generateFromMetaschema(metaschema, new OutputStreamWriter(out));
     }
   }
 }
diff --git a/metaschema-documentation-generator/xml-outline.html b/metaschema-documentation-generator/xml-outline.html
@@ -0,0 +1,51 @@
+<div xmlns="http://www.w3.org/1999/xhtml" class="xml-outline">
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+  <div class="model-container">
+    <details class="OM-entry" open="open">
+      <summary>
+        <span class="sq"><span class="nobr">&lt;<a class="OM-name" href="../xml-reference/#/assessment-plan">assessment-plan</a></span> <span class="nobr" id="/assessment-plan/@uuid"><a class="OM-name" href="../xml-reference/#/assessment-plan/@uuid">uuid</a>="<span class="OM-datatype"><a href="/reference/datatypes/#uuid">uuid</a></span>"</span>&gt;<span class="show-closed"> … <span class="nobr">&lt;/assessment-plan&gt;</span></span></span><span class="sq cardinality"> <span class="OM-cardinality">[1]</span></span>
+      </summary>
+    </details>
+  </div>
+</div>
diff --git a/metaschema-freemarker-support/pom.xml b/metaschema-freemarker-support/pom.xml
@@ -4,7 +4,7 @@
 	<parent>
 		<artifactId>metaschema-framework</artifactId>
 		<groupId>gov.nist.secauto.metaschema</groupId>
-		<version>0.5.1-SNAPSHOT</version>
+		<version>0.7.0-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>metaschema-freemarker-support</artifactId>
@@ -41,7 +41,6 @@
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>
-			<version>2.8.0</version>
 			<scope>test</scope>
 		</dependency>
 	</dependencies>

diff --git a/...main/java/gov/nist/secauto/metaschema/freemarker/support/AbstractFreemarkerGenerator.java b/...main/java/gov/nist/secauto/metaschema/freemarker/support/AbstractFreemarkerGenerator.java
@@ -26,10 +26,10 @@
 
 package gov.nist.secauto.metaschema.freemarker.support;
 
-import gov.nist.secauto.metaschema.model.UsedDefinitionModelWalker;
+import gov.nist.secauto.metaschema.model.common.IAssemblyDefinition;
+import gov.nist.secauto.metaschema.model.common.IDefinition;
 import gov.nist.secauto.metaschema.model.common.IMetaschema;
-import gov.nist.secauto.metaschema.model.common.definition.IAssemblyDefinition;
-import gov.nist.secauto.metaschema.model.common.definition.IDefinition;
+import gov.nist.secauto.metaschema.model.common.UsedDefinitionModelWalker;
 
 import org.jetbrains.annotations.NotNull;
 
@@ -44,29 +44,34 @@
 
 import freemarker.cache.ClassTemplateLoader;
 import freemarker.core.ParseException;
+import freemarker.ext.beans.BeansWrapper;
+import freemarker.ext.beans.BeansWrapperBuilder;
 import freemarker.template.Configuration;
 import freemarker.template.MalformedTemplateNameException;
 import freemarker.template.Template;
 import freemarker.template.TemplateException;
 import freemarker.template.TemplateExceptionHandler;
+import freemarker.template.TemplateHashModel;
 import freemarker.template.TemplateNotFoundException;
+import freemarker.template.Version;
 
-public abstract class AbstractFreemarkerGenerator implements FreemarkerGenerator {
-  private boolean debug = false;
+public abstract class AbstractFreemarkerGenerator implements IFreemarkerGenerator {
+  private static final Version CONFIG_VERSION = Configuration.VERSION_2_3_30;
+  private static final boolean DEBUG = false;
 
   protected Configuration newConfiguration() {
     // Create your Configuration instance, and specify if up to what FreeMarker
     // version (here 2.3.29) do you want to apply the fixes that are not 100%
     // backward-compatible. See the Configuration JavaDoc for details.
-    Configuration cfg = new Configuration(Configuration.VERSION_2_3_30);
+    Configuration cfg = new Configuration(CONFIG_VERSION);
 
     // // Specify the source where the template files come from. Here I set a
     // // plain directory for it, but non-file-system sources are possible too:
     // cfg.setDirectoryForTemplateLoading(new File("/where/you/store/templates"));
     ClassTemplateLoader ctl = new ClassTemplateLoader(getClass(), "/templates");
     cfg.setTemplateLoader(ctl);
 
-    if (debug) {
+    if (DEBUG) {
       cfg.setTemplateExceptionHandler(TemplateExceptionHandler.DEBUG_HANDLER);
     } else {
       cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
@@ -85,25 +90,23 @@ protected Configuration newConfiguration() {
   }
 
   @Override
-  public void generateFromMetaschemas(Collection<@NotNull ? extends IMetaschema> metaschemas, Writer out)
+  public void generateFromMetaschema(@NotNull IMetaschema metaschema, Writer out)
       throws TemplateNotFoundException, MalformedTemplateNameException, TemplateException, ParseException, IOException {
-    Objects.requireNonNull(metaschemas, "metaschemas");
-
     Collection<@NotNull ? extends IDefinition> definitions
-        = UsedDefinitionModelWalker.collectUsedDefinitionsFromMetaschema(metaschemas);
-    generateFromDefinitions(definitions, out);
+        = UsedDefinitionModelWalker.collectUsedDefinitionsFromMetaschema(metaschema);
+    generate(metaschema, definitions, out);
   }
 
-  @Override
-  public void generateFromDefinitions(Collection<@NotNull ? extends IDefinition> definitions, Writer out)
+  protected void generate(@NotNull IMetaschema metaschema, Collection<@NotNull ? extends IDefinition> definitions,
+      Writer out)
       throws TemplateNotFoundException, MalformedTemplateNameException, TemplateException, ParseException, IOException {
     Objects.requireNonNull(definitions, "definitions");
     Set<IMetaschema> metaschemas = new LinkedHashSet<>();
     Set<IAssemblyDefinition> rootAssemblies = new LinkedHashSet<>();
     for (IDefinition definition : definitions) {
-      IMetaschema metaschema = definition.getContainingMetaschema();
-      if (!metaschemas.contains(metaschema)) {
-        metaschemas.add(metaschema);
+      IMetaschema containingMetaschema = definition.getContainingMetaschema();
+      if (!metaschemas.contains(containingMetaschema)) {
+        metaschemas.add(containingMetaschema);
       }
 
       if (definition instanceof IAssemblyDefinition) {
@@ -116,23 +119,35 @@ public void generateFromDefinitions(Collection<@NotNull ? extends IDefinition> d
 
     Configuration cfg = newConfiguration();
 
-    // Create the root hash. We use a Map here, but it could be a JavaBean too.
-    Map<String, Object> root = new HashMap<>();
-
     // add directives
     cfg.setSharedVariable("toCamelCase", new ToCamelCaseMethod());
     cfg.setSharedVariable("markupToHTML", new MarkupToHtmlMethod());
+    cfg.setSharedVariable("markupToMarkdown", new MarkupToMarkdownMethod());
+
+    // add constants
+    BeansWrapper wrapper = new BeansWrapperBuilder(CONFIG_VERSION).build();
+    TemplateHashModel staticModels = wrapper.getStaticModels();
+
+    // add static values
+    cfg.setSharedVariable("statics", staticModels);
+
+    // Create the root hash. We use a Map here, but it could be a JavaBean too.
+    Map<String, Object> root = new HashMap<>(); // NOPMD - Freemarker templates run in a single thread
 
     // add metaschema model
+    root.put("metaschema", metaschema);
     root.put("metaschemas", metaschemas);
     root.put("definitions", definitions);
     root.put("root-definitions", rootAssemblies);
 
     Template template = getTemplate(cfg);
+    buildModel(cfg, root);
 
     template.process(root, out);
   }
 
   protected abstract Template getTemplate(Configuration cfg)
       throws TemplateNotFoundException, MalformedTemplateNameException, ParseException, IOException;
+
+  protected abstract void buildModel(Configuration cfg, Map<String, Object> root) throws IOException, TemplateException;
 }
diff --git a/...eemarker/support/FreemarkerGenerator.java → ...emarker/support/IFreemarkerGenerator.java b/...eemarker/support/FreemarkerGenerator.java → ...emarker/support/IFreemarkerGenerator.java
@@ -27,23 +27,18 @@
 package gov.nist.secauto.metaschema.freemarker.support;
 
 import gov.nist.secauto.metaschema.model.common.IMetaschema;
-import gov.nist.secauto.metaschema.model.common.definition.IDefinition;
 
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.io.Writer;
-import java.util.Collection;
 
 import freemarker.core.ParseException;
 import freemarker.template.MalformedTemplateNameException;
 import freemarker.template.TemplateException;
 import freemarker.template.TemplateNotFoundException;
 
-public interface FreemarkerGenerator {
-  void generateFromMetaschemas(@NotNull Collection<@NotNull ? extends IMetaschema> metaschemas, @NotNull Writer out)
-      throws TemplateNotFoundException, MalformedTemplateNameException, TemplateException, ParseException, IOException;
-
-  void generateFromDefinitions(@NotNull Collection<@NotNull ? extends IDefinition> definitions, @NotNull Writer out)
+public interface IFreemarkerGenerator {
+  void generateFromMetaschema(@NotNull IMetaschema metaschema, @NotNull Writer out)
       throws TemplateNotFoundException, MalformedTemplateNameException, TemplateException, ParseException, IOException;
 }
diff --git a/...port/src/main/java/gov/nist/secauto/metaschema/freemarker/support/MarkupToHtmlMethod.java b/...port/src/main/java/gov/nist/secauto/metaschema/freemarker/support/MarkupToHtmlMethod.java
@@ -29,8 +29,8 @@
 import com.ctc.wstx.api.WstxOutputProperties;
 import com.ctc.wstx.stax.WstxOutputFactory;
 
-import gov.nist.secauto.metaschema.model.common.datatype.markup.MarkupMultiline;
 import gov.nist.secauto.metaschema.model.common.datatype.markup.IMarkupText;
+import gov.nist.secauto.metaschema.model.common.datatype.markup.MarkupMultiline;
 import gov.nist.secauto.metaschema.model.common.datatype.markup.MarkupXmlStreamWriter;
 
 import org.codehaus.stax2.XMLOutputFactory2;
@@ -54,16 +54,13 @@ public class MarkupToHtmlMethod implements TemplateMethodModelEx {
 
   @Override
   public Object exec(@SuppressWarnings("rawtypes") List arguments) throws TemplateModelException {
-
     if (arguments.isEmpty() || arguments.size() < 2 || arguments.size() > 3) {
       throw new TemplateModelException(String.format(
           "This method requires a %s typed object argument, a namspace string argument, and may optionally have a"
               + " prefix string argument.",
           IMarkupText.class.getName()));
     }
 
-    String namespace = DeepUnwrap.unwrap((TemplateModel) arguments.get(1)).toString();
-
     String prefix = null;
     if (arguments.size() == 3) {
       prefix = DeepUnwrap.unwrap((TemplateModel) arguments.get(2)).toString();
@@ -77,6 +74,7 @@ public Object exec(@SuppressWarnings("rawtypes") List arguments) throws Template
     }
 
     IMarkupText text = (IMarkupText) markupObject;
+    String namespace = DeepUnwrap.unwrap((TemplateModel) arguments.get(1)).toString();
 
     MarkupXmlStreamWriter writingVisitor = new MarkupXmlStreamWriter(namespace, text instanceof MarkupMultiline);