Skip to content

Commit

Permalink
Round trip fixes (#5)
Browse files Browse the repository at this point in the history
* Small bug fixes in scripts

* Getting closer to round tripping catalogs
  • Loading branch information
david-waltermire authored Jun 7, 2019
1 parent c2ae5df commit 183c7a3
Show file tree
Hide file tree
Showing 6 changed files with 108 additions and 35 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,5 @@ svrl
!content/**/*.md
!xml/**/*.md
!json/**/*.md
/roundtrip
__pycache__
2 changes: 1 addition & 1 deletion build/ci-cd/python/xmlComparison.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,5 +76,5 @@ def compareElements(e1, e2, xpath, position):
root2 = doc2.getroot()

if not compare(root1, root2):
print "compare failed"
print ("compare failed")
sys.exit(1)
2 changes: 1 addition & 1 deletion build/ci-cd/validate-round-trips.sh
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do

# compare the XML files to see if there is data loss
echo "${P_INFO}Checking XML->JSON->XML conversion for '${P_END}${file}${P_ERROR}'.${P_END}\n"
python ${OSCALDIR}/build/ci-cd/python/xmlComparison.py "$file" "$back_to_xml"
python ${OSCALDIR}/build/ci-cd/python/xmlComparison.py "$back_to_xml" "$file"
cmd_exitcode=$?
if [ $cmd_exitcode != 0 ]; then
echo "${P_ERROR}XML round-trip comparison failed.${P_END}"
Expand Down
49 changes: 39 additions & 10 deletions build/metaschema/json/md-oscal-converter.xsl
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,16 @@
<xsl:output indent="yes"/>

<xsl:param name="target-ns" as="xs:string?">http://csrc.nist.gov/ns/oscal/1.0</xsl:param>
<!-- TO DO: Test numbered and mixed lists; implement tables -->

<xsl:template name="xsl:initial-template" match="/">
<!--<xsl:copy-of select="$tag-replacements"/>-->
<!--<xsl:call-template name="parse">
<xsl:with-param name="str" select="string($examples)"/>
</xsl:call-template>-->
<xsl:for-each select="$line-example">
<xsl:apply-templates select="text()" mode="infer-inlines"/>
</xsl:for-each>
</xsl:template>

<!-- Markdown pseudoparser in XSLT -->

Expand Down Expand Up @@ -268,13 +277,13 @@
</xsl:template>

<xsl:template match="text()" mode="infer-inlines">
<xsl:variable name="markup" expand-text="true">
<xsl:variable name="markup">
<xsl:apply-templates select="$tag-replacements/m:rules">
<xsl:with-param name="original" tunnel="yes" as="text()" select="."/>
</xsl:apply-templates>
</xsl:variable>
<xsl:try select="parse-xml-fragment($markup)">
<xsl:catch expand-text="yes" select="."/>
<xsl:catch select="."/>
</xsl:try>
</xsl:template>

Expand Down Expand Up @@ -352,7 +361,9 @@
<!-- Note that text contents are regex notation for matching so * must be \* -->
<q>"<text/>"</q>

<img alt="!\[{{$text}}\]" src="\({{$text}}\)"/>
<img alt="!\[{{$text}}\]" src="\({{$text}}\)"/>
<insert param-id="\{{{{$nws}}\}}"/>

<a href="\[{{$text}}\]">\(<text/>\)</a>
<code>`<text/>`</code>
<strong>
Expand All @@ -371,7 +382,7 @@
<xsl:text>&lt;</xsl:text>
<xsl:value-of select="local-name()"/>
<!-- coercing the order to ensure correct formation of regegex -->
<xsl:apply-templates mode="#current" select="@href, @alt, @src"/>
<xsl:apply-templates mode="#current" select="@*"/>
<xsl:text>&gt;</xsl:text>

<xsl:apply-templates mode="#current" select="*"/>
Expand All @@ -389,10 +400,20 @@
<xsl:value-of select="replace(., '\{\$text\}', '(.*)?')"/>
</xsl:template>

<xsl:template match="@*[matches(., '\{\$nws\}')]" mode="write-match">
<!--<xsl:value-of select="."/>-->
<!--<xsl:value-of select="replace(., '\{\$nws\}', '(\S*)?')"/>-->
<xsl:value-of select="replace(., '\{\$nws\}', '\\s*(\\S+)?\\s*')"/>
</xsl:template>

<xsl:template match="m:text" mode="write-replace">
<xsl:text>$1</xsl:text>
</xsl:template>

<xsl:template match="m:insert/@param-id" mode="write-replace">
<xsl:text> param-id="$1"</xsl:text>
</xsl:template>

<xsl:template match="m:a/@href" mode="write-replace">
<xsl:text> href="$2"</xsl:text>
<!--<xsl:value-of select="replace(.,'\{\$insert\}','\$2')"/>-->
Expand All @@ -412,16 +433,24 @@
<xsl:text>(.*?)</xsl:text>
</xsl:template>

<!--<xsl:variable name="examples" xml:space="preserve">
<xsl:variable name="line-example" xml:space="preserve"> { insertion } </xsl:variable>

<xsl:variable name="examples" xml:space="preserve">
<p>**Markdown**</p>
<p>
## My test file!


{ ac-4.4_prm_2 }


Extra long x
y and z


Here's a text with a parameter insertion: { insert }

{insert-me}

And interesting.

Expand Down Expand Up @@ -465,6 +494,6 @@ And stuff.
<p>`code` may occasionally turn up `in the middle`.</p>
<p>Here's a ***really interesting*** markdown string.</p>
<p>Some paragraphs might have [links elsewhere](https://link.org).</p>
</xsl:variable>-->
</xsl:variable>

</xsl:stylesheet>
60 changes: 40 additions & 20 deletions build/metaschema/json/produce-json-converter.xsl
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
<!-- copying in templates from md-oscal-converter.xsl - everything but top-level parameters -->
<xsl:comment> 00000000000000000000000000000000000000000000000000000000000000 </xsl:comment>
<xsl:comment> Markdown converter</xsl:comment>
<xsl:copy-of select="document('md-oscal-converter.xsl')/xsl:*/(xsl:* except xsl:param)"/>
<xsl:copy-of select="document('md-oscal-converter.xsl')/xsl:*/(xsl:* except (xsl:param|xsl:variable[@name='examples']|xsl:template[@name='xsl:initial-template']))"/>
</XSLT:stylesheet>
</xsl:template>

Expand Down Expand Up @@ -83,10 +83,12 @@
</XSLT:attribute>
</XSLT:template>
</xsl:template>

<xsl:template match="define-flag/* | define-field/* | define-assembly/*"/>


<!-- 'any' keyword not handled -->
<xsl:template match="any" priority="10"/>

<xsl:template match="model | choice" priority="2">
<xsl:apply-templates/>
</xsl:template>
Expand All @@ -108,21 +110,35 @@
<xsl:for-each select="@address">
<XSLT:attribute name="{.}" select="../@key"/>
</xsl:for-each>
<XSLT:apply-templates mode="as-attribute"/>
<xsl:apply-templates/>
<XSLT:apply-templates mode="json2xml" select="*[@key='{ if (@as='mixed') then $markdown-value-label else $string-value-label }']"/>
<XSLT:apply-templates select="*" mode="as-attribute"/>
<xsl:apply-templates select="." mode="field-text"/>
</XSLT:element>
</XSLT:template>
<xsl:call-template name="drop-address"/>
</xsl:template>

<xsl:template match="define-field" mode="field-text">
<XSLT:apply-templates mode="json2xml"/>
</xsl:template>

<xsl:template match="define-field[@as='mixed']" mode="field-text">
<XSLT:variable name="markup">
<XSLT:apply-templates mode="infer-inlines"/>
</XSLT:variable>
<XSLT:apply-templates mode="cast-ns" select="$markup"/>
</xsl:template>

<xsl:template priority="2" match="define-field[exists(flag)]" mode="field-text">
<XSLT:apply-templates mode="json2xml" select="string[@key=('{$string-value-label}','{$markdown-value-label}')]"/>
</xsl:template>

<xsl:template match="define-assembly" expand-text="true">
<xsl:variable name="assembly-match" as="xs:string">*[@key='{@name}']{ @group-as/(' | *[@key=''' || . || ''']/*') }</xsl:variable>
<xsl:comment> 000 Handling assembly "{ @name }" 000 </xsl:comment>
<XSLT:template match="{$assembly-match}" priority="2" mode="json2xml">
<XSLT:element name="{@name}" namespace="{$target-namespace}">
<xsl:for-each select="@address">
<XSLT:attribute name="{.}" select="../@key"/>
<XSLT:attribute name="{.}" select="@key"/>
</xsl:for-each>
<XSLT:apply-templates mode="as-attribute"/>
<xsl:apply-templates/>
Expand Down Expand Up @@ -152,8 +168,8 @@
<XSLT:variable name="json-xml" select="unparsed-text($json-file) ! json-to-xml(.)"/>

<XSLT:template name="xsl:initial-template" match="/">
<XSLT:choose>
<XSLT:when test="exists($json-xml/map)">
<XSLT:choose>
<XSLT:when test="matches($json-file,'\S') and exists($json-xml/map)">
<XSLT:apply-templates select="$json-xml" mode="json2xml"/>
</XSLT:when>
<XSLT:otherwise>
Expand All @@ -172,17 +188,21 @@
<XSLT:apply-templates mode="#current"/>
</XSLT:template>

<XSLT:template match="array[@key='prose']/*" priority="5" mode="json2xml">
<XSLT:element name="p" namespace="{$target-namespace}">
<XSLT:variable name="text-contents" select="string-join(string,'&#xA;')"/>
<XSLT:call-template name="parse">
<XSLT:with-param name="str" select="$text-contents"/>
</XSLT:call-template>
</XSLT:element>
<XSLT:template match="array[@key='prose']" priority="5" mode="json2xml">
<XSLT:variable name="text-contents" select="string-join(string,'&#xA;')"/>
<XSLT:call-template name="parse">
<XSLT:with-param name="str" select="$text-contents"/>
</XSLT:call-template>
</XSLT:template>

<XSLT:template match="string[@key='{$markdown-value-label}']" mode="json2xml">
<XSLT:call-template name="parse"/>
<!-- drops all strings except those expecting the inline markup -->
<XSLT:template match="string" mode="handle-inlines"/>

<XSLT:template match="string[@key='{$markdown-value-label}']" mode="json2xml handle-inlines">
<XSLT:variable name="markup">
<XSLT:apply-templates mode="infer-inlines"/>
</XSLT:variable>
<XSLT:apply-templates mode="cast-ns" select="$markup"/>
</XSLT:template>

<XSLT:template match="string[@key='{$string-value-label}']" mode="json2xml">
Expand All @@ -191,9 +211,9 @@

<XSLT:template mode="as-attribute" match="*"/>

<XSLT:template mode="as-attribute" match="map">
<!--<XSLT:template mode="as-attribute" match="map">
<XSLT:apply-templates mode="#current"/>
</XSLT:template>
</XSLT:template>-->

<XSLT:template mode="as-attribute" match="string[@key='id']" priority="0.4">
<XSLT:attribute name="id">
Expand Down
28 changes: 25 additions & 3 deletions build/metaschema/xml/produce-xml-converter.xsl
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@
<!-- no special provision made for addressing by @id that happens at the other end -->
<xsl:template match="define-assembly">
<XSLT:template match="{@name}" mode="xml2json">
<map key="{@name}">
<map key="{@name}">
<xsl:for-each select="@address">
<xsl:attribute name="key">{@<xsl:value-of select="."/>}</xsl:attribute>
</xsl:for-each>
Expand All @@ -154,11 +154,14 @@
</XSLT:template>
</xsl:template>

<!-- dropping a flag that has been used as an address -->
<xsl:template match="flag[@name=../@address]"/>

<xsl:template match="flag">
<!-- no datatyping support yet -->
<XSLT:apply-templates mode="as-string" select="@{@name}"/>
</xsl:template>

<xsl:template match="model">
<xsl:apply-templates/>
</xsl:template>
Expand Down Expand Up @@ -251,13 +254,23 @@

<!--<XSLT:key name="parameters" match="param" use="@id"/>-->

<XSLT:template name="conditional-lf">
<XSLT:variable name="predecessor"
select="preceding-sibling::p | preceding-sibling::ul | preceding-sibling::ol | preceding-sibling::table | preceding-sibling::pre"/>
<XSLT:if test="exists($predecessor)">
<string/>
</XSLT:if>
</XSLT:template>

<XSLT:template mode="md" match="p | link | part/*">
<XSLT:call-template name="conditional-lf"/>
<string>
<XSLT:apply-templates mode="md"/>
</string>
</XSLT:template>

<XSLT:template mode="md" match="h1 | h2 | h3 | h4 | h5 | h6">
<XSLT:call-template name="conditional-lf"/>
<string>
<XSLT:apply-templates select="." mode="mark"/>
<XSLT:apply-templates mode="md"/>
Expand All @@ -272,6 +285,7 @@
<XSLT:template mode="mark" match="h6">###### </XSLT:template>

<XSLT:template mode="md" match="table">
<XSLT:call-template name="conditional-lf"/>
<XSLT:apply-templates select="*" mode="md"/>
</XSLT:template>

Expand All @@ -297,6 +311,7 @@
</XSLT:template>

<XSLT:template mode="md" priority="1" match="pre">
<XSLT:call-template name="conditional-lf"/>
<string>```</string>
<string>
<XSLT:apply-templates mode="md"/>
Expand All @@ -305,7 +320,7 @@
</XSLT:template>

<XSLT:template mode="md" priority="1" match="ul | ol">
<string/>
<XSLT:call-template name="conditional-lf"/>
<XSLT:apply-templates mode="md"/>
<string/>
</XSLT:template>
Expand Down Expand Up @@ -361,6 +376,13 @@
<XSLT:text>"</XSLT:text>
</XSLT:template>

<!-- <insert param-id="ac-1_prm_1"/> -->
<XSLT:template mode="md" match="insert">
<XSLT:text>{ </XSLT:text>
<XSLT:value-of select="@param-id"/>
<XSLT:text> }</XSLT:text>
</XSLT:template>

<XSLT:key name="element-by-id" match="*[exists(@id)]" use="@id"/>

<XSLT:template mode="md" match="a">
Expand Down

0 comments on commit 183c7a3

Please sign in to comment.