Skip to content

Commit

Permalink
Passing key into update/remove process xslt for iso 19139 to fix issu…
Browse files Browse the repository at this point in the history 
…e with updating/deleting resources with same url (#7431)

* Passing the index into remove process xslt for iso 19139

* Updated resource logic to use 2 new field called idx and hash and revert id back to the old method.  This will ensure backward compatibility
Also update xslt for add and remove thumbnail/onlines recources

* Add missing <xsl:copy> that was causing bugs.

* unit test fix

* Fix issues based on code review.
  - Fix issue with ordering of elements.
  - Fix issue with md5Hex not working

* Update schemas/iso19139/src/main/plugin/iso19139/process/thumbnail-remove.xsl

Co-authored-by: Jose García <[email protected]>

* Update schemas/iso19139/src/main/plugin/iso19139/process/onlinesrc-add.xsl

Co-authored-by: Jose García <[email protected]>

* Update schemas/iso19139/src/main/plugin/iso19139/process/onlinesrc-remove.xsl

Co-authored-by: Jose García <[email protected]>

* Update schemas/iso19139/src/main/plugin/iso19139/process/onlinesrc-remove.xsl

Co-authored-by: Ian <[email protected]>

* Update schemas/iso19139/src/main/plugin/iso19139/process/thumbnail-remove.xsl

Co-authored-by: Ian <[email protected]>

* Add hash and index to search indexing. Remove thumbnail hash and indexing as the backend file/store will get deleted.

* Fix unit test

* Remove unwanted method in XslUtils

* fix unit test

* code review comment

---------

Co-authored-by: Ian Allen <[email protected]>
Co-authored-by: Jose García <[email protected]>
  • Loading branch information
@wangf1122 @ianwallen @josegar74
3 people authored Mar 21, 2024
1 parent 54b949b commit 7d7db43
Show file tree
Hide file tree
Showing 8 changed files with 257 additions and 102 deletions.
108 changes: 62 additions & 46 deletions schemas/iso19139/src/main/plugin/iso19139/extract-relations.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
xmlns:gmx="http://www.isotc211.org/2005/gmx"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:util="java:org.fao.geonet.util.XslUtil"
xmlns:digestUtils="java:org.apache.commons.codec.digest.DigestUtils"
xmlns:exslt="http://exslt.org/common"
xmlns:gn-fn-rel="http://geonetwork-opensource.org/xsl/functions/relations"
version="2.0"
exclude-result-prefixes="#all">
Expand Down Expand Up @@ -85,7 +87,7 @@
</xsl:template>

<!-- Relation contained in the metadata record has to be returned
It could be document or thumbnails
It could be a document or thumbnails
-->
<xsl:template mode="relation"
match="metadata[gmd:MD_Metadata or *[contains(@gco:isoType, 'MD_Metadata')]]"
Expand All @@ -97,69 +99,83 @@
</xsl:call-template>
</xsl:variable>

<xsl:if test="count(*/descendant::*[name(.) = 'gmd:graphicOverview']/*) > 0">
<xsl:if test="count(*//gmd:graphicOverview) > 0">
<thumbnails>
<xsl:for-each select="*/descendant::*[name(.) = 'gmd:graphicOverview']/*">
<xsl:for-each select="*//gmd:graphicOverview">
<item>
<id>
<xsl:value-of select="gmd:fileName/gco:CharacterString"/>
<xsl:value-of select="gmd:MD_BrowseGraphic/gmd:fileName/gco:CharacterString"/>
</id>
<idx>
<xsl:value-of select="position()"/>
</idx>
<hash>
<xsl:value-of select="digestUtils:md5Hex(string(exslt:node-set(normalize-space(.))))"/>
</hash>
<url>
<xsl:apply-templates mode="get-iso19139-localized-string"
select="gmd:fileName"/>
select="gmd:MD_BrowseGraphic/gmd:fileName"/>
</url>
<title>
<xsl:apply-templates mode="get-iso19139-localized-string"
select="gmd:fileDescription"/>
select="gmd:MD_BrowseGraphic/gmd:fileDescription"/>
</title>
<type>thumbnail</type>
</item>
</xsl:for-each>
</thumbnails>
</xsl:if>

<xsl:if test="count(*/descendant::*[name(.) = 'gmd:onLine']/*[gmd:linkage/gmd:URL!='']) > 0">
<xsl:if test="count(*//gmd:MD_DigitalTransferOptions/gmd:onLine[gmd:CI_OnlineResource[gmd:linkage/gmd:URL!='']]) > 0">
<onlines>
<xsl:for-each select="*/descendant::*[name(.) = 'gmd:onLine']/*[gmd:linkage/gmd:URL!='']">
<item>
<xsl:variable name="langCode">
<xsl:value-of select="concat('#', upper-case(util:twoCharLangCode($lang, 'EN')))"/>
</xsl:variable>
<xsl:variable name="url" select="gmd:linkage/gmd:URL"/>
<id>
<xsl:value-of select="$url"/>
</id>
<title>
<xsl:apply-templates mode="get-iso19139-localized-string"
select="gmd:name"/>
</title>
<url>
<value lang="{$mainLanguage}">
<xsl:for-each select="*//gmd:MD_DigitalTransferOptions/gmd:onLine">
<xsl:if test="gmd:CI_OnlineResource[gmd:linkage/gmd:URL!='']">
<item>
<xsl:variable name="langCode">
<xsl:value-of select="concat('#', upper-case(util:twoCharLangCode($lang, 'EN')))"/>
</xsl:variable>
<xsl:variable name="url" select="gmd:CI_OnlineResource/gmd:linkage/gmd:URL"/>
<id>
<xsl:value-of select="$url"/>
</value>
</url>
<function>
<xsl:value-of select="gmd:function/*/@codeListValue"/>
</function>
<applicationProfile>
<xsl:value-of select="gmd:applicationProfile/*/text()"/>
</applicationProfile>
<description>
<xsl:apply-templates mode="get-iso19139-localized-string"
select="gmd:description"/>
</description>
<protocol>
<xsl:value-of select="gn-fn-rel:translate(gmd:protocol, $langCode)"/>
</protocol>
<mimeType>
<xsl:value-of select="if (*/gmx:MimeFileType)
then */gmx:MimeFileType/@type
else if (starts-with(gmd:protocol/gco:CharacterString, 'WWW:DOWNLOAD:'))
then replace(gmd:protocol/gco:CharacterString, 'WWW:DOWNLOAD:', '')
else ''"/>
</mimeType>
<type>onlinesrc</type>
</item>
</id>
<idx>
<xsl:value-of select="position()"/>
</idx>
<hash>
<xsl:value-of select="digestUtils:md5Hex(string(exslt:node-set(normalize-space(.))))"/>
</hash>
<title>
<xsl:apply-templates mode="get-iso19139-localized-string"
select="gmd:CI_OnlineResource/gmd:name"/>
</title>
<url>
<value lang="{$mainLanguage}">
<xsl:value-of select="$url"/>
</value>
</url>
<function>
<xsl:value-of select="gmd:CI_OnlineResource/gmd:function/*/@codeListValue"/>
</function>
<applicationProfile>
<xsl:value-of select="gmd:CI_OnlineResource/gmd:applicationProfile/*/text()"/>
</applicationProfile>
<description>
<xsl:apply-templates mode="get-iso19139-localized-string"
select="gmd:CI_OnlineResource/gmd:description"/>
</description>
<protocol>
<xsl:value-of select="gn-fn-rel:translate(gmd:CI_OnlineResource/gmd:protocol, $langCode)"/>
</protocol>
<mimeType>
<xsl:value-of select="if (gmd:CI_OnlineResource/*/gmx:MimeFileType)
then gmd:CI_OnlineResource/*/gmx:MimeFileType/@type
else if (starts-with(gmd:CI_OnlineResource/gmd:protocol/gco:CharacterString, 'WWW:DOWNLOAD:'))
then replace(gmd:CI_OnlineResource/gmd:protocol/gco:CharacterString, 'WWW:DOWNLOAD:', '')
else ''"/>
</mimeType>
<type>onlinesrc</type>
</item>
</xsl:if>
</xsl:for-each>
</onlines>
</xsl:if>
Expand Down
4 changes: 4 additions & 0 deletions schemas/iso19139/src/main/plugin/iso19139/index-fields/index.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:gn-fn-index="http://geonetwork-opensource.org/xsl/functions/index"
xmlns:index="java:org.fao.geonet.kernel.search.EsSearchManager"
xmlns:digestUtils="java:org.apache.commons.codec.digest.DigestUtils"
xmlns:exslt="http://exslt.org/common"
xmlns:util="java:org.fao.geonet.util.XslUtil"
xmlns:date-util="java:org.fao.geonet.utils.DateUtil"
xmlns:daobs="http://daobs.org"
Expand Down Expand Up @@ -1143,6 +1145,8 @@
<atomfeed><xsl:value-of select="gmd:linkage/gmd:URL"/></atomfeed>
</xsl:if>
<link type="object">{
"hash": "<xsl:value-of select="digestUtils:md5Hex(string(exslt:node-set(normalize-space(.))))"/>",
"idx": <xsl:value-of select="position()"/>,
"protocol":"<xsl:value-of select="util:escapeForJson((gmd:protocol/*/text())[1])"/>",
"mimeType":"<xsl:value-of select="if (*/gmx:MimeFileType)
then util:escapeForJson(*/gmx:MimeFileType/@type)
Expand Down
133 changes: 93 additions & 40 deletions schemas/iso19139/src/main/plugin/iso19139/process/onlinesrc-add.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,20 @@
<!--
Processing to insert or update an online resource element.
Insert is made in first transferOptions found.
Note: It assumes that it will be adding new items in
the first /gmd:distributionInfo
and first /gmd:MD_Distribution
and first /gmd:transferOptions
-->
<xsl:stylesheet xmlns:gmd="http://www.isotc211.org/2005/gmd"
xmlns:gco="http://www.isotc211.org/2005/gco"
xmlns:gmx="http://www.isotc211.org/2005/gmx"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:digestUtils="java:org.apache.commons.codec.digest.DigestUtils"
xmlns:exslt="http://exslt.org/common"
exclude-result-prefixes="#all"
version="2.0">

<!-- Main properties for the link.
Expand All @@ -52,17 +61,29 @@ Insert is made in first transferOptions found.
in this one. -->
<xsl:param name="extra_metadata_uuid"/>

<!-- Target element to update. The key is based on the concatenation
of URL+Protocol+Name -->
<xsl:param name="updateKey"/>
<!-- Target element to update.
updateKey is used to identify the resource name to be updated - it is for backwards compatibility. Will not be used if resourceHash is set.
The key is based on the concatenation of URL+Protocol+Name
resourceHash is hash value of the object to be removed which will ensure the correct value is removed. It will override the usage of updateKey
resourceIdx is the index location of the object to be removed - can be used when duplicate entries exists to ensure the correct one is removed.
-->

<xsl:param name="updateKey" select="''"/>
<xsl:param name="resourceHash" select="''"/>
<xsl:param name="resourceIdx" select="''"/>

<xsl:variable name="update_flag">
<xsl:value-of select="boolean($updateKey != '' or $resourceHash != '' or $resourceIdx != '')"/>
</xsl:variable>

<xsl:variable name="mainLang">
<xsl:value-of
select="(gmd:MD_Metadata|*[@gco:isoType='gmd:MD_Metadata'])/gmd:language/gmd:LanguageCode/@codeListValue"/>
</xsl:variable>

<xsl:template match="gmd:MD_Metadata|*[@gco:isoType='gmd:MD_Metadata']">
<!-- Add new gmd:onLine and consider cases where parent elements don't exist -->
<!-- <gmd:distributionInfo> does not exist-->
<xsl:template match="gmd:MD_Metadata[not(gmd:distributionInfo) and $update_flag = false()]|*[@gco:isoType='gmd:MD_Metadata' and not(gmd:distributionInfo) and $update_flag = false()]">
<xsl:copy>
<xsl:apply-templates select="@*"/>
<xsl:apply-templates
Expand All @@ -86,41 +107,15 @@ Insert is made in first transferOptions found.

<gmd:distributionInfo>
<gmd:MD_Distribution>
<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/gmd:distributionFormat"/>
<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/gmd:distributor"/>
<gmd:transferOptions>
<gmd:MD_DigitalTransferOptions>
<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/
gmd:transferOptions[1]/gmd:MD_DigitalTransferOptions/gmd:unitsOfDistribution"/>
<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/
gmd:transferOptions[1]/gmd:MD_DigitalTransferOptions/gmd:transferSize"/>
<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/
gmd:transferOptions[1]/gmd:MD_DigitalTransferOptions/gmd:onLine"/>


<xsl:if test="$updateKey = ''">
<xsl:call-template name="createOnlineSrc"/>
</xsl:if>

<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/
gmd:transferOptions[1]/gmd:MD_DigitalTransferOptions/gmd:offLine"/>
<xsl:call-template name="createOnlineSrc"/>
</gmd:MD_DigitalTransferOptions>
</gmd:transferOptions>


<xsl:apply-templates
select="gmd:distributionInfo/gmd:MD_Distribution/
gmd:transferOptions[position() > 1]"/>

</gmd:MD_Distribution>
</gmd:distributionInfo>


<xsl:apply-templates
select="gmd:dataQualityInfo|
gmd:portrayalCatalogueInfo|
Expand All @@ -135,14 +130,72 @@ Insert is made in first transferOptions found.
</xsl:copy>
</xsl:template>

<!-- <gmd:MD_Distribution> does not exist-->
<xsl:template match="*/gmd:distributionInfo[not(gmd:MD_Distribution) and $update_flag = false() and position() = 1]">
<xsl:copy>
<xsl:apply-templates select="node()|@*"/>
<gmd:MD_Distribution>
<gmd:transferOptions>
<gmd:MD_DigitalTransferOptions>
<xsl:call-template name="createOnlineSrc"/>
</gmd:MD_DigitalTransferOptions>
</gmd:transferOptions>
</gmd:MD_Distribution>
</xsl:copy>
</xsl:template>

<!-- <gmd:transferOptions> does not exist-->
<xsl:template match="*/gmd:distributionInfo[1]/gmd:MD_Distribution[not(gmd:transferOptions) and $update_flag = false() and position() = 1]">
<xsl:copy>
<xsl:apply-templates select="node()|@*"/>
<gmd:transferOptions>
<gmd:MD_DigitalTransferOptions>
<xsl:call-template name="createOnlineSrc"/>
</gmd:MD_DigitalTransferOptions>
</gmd:transferOptions>
</xsl:copy>
</xsl:template>

<!-- <gmd:MD_DigitalTransferOptions> does not exist-->
<xsl:template match="*/gmd:distributionInfo[1]/gmd:MD_Distribution[1]/gmd:transferOptions[not(gmd:MD_DigitalTransferOptions) and $update_flag = false() and position() = 1]">
<xsl:copy>
<xsl:apply-templates select="node()|@*"/>
<gmd:MD_DigitalTransferOptions>
<xsl:call-template name="createOnlineSrc"/>
</gmd:MD_DigitalTransferOptions>
</xsl:copy>
</xsl:template>

<!-- Add new gmd:gmd:onLine-->
<xsl:template match="*/gmd:distributionInfo[1]/gmd:MD_Distribution[1]/gmd:transferOptions[1]/gmd:MD_DigitalTransferOptions[$update_flag = false() and position() = 1]">
<xsl:copy>
<xsl:apply-templates select="@*"/>
<xsl:apply-templates
select="gmd:unitsOfDistribution|
gmd:transferSize|
gmd:onLine"/>

<xsl:call-template name="createOnlineSrc"/>

<xsl:apply-templates select="gmd:offLine"/>
</xsl:copy>
</xsl:template>

<!-- Updating the link matching the update key. -->
<xsl:template match="gmd:onLine[$updateKey != '' and
normalize-space($updateKey) = concat(
gmd:CI_OnlineResource/gmd:linkage/gmd:URL,
gmd:CI_OnlineResource/gmd:protocol/*,
gmd:CI_OnlineResource/gmd:name/gco:CharacterString)
]">
<!-- End of inserting gmd:onLine -->


<!-- Updating the gmd:onLine based on update parameters -->
<!-- Note: first part of the match needs to match the xsl:for-each select from extract-relations.xsl in order to get the position() to match -->
<!-- The unique identifier is marked with resourceIdx which is the position index and resourceHash which is hash code of the current node (combination of url, resource name, and description) -->
<xsl:template
match="*//gmd:MD_DigitalTransferOptions/gmd:onLine
[gmd:CI_OnlineResource[gmd:linkage/gmd:URL!=''] and ($resourceIdx = '' or position() = xs:integer($resourceIdx))]
[($resourceHash != '' or ($updateKey != '' and normalize-space($updateKey) = concat(
gmd:CI_OnlineResource/gmd:linkage/gmd:URL,
gmd:CI_OnlineResource/gmd:protocol/*,
gmd:CI_OnlineResource/gmd:name/gco:CharacterString)))
and ($resourceHash = '' or digestUtils:md5Hex(string(exslt:node-set(normalize-space(.)))) = $resourceHash)]"
priority="2">
<xsl:call-template name="createOnlineSrc"/>
</xsl:template>

Expand Down Expand Up @@ -170,7 +223,7 @@ Insert is made in first transferOptions found.

<xsl:if test="$url">
<!-- In case the protocol is an OGC protocol
the name parameter may contains a list of layers
the name parameter may contain a list of layers
separated by comma.
In that case on one online element is added per
layer/featureType.
Expand Down
Loading

0 comments on commit 7d7db43

Please sign in to comment.