Skip to content

Commit

Permalink
DHFPROD-6955: Update all merge tests that use XML merge options
Browse files Browse the repository at this point in the history 
more
  • Loading branch information
@fsnow @SameeraPriyathamTadikonda
fsnow authored and SameeraPriyathamTadikonda committed Feb 26, 2021
1 parent fb01bb4 commit b6752d5
Show file tree
Hide file tree
Showing 81 changed files with 2,090 additions and 1,043 deletions.
4 changes: 2 additions & 2 deletions ...c-data-hub/src/main/resources/ml-modules/root/com.marklogic.smart-mastering/collector.xqy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ declare option xdmp:mapping "false";
declare function plugin:collect(
$options as map:map) as xs:string*
{
let $merge-options := merging:get-options($options => map:get("options"), $const:FORMAT-XML)
let $matching-options := matcher:get-options(fn:string($merge-options/merging:match-options), $const:FORMAT-XML)
let $merge-options := merging:get-JSON-options($options => map:get("options"))
let $matching-options := matcher:get-options(fn:string($merge-options => map:get("matchOptions"), $const:FORMAT-XML)
return
cts:uris((), (), cts:and-query((
coll:content-collections($matching-options) ! cts:collection-query(.),
Expand Down
84 changes: 47 additions & 37 deletions ...ata-hub/src/main/resources/ml-modules/root/com.marklogic.smart-mastering/impl/process.xqy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ declare option xdmp:mapping "false";
declare function proc-impl:process-match-and-merge($input as item()*)
as item()*
{
let $merging-options := merging:get-options($const:FORMAT-XML)
let $merging-options := merging:get-JSON-options()
return
if (fn:exists($merging-options)) then
for $merge-options in $merging-options
Expand All @@ -65,14 +65,19 @@ declare function proc-impl:process-match-and-merge(
$filter-query as cts:query)
as item()*
{
let $all-options := xdmp:invoke-function(function() {
let $merge-options := merging:get-options($option-name, $const:FORMAT-XML)
let $match-options := matcher:get-options(fn:string($merge-options/merging:match-options), $const:FORMAT-XML)
return
map:map()
=> map:with("merge-options", $merge-options)
=> map:with("match-options", $match-options)
}, map:entry("update", "false"))
let $all-options :=
xdmp:invoke-function(
function() {
let $merge-options := merging:get-JSON-options($option-name)
let $match-options-name := $merge-options/options/matchOptions/fn:string()
let $match-options := matcher:get-options($match-options-name, $const:FORMAT-XML)
return
map:map()
=> map:with("merge-options", $merge-options)
=> map:with("match-options", $match-options)
},
map:entry("update", "false")
)
return
proc-impl:process-match-and-merge-with-options-save(
$input,
Expand Down Expand Up @@ -778,39 +783,44 @@ declare function proc-impl:build-content-objects-from-match-summary(
else ()
let $custom-action-options :=
let $match-results := $custom-action => map:get("matchResults")
let $custom-action-options := map:new(
if (fn:ends-with(xdmp:function-module($action-func), "js")) then (
map:entry("match-results",$match-results),
map:entry("merge-options",
typeswitch($merge-options)
case object-node() return
xdmp:from-json($merge-options)
case element() return
merge-impl:options-to-json($merge-options)
default return
$merge-options
)
) else (
map:entry("match-results",proc-impl:matches-to-xml($match-results)),
map:entry("merge-options",
typeswitch($merge-options)
case element() return
$merge-options
case object-node() return
if ($is-hub-central-format) then
let $custom-action-options :=
map:new(
if (fn:ends-with(xdmp:function-module($action-func), "js")) then (
map:entry("match-results",$match-results),
map:entry("merge-options",
typeswitch($merge-options)
case object-node() return
xdmp:from-json($merge-options)
case element() return
merge-impl:options-to-json($merge-options)
default return
$merge-options
else
merge-impl:options-from-json($merge-options)
default return
let $merge-node := xdmp:to-json($merge-options)
return
)
)
else (
map:entry("match-results", proc-impl:matches-to-xml($match-results)),
map:entry("merge-options",
typeswitch($merge-options)
case element() return
(fn:error((), "Should never hit this case"), $merge-options)
case object-node() return
if ($is-hub-central-format) then
$merge-node
$merge-options
else
merge-impl:options-from-json($merge-node)
(: FSnow 2/24/2021: one of the only places where we want to convert options from
JSON to XML is here, where we are calling a custom XQuery merge action :)
merge-impl:options-from-json($merge-options)
default return
let $merge-node := xdmp:to-json($merge-options)
return
if ($is-hub-central-format) then
$merge-node
else
merge-impl:options-from-json($merge-node)
(: $merge-node :)
)
)
)
)
return $custom-action-options
return
xdmp:apply(
Expand Down
43 changes: 42 additions & 1 deletion ...gic-data-hub/src/main/resources/ml-modules/root/com.marklogic.smart-mastering/merging.xqy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,17 @@ declare function merging:get-option-names($format as xs:string)
};

(:
: Return all previously save merge options.
: Return all previously saved merge options.
:
: @return An array of JSON options objects.
:)
declare function merging:get-JSON-options()
{
impl:get-JSON-options()
};

(:
: Return all previously saved merge options.
:
: @param $format either $const:FORMAT-XML or $const:FORMAT-JSON
: @return A sequence of elements with the options or a JSON array with option objects.
Expand Down Expand Up @@ -174,3 +184,34 @@ declare function merging:save-options(
{
impl:save-options($name, $options)
};

(:
: Retrieve a named set of merge options from the DB, stored as JSON, returned as JSON.
:
: @param $options-name the name under which the options were saved
: @return A JSON object
:)
declare function merging:get-JSON-options($options-name as xs:string)
{
impl:get-JSON-options($options-name)
};

(: call from JavaScript as getJsonOptions() :)
declare function merging:get-json-options($options-name as xs:string)
{
impl:get-JSON-options($options-name)
};

(:
: Save a set of merging options to the database as JSON.
: @param $name the name under which the options are to be stored
: @param $options the JSON options.
: @return ()
:)
declare function merging:save-JSON-options(
$name as xs:string,
$options as node()
) as empty-sequence()
{
impl:save-JSON-options($name, $options)
};
119 changes: 91 additions & 28 deletions .../resources/ml-modules/root/com.marklogic.smart-mastering/survivorship/merging/options.xqy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,34 @@ declare variable $event-names-xml as xs:QName+ := (xs:QName("merging:on-merge"),
: Functions related to merge options.
:~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~:)

declare function merge-impl:get-JSON-options() as object-node()*
{
let $query :=
cts:and-query((
cts:collection-query($const:OPTIONS-COLL),
(: In future version, remove mdm-merge collection from query
Currently part of the query to avoid breaking changes.
:)
cts:collection-query(('mdm-merge', $const:MERGE-OPTIONS-COLL))
))
let $options := cts:search(doc(), $query, "unfiltered")
return $options/object-node()
(:
return array-node { $options ! xdmp:from-json(.) }
:)
};

declare function merge-impl:get-options($format as xs:string)
{
let $options :=
cts:search(fn:collection(), cts:and-query((
cts:collection-query($const:OPTIONS-COLL),
(: In future version, remove mdm-merge collection from query
Currently part of the query to avoid breaking changes.
:)
cts:collection-query(('mdm-merge',$const:MERGE-OPTIONS-COLL))
)))/merging:options
let $query :=
cts:and-query((
cts:collection-query($const:OPTIONS-COLL),
(: In future version, remove mdm-merge collection from query
Currently part of the query to avoid breaking changes.
:)
cts:collection-query(('mdm-merge', $const:MERGE-OPTIONS-COLL))
))
let $options := cts:search(fn:collection(), $query)/merging:options
return
if ($format eq $const:FORMAT-XML) then
$options
Expand All @@ -63,7 +81,9 @@ declare function merge-impl:get-options($format as xs:string)

declare function merge-impl:get-options($options-name, $format as xs:string)
{
let $options := fn:doc($MERGING-OPTIONS-DIR||$options-name||".xml")/merging:options
let $options-uri := $MERGING-OPTIONS-DIR || $options-name || ".xml"
let $options := fn:doc($options-uri)/merging:options
let $log := xdmp:log($options)
return
if ($format eq $const:FORMAT-XML) then
$options
Expand All @@ -73,11 +93,22 @@ declare function merge-impl:get-options($options-name, $format as xs:string)
httputils:throw-bad-request(xs:QName("SM-INVALID-FORMAT"), "merge-impl:get-options called with invalid format " || $format)
};

declare function merge-impl:get-JSON-options($options-name as xs:string) as object-node()?
{
let $log := if (xdmp:trace-enabled($const:TRACE-MERGE-RESULTS)) then xdmp:trace($const:TRACE-MERGE-RESULTS, "get-JSON-options: " || $options-name) else ()
let $options-uri := $MERGING-OPTIONS-DIR || $options-name || ".json"
let $options := fn:doc($options-uri)/object-node()
let $log := if (xdmp:trace-enabled($const:TRACE-MERGE-RESULTS)) then xdmp:trace($const:TRACE-MERGE-RESULTS, $options) else ()
return $options (: ! xdmp:from-json(.) :)
};

declare function merge-impl:save-options(
$name as xs:string,
$options as node()
) as empty-sequence()
{
let $_ := fn:error((), "Don't call this")

let $options :=
if ($options instance of document-node()) then
$options/node()
Expand All @@ -90,13 +121,26 @@ declare function merge-impl:save-options(
$options
return
xdmp:document-insert(
$MERGING-OPTIONS-DIR||$name||".xml",
$MERGING-OPTIONS-DIR || $name || ".xml",
$options,
config:get-default-data-hub-permissions(),
($const:OPTIONS-COLL, $const:MERGE-OPTIONS-COLL)
)
};

declare function merge-impl:save-JSON-options(
$name as xs:string,
$options as node())
as empty-sequence()
{
xdmp:document-insert(
$MERGING-OPTIONS-DIR || $name || ".json",
$options,
config:get-default-data-hub-permissions(),
($const:OPTIONS-COLL, $const:MERGE-OPTIONS-COLL)
)
};

declare variable $options-json-config := merge-impl:_options-json-config();

(: Removes whitespace nodes to keep the output json from options-to-json clean :)
Expand All @@ -119,7 +163,9 @@ declare function merge-impl:remove-whitespace($xml)
(:
: Convert merge options from XML to JSON.
:)
declare function merge-impl:options-to-json($options-xml as element(merging:options))
declare function merge-impl:options-to-json(
$options-xml as element(merging:options))
as node()?
{
if (fn:exists($options-xml)) then
xdmp:to-json(
Expand Down Expand Up @@ -296,7 +342,15 @@ declare function merge-impl:build-namespace-map($source as element()?)
declare function merge-impl:options-from-json($options-json as item())
as element(merging:options)
{
let $options-json := if ($options-json instance of json:object) then xdmp:to-json($options-json)/object-node() else $options-json
(: let $_ := fn:error((), "Don't call this") :)

let $options-json :=
if ($options-json instance of json:object) then
xdmp:to-json($options-json)/object-node()
else if ($options-json instance of document-node()) then
$options-json/object-node()
else
$options-json
return
<options xmlns="http://marklogic.com/smart-mastering/merging">
{
Expand Down Expand Up @@ -415,11 +469,12 @@ declare private function merge-impl:construct-merging-element($options-json as o
}
};

declare variable $merge-spec-json-config := json:config("custom")
=> map:with("element-namespace", "http://marklogic.com/smart-mastering/merging")
=> map:with("camel-case", fn:true())
=> map:with("whitespace", "ignore")
=> map:with("attribute-names", ("default", "name", "weight", "strategy", "propertyName", "algorithmRef", "maxValues", "maxSources", "documentUri"));
declare variable $merge-spec-json-config :=
json:config("custom")
=> map:with("element-namespace", "http://marklogic.com/smart-mastering/merging")
=> map:with("camel-case", fn:true())
=> map:with("whitespace", "ignore")
=> map:with("attribute-names", ("default", "name", "weight", "strategy", "propertyName", "algorithmRef", "maxValues", "maxSources", "documentUri"));

declare function merge-impl:propertyspec-to-xml($merging-objects as item()*, $type as xs:QName) as element()* {
let $normalized-merging-objects :=
Expand Down Expand Up @@ -507,17 +562,25 @@ declare function merge-impl:_options-json-config()
declare function merge-impl:get-option-names($format as xs:string)
{
if ($format eq $const:FORMAT-XML) then
let $options := cts:uris('', (), cts:and-query((
cts:collection-query($const:OPTIONS-COLL),
(: In future version, remove mdm-merge collection from query
Currently part of the query to avoid breaking changes.
:)
cts:collection-query(('mdm-merge',$const:MERGE-OPTIONS-COLL))
)))
let $option-names := $options ! fn:replace(
fn:replace(., $MERGING-OPTIONS-DIR, ""),
"\.xml$", ""
)
let $options-uris :=
cts:uris('', (),
cts:and-query((
cts:collection-query($const:OPTIONS-COLL),
(: In future version, remove mdm-merge collection from query
Currently part of the query to avoid breaking changes.
:)
cts:collection-query(('mdm-merge', $const:MERGE-OPTIONS-COLL))
))
)
let $option-names :=
$options-uris !
fn:replace(
fn:replace(
fn:replace(., $MERGING-OPTIONS-DIR, ""),
"\.xml$", ""
),
"\.json$", ""
)
return
element merging:options {
for $name in $option-names
Expand Down
4 changes: 2 additions & 2 deletions .../test/suites/data-hub/5/smart-mastering/match-and-merge/asymmetric-match-merge-notify.xqy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ for $uri in cts:uris()
return xdmp:log($uri)
,

merging:save-options($lib:MERGE-OPTIONS-NAME, test:get-test-file("merge-options.xml"))
merging:save-JSON-options($lib:MERGE-OPTIONS-NAME, test:get-test-file("merge-options.json"))
,
for $uri in ($lib:URI-DOB1, $lib:URI-DOB2)
let $doc := test:get-test-file(map:get($lib:TEST-DATA, $uri))
Expand Down Expand Up @@ -79,7 +79,7 @@ let $actual :=
xdmp:invoke-function(
function() {
let $match-options := matcher:get-options($lib:MATCH-OPTIONS-CUST-DOB-NAME, $const:FORMAT-XML)
let $merge-options := merging:get-options($lib:MERGE-OPTIONS-NAME, $const:FORMAT-XML)
let $merge-options := merging:get-JSON-options($lib:MERGE-OPTIONS-NAME)
return process:process-match-and-merge-with-options(($lib:URI-DOB2, $lib:URI-DOB1), $merge-options, $match-options, cts:true-query(), fn:false())
},
$lib:INVOKE_OPTIONS
Expand Down
Loading

0 comments on commit b6752d5

Please sign in to comment.