diff --git a/src/ftr/extraction/ig/core.clj b/src/ftr/extraction/ig/core.clj index d07532d..73824ef 100644 --- a/src/ftr/extraction/ig/core.clj +++ b/src/ftr/extraction/ig/core.clj @@ -231,7 +231,7 @@ concepts (transduce (comp (mapcat :fhir/concepts) (map (fn [concept] {:path [(:system concept) - (:id concept)] + (:code concept)] :value concept}))) (completing (fn [acc {:keys [path value]}] diff --git a/src/ftr/extraction/ig/value_set_expand.clj b/src/ftr/extraction/ig/value_set_expand.clj index 1712609..4a9d2bf 100644 --- a/src/ftr/extraction/ig/value_set_expand.clj +++ b/src/ftr/extraction/ig/value_set_expand.clj @@ -1,15 +1,11 @@ (ns ftr.extraction.ig.value-set-expand (:require + [zen.utils] + [clojure.set :as set] [clojure.string :as str])) -(defn vs-compose-system-fn [ztx value-set system version] - (when (some? system) - (fn vs-compose-system [concept] - (= system (:system concept))))) - - -(defn vs-compose-concept-fn [ztx value-set system version concepts] +(defn vs-compose-concept-fn [value-set system version concepts] (when (seq concepts) (let [concept-codes (into #{} (map :code) concepts)] (fn vs-compose-concept [concept] @@ -18,66 +14,66 @@ (defmulti filter-op - (fn [_ztx _value-set _system _version filter] + (fn [_value-set _system _version filter] (:op filter))) -(defmethod filter-op :default [_ztx _value-set _system _version _filter] +(defmethod filter-op :default [_value-set _system _version _filter] (constantly false)) -(defmethod filter-op "=" [_ztx _value-set _system _version filter] +(defmethod filter-op "=" [_value-set _system _version filter] (fn eq-op [concept] (= (get (:property concept) (:property filter)) (:value filter)))) -(defmethod filter-op "in" [_ztx _value-set _system _version filter] +(defmethod filter-op "in" [_value-set _system _version filter] (fn in-op [concept] (get (into #{} (map str/trim) (str/split (or (:value filter) "") #",")) (get (:property concept) (:property filter))))) -(defmethod filter-op "not-in" [_ztx _value-set _system _version filter] +(defmethod filter-op "not-in" [_value-set _system _version filter] (fn not-in-op [concept] (not (get (into #{} (map str/trim) (str/split (or (:value filter) "") #",")) (get (:property concept) (:property filter)))))) -(defmethod filter-op "exists" [_ztx _value-set _system _version filter] +(defmethod filter-op "exists" [_value-set _system _version filter] (if (= "false" (some-> (:value filter) str/lower-case str/trim)) (fn not-exists-op [concept] (nil? (get (:property concept) (:property filter)))) (fn exists-op [concept] (some? (get (:property concept) (:property filter)))))) -(defmethod filter-op "is-a" [_ztx _value-set _system _version filter] +(defmethod filter-op "is-a" [_value-set _system _version filter] (fn is-a-op [concept] (or (= (:code concept) (:value filter)) (contains? (:zen.fhir/parents concept) (:value filter))))) -(defmethod filter-op "descendent-of" [_ztx _value-set _system _version filter] +(defmethod filter-op "descendent-of" [_value-set _system _version filter] (fn descendatnd-of-op [concept] (contains? (set (:hierarchy concept)) (:value filter)))) -(defmethod filter-op "is-not-a" [_ztx _value-set _system _version filter] +(defmethod filter-op "is-not-a" [_value-set _system _version filter] (fn is-not-a-op [concept] ;; TODO: not sure this is correct impl by spec (and (not (contains? (set (:hierarchy concept)) (:value filter))) (not= (:code concept) (:value filter))))) -(defmethod filter-op "regex" [_ztx _value-set _system _version filter] +(defmethod filter-op "regex" [_value-set _system _version filter] (fn regex-op [concept] (when-let [prop (get (:property concept) (:property filter))] (re-matches (re-pattern (:value filter)) (str prop))))) -(defn vs-compose-filter-fn [ztx value-set system version filters] +(defn vs-compose-filter-fn [value-set system version filters] (when (seq filters) (let [filter-fn (->> filters - (map #(filter-op ztx value-set system version %)) + (map #(filter-op value-set system version %)) (apply every-pred))] (fn compose-filter [concept] (and (= system (:system concept)) @@ -87,133 +83,349 @@ (declare compose) -(defn vs-compose-value-set-fn [ztx value-set value-set-urls] - (when-let [composes - (some->> value-set-urls - (keep #(when-let [vs (get-in @ztx [:fhir/inter "ValueSet" %])] - (compose ztx vs))) - not-empty)] - (when-let [check-fn - (some->> composes - (keep :check-concept-fn) - not-empty - (apply every-pred))] - {:systems (mapcat :systems composes) - :check-fn check-fn}))) - - -(defn check-if-concept-is-in-this-compose-el-fn [ztx value-set compose-el] - (let [code-system-pred (or (vs-compose-concept-fn ztx value-set - (:system compose-el) - (:version compose-el) - (:concept compose-el)) - (vs-compose-filter-fn ztx value-set - (:system compose-el) - (:version compose-el) - (:filter compose-el)) - (vs-compose-system-fn ztx value-set - (:system compose-el) - (:version compose-el))) - - {value-set-systems :systems, value-set-pred :check-fn} - (vs-compose-value-set-fn ztx value-set (:valueSet compose-el)) - - check-fn (some->> [code-system-pred value-set-pred] - (remove nil?) - not-empty - (apply every-pred))] - (when check-fn - {:systems (conj value-set-systems (:system compose-el)) - :check-fn check-fn}))) - - -(defn update-fhir-vs-expansion-index [ztx vs concept-identity-keys] #_"TODO: support recursive expansion :contains" - (let [expansion-contains (get-in vs [:expansion :contains]) - full-expansion? (and (= (count expansion-contains) (get-in vs [:expansion :total])) - (empty? (get-in vs [:expansion :parameter]))) - #_#_concepts (into #{} (map #(select-keys % concept-identity-keys)) expansion-contains) - - concepts-index (reduce (fn [acc concept] - (assoc! acc (:system concept) - (if-let [sys-idx (get acc (:system concept))] - (conj! sys-idx (:code concept)) - (transient #{(:code concept)})))) - (transient {}) - expansion-contains)] - (swap! ztx assoc-in [:fhir/vs-expansion-index (:url vs)] {#_#_:concepts concepts - :concepts-transient-index concepts-index - :full? full-expansion?}))) - - -(defn compose [ztx vs] - (let [concept-identity-keys [:code :system] - _ (when (and (get-in vs [:expansion :contains]) - (not (get-in @ztx [:fhir/vs-expansion-index (:url vs)]))) - (update-fhir-vs-expansion-index ztx vs concept-identity-keys)) - vs-url (:url vs) - full-expansion? (get-in @ztx [:fhir/vs-expansion-index vs-url :full?]) - vs-concepts-index (-> @ztx - (get :fhir/vs-expansion-index) - (get vs-url) - (get :concepts-transient-index)) - expansion-fn (fn expansion-fn [{concept :zen.fhir/resource}] - (-> vs-concepts-index (get (:system concept)) (get (:code concept)))) - includes (some->> (get-in vs [:compose :include]) - (keep (partial check-if-concept-is-in-this-compose-el-fn ztx vs)) - not-empty) - include-fn (or (some->> includes - (map :check-fn) - (apply some-fn)) - (constantly false)) - - excludes (some->> (get-in vs [:compose :exclude]) - (keep (partial check-if-concept-is-in-this-compose-el-fn ztx vs)) - not-empty) - exclude-fn (or (some->> excludes - (map :check-fn) - (apply some-fn) - complement) - (constantly true)) - - include-and-exclude-fn (every-pred include-fn exclude-fn) - - check-concept-fn (if full-expansion? - expansion-fn - (some-fn expansion-fn include-and-exclude-fn)) - - includes-and-excludes (concat includes excludes) - - systems (into #{} - (mapcat (comp not-empty :systems)) - includes-and-excludes)] - {:systems (not-empty systems) - :check-concept-fn check-concept-fn})) - - -(defn denormalize-into-concepts [ztx valuesets concepts-map] - (reduce - (fn reduce-valuesets [concepts-acc vs] - (let [{systems :systems - concept-in-vs? :check-concept-fn} - (compose ztx vs)] - (reduce - (fn reduce-codesystems [acc [system concepts]] - (reduce - (fn reduce-concepts [acc [concept-id concept]] - (if (concept-in-vs? concept) - (update-in acc [system concept-id :valueset] - (fnil conj #{}) - (:url vs)) - acc)) - acc - concepts)) - concepts-acc - (select-keys concepts-acc systems)))) - concepts-map +(defn check-concept-in-compose-el-fn [value-set compose-el] + (let [code-system-pred + (or (vs-compose-concept-fn value-set + (:system compose-el) + (:version compose-el) + (:concept compose-el)) + (vs-compose-filter-fn value-set + (:system compose-el) + (:version compose-el) + (:filter compose-el)))] + (some-> {:check-fn code-system-pred + :system (:system compose-el) + :depends-on (:valueSet compose-el)} + zen.utils/strip-nils + not-empty + (assoc :vs-url (:url value-set))))) + + +(defn vs-expansion-index [vs] #_"TODO: support recursive expansion :contains" + (when (get-in vs [:expansion :contains]) + (let [expansion-contains (get-in vs [:expansion :contains]) + full-expansion? (and (= (count expansion-contains) (get-in vs [:expansion :total])) + (empty? (get-in vs [:expansion :parameter]))) + concepts-index (persistent! + (reduce (fn [acc concept] + (assoc! acc (:system concept) + (if-let [sys-idx (get acc (:system concept))] + (conj sys-idx (:code concept)) + #{(:code concept)}))) + (transient {}) + expansion-contains))] + {:concepts-index concepts-index + :full? full-expansion?}))) + + +(defn compose [vs] + (let [{full-expansion? :full? + vs-concepts-index :concepts-index} + (vs-expansion-index vs) + + includes (some->> (get-in vs [:compose :include]) + (keep (partial check-concept-in-compose-el-fn vs)) + not-empty) + + excludes (some->> (get-in vs [:compose :exclude]) + (keep (partial check-concept-in-compose-el-fn vs)) + not-empty) + + #_#_systems (into #{} + (mapcat (comp not-empty :system)) + (concat includes excludes))] + + {#_#_:systems (not-empty systems) + :vs-url (:url vs) + :full-expansion? full-expansion? + :expansion-index vs-concepts-index + :includes includes + :excludes excludes})) + + +(defn push-compose-el-to-vs-queue [vs-queue el-type {:keys [depends-on system check-fn]}] + (let [has-dependencies? (seq depends-on) + allow-any-concept? (nil? check-fn) + has-concept-check? (some? check-fn) + any-system? (nil? system) + depends-on (vec depends-on) + + _ (when (not has-concept-check?) + (assert (or (some? system) has-dependencies?) + "check fn may be missing only when depending on another value set or there's a system")) + _ (assert (or (not any-system?) has-dependencies?) + "system may be missing only when depending on another value set") + + queue-path (concat [el-type] + (if any-system? + [:any-system] + [:systems system]) + (if allow-any-concept? + [depends-on :allow-any-concept] + [depends-on :pred-fns]))] + (cond-> vs-queue + allow-any-concept? + (assoc-in queue-path true) + + has-concept-check? + (update-in queue-path conj check-fn) + + has-dependencies? + (update :deps (fnil into #{}) depends-on)))) + + +(defn push-compose-els-to-vs-queue [vs-queue el-type els] + (reduce #(push-compose-el-to-vs-queue %1 el-type %2) + vs-queue + els)) + + +(defn push-entries-to-vs-queue [vs-queue {:keys [full-expansion? expansion-index includes excludes]}] + (-> vs-queue + (push-compose-els-to-vs-queue :include includes) + (push-compose-els-to-vs-queue :exclude excludes) + (zen.utils/assoc-some :full-expansion? full-expansion? + :expansion-index expansion-index))) + + +(defn pop-entry-from-vs-queue [acc vs-url] + (update acc :vs-queue dissoc vs-url)) + + +(defn push-concept-into-vs-idx [vs-idx concept] + (update vs-idx + (:system concept) + (fnil conj #{}) + (:code concept))) + + +(defn get-acc-key [any-system? mode] + (case [any-system? mode] + [true :include] :any-sys-include-acc + [false :include] :include-acc + [true :exclude] :any-sys-exclude-acc + [false :exclude] :exclude-acc)) + + +(defn update-if-some-result [m k f & args] + (if-let [result (apply f (get m k) args)] + (assoc m k result) + m)) + + +(defn vs-selected-system-intersection->vs-idx [acc concepts-map expansion-index vs-url sys vs-urls checks + & {:keys [any-system? mode]}] + (let [acc-key (get-acc-key any-system? mode)] + (if-let [concepts + (cond + (seq vs-urls) + (some->> vs-urls + (map (fn [dep-vs-url] + (get-in acc [:vs-idx-acc dep-vs-url sys] #{}))) + (apply set/intersection) + seq + (map (fn [code] [code (get-in concepts-map [sys code])])) + (into {})) + + sys + (get concepts-map sys) + + :else + (throw (ex-info "ValueSet or system should be present" {:vs-url vs-url, :sys sys, :deps vs-urls})))] + (if-let [check-fns (seq (:pred-fns checks))] + (update-in acc [acc-key vs-url] + (fn [vs-idx-acc] + (transduce (filter (fn [[concept-code concept]] (and #_(not-any? #(% concept) exclude-check-fns) #_"TODO: instead of building exclude idx maybe check exclude on building include idx?" + (or (when (= :include mode) + #_"NOTE: this when can not tested, because if expansion is included without checking exclude." + #_"NOTE: Without the 'when exclude gets a codes from expansion and it forbids to include these values," + #_"NOTE: but these values are already included, thus this effect is not observable form outside" + (get-in expansion-index [(:system concept) concept-code])) + (some #(% concept) check-fns))))) + (completing (fn [acc [_concept-code concept]] (push-concept-into-vs-idx acc concept))) + (or vs-idx-acc {}) + concepts))) + (if (:allow-any-concept checks) + (update-in acc [acc-key vs-url sys] (fnil into #{}) (keys concepts)) + (throw (ex-info "must be either predicate fn or whole system allow" + {:vs-url vs-url, :sys sys, :deps vs-urls})))) + (update-in acc [acc-key vs-url] #(or % {}))))) + + +(defn select-all-dep-systems [vs-idx-acc deps-vs-urls] + (mapcat #(keys (get vs-idx-acc %)) + deps-vs-urls)) + + +(defn vs-selected-systems->mode-acc [acc concepts-map expansion-index vs-url dep-system-url vs-urls checks mode] + (let [any-system? (= ::any-system dep-system-url) + selected-systems (if any-system? + (select-all-dep-systems (:vs-idx-acc acc) vs-urls) + [dep-system-url])] + (reduce (fn [acc sys] + (vs-selected-system-intersection->vs-idx acc concepts-map expansion-index vs-url sys vs-urls checks + {:any-system? any-system? + :mode mode})) + acc + selected-systems))) + + +(declare refs-in-vs->vs-idx) + + +(defn ensure-deps-processed [acc concepts-map vs-urls] + (transduce (filter #(seq (get-in acc [:vs-queue %]))) + (completing #(refs-in-vs->vs-idx %1 concepts-map %2)) + acc + vs-urls)) + + +(defn collect-mode-acc [acc concepts-map expansion-index vs-url system compose-els mode] + (reduce-kv (fn [acc vs-urls checks] + (vs-selected-systems->mode-acc acc concepts-map expansion-index vs-url system vs-urls checks mode)) + acc + compose-els)) + + +(defn push-include-exclude->vs-idx [acc vs-url dep-system-url] + (let [any-system? (= ::any-system dep-system-url) + incl-acc-key (get-acc-key any-system? :include) + vs-sys-idx-acc (get-in acc [:vs-idx-acc vs-url] {}) + sys-exclude-idx (get-in acc [:any-sys-exclude-acc vs-url]) + any-exclude-idx (get-in acc [:exclude-acc vs-url]) + exclude-contains-sys? (fn [system] + (or (get-in sys-exclude-idx [system]) + (get-in any-exclude-idx [system]))) + exclude-remove (when (or (seq sys-exclude-idx) (seq any-exclude-idx)) + (fn [system code] + (or (get-in sys-exclude-idx [system code]) + (get-in any-exclude-idx [system code])))) + + new-vs-sys-idx #_"TODO: refactor" + (if any-system? + (if exclude-remove + (reduce-kv (fn [acc sys concepts] + (if (exclude-contains-sys? sys) + (update-if-some-result acc sys + (fn [idx-concepts] + (not-empty + (into (or idx-concepts #{}) + (remove #(exclude-remove sys %)) + concepts)))) + (update acc sys + #(if (some? %1) + (into %1 %2) + %2) + concepts))) + vs-sys-idx-acc + (get-in acc [incl-acc-key vs-url])) + (merge-with #(if (some? %1) + (into %1 %2) + %2) + vs-sys-idx-acc + (get-in acc [incl-acc-key vs-url]))) + (update-if-some-result + vs-sys-idx-acc + dep-system-url + (if (and exclude-remove (exclude-contains-sys? dep-system-url)) + (fn [idx-concepts new-concepts] + (not-empty (into (or idx-concepts #{}) + (remove #(exclude-remove dep-system-url %)) + new-concepts))) + #(if (some? %1) + (into %1 %2) + %2)) + (get-in acc [incl-acc-key vs-url dep-system-url])))] + (assoc-in acc [:vs-idx-acc vs-url] new-vs-sys-idx))) + + +(defn refs-in-vs->vs-idx [acc concepts-map vs-url] + (let [{:keys [deps include exclude full-expansion? expansion-index]} (get-in acc [:vs-queue vs-url]) + + acc (-> acc + (pop-entry-from-vs-queue vs-url) + (ensure-deps-processed concepts-map deps))] + (if full-expansion? + (update-in acc [:vs-idx-acc vs-url] #(merge-with into % expansion-index)) + (let [acc (cond-> acc + (seq expansion-index) + (update-in [:vs-idx-acc vs-url] #(merge-with into % expansion-index))) + + acc (collect-mode-acc acc concepts-map expansion-index vs-url ::any-system (:any-system exclude) :exclude) + + need-to-process-all-excludes? (:any-system include) + + acc (if need-to-process-all-excludes? #_"NOTE: can process not all, but only ones that will be used in any-system include" + (reduce-kv (fn [acc dep-system-url exclude] + (-> acc + (collect-mode-acc concepts-map expansion-index vs-url dep-system-url exclude :exclude) + (push-include-exclude->vs-idx vs-url dep-system-url))) + acc + (:systems exclude)) + acc) + + acc (reduce-kv (fn [acc dep-system-url include] + (-> acc + (cond-> (not need-to-process-all-excludes?) + (collect-mode-acc concepts-map expansion-index vs-url dep-system-url + (get-in exclude [:systems dep-system-url]) + :exclude)) + (collect-mode-acc concepts-map expansion-index vs-url dep-system-url include :include) + (push-include-exclude->vs-idx vs-url dep-system-url))) + acc + (:systems include)) + + acc (-> acc + (collect-mode-acc concepts-map expansion-index vs-url ::any-system (:any-system include) :include) + (push-include-exclude->vs-idx vs-url ::any-system))] + acc)))) + + +(defn all-vs-nested-refs->vs-idx [concepts-map nested-vs-refs-queue] + (loop [acc {:vs-queue nested-vs-refs-queue + :vs-idx-acc {}}] + (let [res-acc (refs-in-vs->vs-idx acc concepts-map (ffirst (:vs-queue acc)))] + (if (seq (:vs-queue res-acc)) + (recur res-acc) + (:vs-idx-acc res-acc))))) + + +(defn push-vs-url-into-concepts-map [concepts-map system code vs-url] + (update-in concepts-map + [system code :valueset] + (fnil conj #{}) + vs-url)) + + +(defn reduce-vs-idx-into-concepts-map [concepts-map vs-idx] + (reduce-kv (fn [acc vs-url vs-cs-idx] + (reduce-kv (fn [acc sys codes] + (reduce (fn [acc code] + (push-vs-url-into-concepts-map acc sys code vs-url)) + acc + codes)) + acc + vs-cs-idx)) + concepts-map + vs-idx)) + + +(defn build-valuesets-compose-idx [valuesets] + (transduce + (map compose) + (completing (fn [queue {:as vs-comp-res :keys [vs-url]}] + (update queue vs-url push-entries-to-vs-queue vs-comp-res))) + {} valuesets)) +(defn denormalize-into-concepts [valuesets concepts-map] + (let [nested-vs-refs-queue (build-valuesets-compose-idx valuesets) + new-vs-idx-entries (all-vs-nested-refs->vs-idx concepts-map nested-vs-refs-queue)] + (reduce-vs-idx-into-concepts-map concepts-map new-vs-idx-entries))) + + (defn denormalize-value-sets-into-concepts [ztx] (swap! ztx update-in [:fhir/inter "Concept"] (partial denormalize-into-concepts - ztx (vals (get-in @ztx [:fhir/inter "ValueSet"]))))) + (vals (get-in @ztx [:fhir/inter "ValueSet"]))))) diff --git a/test/ftr/extraction/ig/value_set_expand_test.clj b/test/ftr/extraction/ig/value_set_expand_test.clj new file mode 100644 index 0000000..b5f1099 --- /dev/null +++ b/test/ftr/extraction/ig/value_set_expand_test.clj @@ -0,0 +1,353 @@ +(ns ftr.extraction.ig.value-set-expand-test + (:require [ftr.extraction.ig.value-set-expand :as sut] + [clojure.string :as str] + [matcho.core :as match] + [clojure.test :as t])) + + +#_"NOTE: Concept resource" +{:code "string" + :system "string"} + +#_"NOTE: ValueSet resource" +{:url "string" + :compose {:include [#_"NOTE: contains union of rules" + {#_"NOTE: rule defining a selection of concepts" + #_"NOTE: Must have :system OR :valueSet. Can have both" + #_"NOTE: Can't have both :concept and :filter. Can have neither" + :system "string" + :concept [{:code "string"} #_...]} + {:system "string" + :filter {:property "string" + :op "string" #_"NOTE: = | is-a | descendent-of | is-not-a | regex | in | not-in | generalizes | exists" + :value "string"}} + {:valueSet ["string" #_"NOTE: intersection of other value sets"]}] + :exclude [#_"NOTE: structure is the same as include" + #_"NOTE: excludes union of rules here from the union defined in include" + {}]}} + +#_"NOTE: CodeSystem index" +{"" + {"" + {#_"" + :valueset [""] + #_"NOTE: this attr is empty before the start of the algorithm, must be populated as the result"}}} + +#_"NOTE: ValueSet.compose process queue" +{"" + {:deps #{""} + + :include {:systems + {"" {[""] #_"NOTE: value set intersection, can be empty" + {#_"NOTE: can't have both" + :allow-any-concept true + :pred-fn [#_"function that accepts a Concept and returns a boolean"]}}} + + :any-system #_"NOTE: when no system specified, deduce from provided value sets" + {[""] #_"NOTE: value set intersection, CAN NOT be empty" + {#_"NOTE: can't have both" + :allow-any-concept true + :pred-fn [#_"function that accepts a Concept and returns a boolean"]}}} + :exclude {#_"same structure as :include"} + + :full-expansion? true ;; boolean + :expansion-index {"" + #{""}}}} + +#_"NOTE: ValueSet content index" +{"" + {"" + #{""}}} + +#_"NOTE: after this index is produced it is normalized into the CodeSystem index [system code :valueset]" + + +(t/deftest nested-vs-refs-process-test + + (def concepts-index-fixture + {"sys1" {"code11" {:system "sys1" + :code "code11"} + "code12" {:system "sys1" + :code "code12"}} + "sys2" {"code21" {:system "sys2" + :code "code21"} + "code22" {:system "sys2" + :code "code22"}}}) + + (def valuesets-fixtures + [{:url "simple-include" + :compose {:include [{:system "sys1"} + {:system "sys2"}]}} + {:url "simpler-include" + :compose {:include [{:system "sys2"}]}} + {:url "include-with-concept-enumeration" + :compose {:include [{:system "sys1" + :concept [{:code "code11"}]} + {:system "sys2"}]}} + {:url "include-with-filter" + :compose {:include [{:system "sys1" + :filter [{:op "is-a" + :property "code" + :value "code11"}]} + {:system "sys2"}]}} + + {:url "empty-vs" + :compose {:include [{:system "sys1" + :filter [{:op "is-a" + :property "code" + :value "???"}]}]}} + + {:url "full-expansion" + :compose {:include [{:system "sys1" + :filter [{:op "is-a" + :property "missing" + :value "foo"}]}]} + :expansion {:total 1 + :offset 0 + #_#_:parameter {:offset 0 :count 1000} #_"TODO: check if there no meaningful parameters" + :contains [{:system "sys1" #_"TODO: add recursive :contains" + :code "code12"}]}} + + {:url "not-full-expansion" + :compose {:include [{:system "sys1" + :filter [{:op "is-a" + :property "code" + :value "code11"}]} + {:system "sys2" + :filter [{:op "is-a" + :property "missing" + :value "foo"}]}]} + :expansion {:contains [{:system "sys2" + :code "code21"}]}} + + {:url "depends-on-valueset" + :compose {:include [{:valueSet ["simple-include"]}]}} + + {:url "depends-on-valueset-intersection" + :compose {:include [{:valueSet ["simple-include" + "full-expansion"]}]}} + + {:url "intersection-with-empty-valueset" + :compose {:include [{:valueSet ["simple-include" + "empty-vs"]}]}} + + {:url "empty-intersection" + :compose {:include [{:valueSet ["full-expansion" + "depends-on-valueset-and-filters-by-sys-and-pred"]}]}} + + {:url "depends-on-valueset-intersection-and-filters-by-sys" + :compose {:include [{:system "sys1" + :valueSet ["simple-include" + "not-full-expansion"]}]}} + + {:url "depends-on-valueset-and-filters-by-sys" + :compose {:include [{:system "sys1" + :valueSet ["simple-include"]}]}} + + {:url "depends-on-valueset-and-filters-by-sys-and-pred" + :compose {:include [{:system "sys1" + :filter [{:op "is-a" + :property "code" + :value "code11"}] + :valueSet ["simple-include"]}]}} + + {:url "exclude-system" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys2"}]}} + {:url "exclude-code" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys2" + :concept [{:code "code22"}]}]}} + {:url "multiple-exclude" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys1" + :concept [{:code "code12"}]} + {:system "sys2" + :concept [{:code "code22"}]}]}} + {:url "multiple-exclude-with-vs" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys1" + :valueSet ["full-expansion"]} + {:system "sys2" + :concept [{:code "code22"}]}]}} + {:url "multiple-exclude-with-vs-no-system" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:valueSet ["full-expansion"]} + {:system "sys2" + :concept [{:code "code22"}]}]}} + {:url "exclude-filter" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys2" + :filter [{:op "is-a" + :property "code" + :value "code22"}]}]}} + {:url "exclude-valueset-with-system" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys1" + :valueSet ["simple-include"]}]}} + {:url "exclude-valueset-with-pred" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys1" + :concept [{:code "code12"}] + :valueSet ["simple-include"]}]}} + {:url "exclude-valueset-with-pred-not-in-vs" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys1" + :filter [{:op "is-a" + :property "code" + :value "???"}] + :valueSet ["simple-include"]}]}} + {:url "exclude-valueset-no-system" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:valueSet ["depends-on-valueset-intersection"]}]}} + {:url "exclude-all" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys1"} + {:system "sys2"}]}} + {:url "exclude-all-vs" + :compose {:include [{:valueSet ["simple-include"]}] + :exclude [{:valueSet ["simple-include"]}]}} + {:url "exclude-valueset-system-not-present-in-vs" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys2" + :valueSet ["depends-on-valueset-intersection"]}]}} + {:url "exclude-empty-valueset" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:valueSet ["empty-vs"]}]}} + {:url "exclude-valueset-intersection" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:valueSet ["simple-include" + "full-expansion"]}]}} + {:url "exclude-empty-valueset-intersection" + :compose {:include [{:system "sys1"}] + :exclude [{:system "sys1" + :valueSet ["depends-on-valueset-intersection" + "depends-on-valueset-intersection-and-filters-by-sys"]}]}} + {:url "exclude-empty-valueset-intersection" + :compose {:include [{:valueSet ["depends-on-valueset-intersection"]}] + :exclude [{:system "sys1" + :valueSet ["depends-on-valueset-intersection-and-filters-by-sys"]}]}} + {:url "exclude-empty-valueset-intersection-system" + :compose {:include [{:system "sys1" + :valueSet ["depends-on-valueset-intersection"]}] + :exclude [{:system "sys1" + :valueSet ["depends-on-valueset-intersection-and-filters-by-sys"]}]}} + {:url "exclude-valueset-not-intersection-no-system" + :compose {:include [{:valueSet ["depends-on-valueset-intersection"]}] + :exclude [{:valueSet ["depends-on-valueset-intersection-and-filters-by-sys"]}]}} + {:url "exclude-valueset-not-intersection-no-system-no-system-in-exclude" + :compose {:include [{:valueSet ["simple-include"]}] + :exclude [{:system "sys1" + :valueSet ["simpler-include"]} + {:system "sys1" + :filter [{:op "is-a" + :property "code" + :value "code12"}]}]}} + {:url "exclude-with-expansion" + :compose {:include [{:valueSet ["simple-include"] + :filter [{:op "is-a" + :property "missing" + :value "???"}]}] + :exclude [{:system "sys2" + :concept [{:code "code22"}]}]} + :expansion {:contains [{:system "sys2" + :code "code21"}]}} + #_{:url "exclude-withexpansion" + :compose {:include [{:system "sys1"} + {:system "sys2"}] + :exclude [{:system "sys2" #_"FIXME: WHAT TO DO WHEN WE CAN'T EVAL EXCLUDE????" + :filter [{:op "is-a" + :property "missing" + :value "???"}]}]} + :expansion {:contains [{:system "sys2" + :code "code21"}]}} + {:url "include-vs-exclude-other-vs" + :compose {:include [{:valueSet ["simpler-include" "simple-include"]}] + :exclude [{:valueSet ["not-full-expansion"]}]}}]) + + (def valuesets-index-assert + {"simple-include" {"sys1" #{"code11" "code12"} + "sys2" #{"code21" "code22"}} + + "simpler-include" {"sys2" #{"code21" "code22"}} + + "include-with-concept-enumeration" {"sys1" #{"code11"} + "sys2" #{"code21" "code22"}} + + "include-with-filter" {"sys1" #{"code11"} + "sys2" #{"code21" "code22"}} + + "empty-vs" {} + + "full-expansion" {"sys1" #{"code12"}} + + "not-full-expansion" {"sys1" #{"code11"} + "sys2" #{"code21"}} + + "depends-on-valueset" {"sys1" #{"code11" "code12"} + "sys2" #{"code21" "code22"}} + + "depends-on-valueset-intersection" {"sys1" #{"code12"}} + + "depends-on-valueset-intersection-and-filters-by-sys" {"sys1" #{"code11"}} + + "intersection-with-empty-valueset" {} + + "empty-intersection" {} + + "depends-on-valueset-and-filters-by-sys" {"sys1" #{"code11" "code12"}} + + "depends-on-valueset-and-filters-by-sys-and-pred" {"sys1" #{"code11"}} + + "exclude-system" {"sys1" #{"code11" "code12"}} + "exclude-code" {"sys1" #{"code11" "code12"} + "sys2" #{"code21"}} + "multiple-exclude" {"sys1" #{"code11"} + "sys2" #{"code21"}} + "multiple-exclude-with-vs" {"sys1" #{"code11"} + "sys2" #{"code21"}} + "multiple-exclude-with-vs-no-system" {"sys1" #{"code11"} + "sys2" #{"code21"}} + "exclude-filter" {"sys1" #{"code11" "code12"} + "sys2" #{"code21"}} + "exclude-valueset-with-system" {"sys2" #{"code21" "code22"}} + "exclude-valueset-with-pred" {"sys1" #{"code11"} + "sys2" #{"code21" "code22"}} + "exclude-valueset-with-pred-not-in-vs" {"sys1" #{"code11" "code12"} + "sys2" #{"code21" "code22"}} + "exclude-valueset-no-system" {"sys1" #{"code11"} + "sys2" #{"code21" "code22"}} + "exclude-all" {} + "exclude-all-vs" {} + "exclude-valueset-system-not-present-in-vs" {"sys1" #{"code11" "code12"} + "sys2" #{"code21" "code22"}} + "exclude-empty-valueset" {"sys1" #{"code11" "code12"} + "sys2" #{"code21" "code22"}} + "exclude-valueset-intersection" {"sys1" #{"code11"} + "sys2" #{"code21" "code22"}} + "exclude-empty-valueset-intersection" {"sys1" #{"code12"}} + "exclude-empty-valueset-intersection-system" {"sys1" #{"code12"}} + "exclude-valueset-not-intersection-no-system" {"sys1" #{"code12"}} + "exclude-valueset-not-intersection-no-system-no-system-in-exclude" {"sys1" #{"code11"} + "sys2" #{"code21" "code22"}} + "exclude-with-expansion" {"sys2" #{"code21"}} + "include-vs-exclude-other-vs" {"sys2" #{"code22"}}}) + + (t/is (= valuesets-index-assert + (sut/all-vs-nested-refs->vs-idx + concepts-index-fixture + (sut/build-valuesets-compose-idx valuesets-fixtures))))) diff --git a/test/ftr/zen_package_test.clj b/test/ftr/zen_package_test.clj index 886b58d..21aac91 100644 --- a/test/ftr/zen_package_test.clj +++ b/test/ftr/zen_package_test.clj @@ -456,22 +456,22 @@ (t/testing "built ftr shape is ok" (matcho/match - (test-utils/fs-tree->tree-map profile-lib-path) - {"ftr" - {"ig" - {"tags" {"v1.ndjson.gz" {}} - "vs" {"gender2-cs-entire-code-system" nil - "gender2-vs" {} - "gender3-cs-entire-code-system" nil - "gender3-vs" nil - "gender4-cs-entire-code-system" {} - "gender4-vs" nil - "gender5-cs-entire-code-system" {} - "gender5-vs" nil - "gender6-cs-entire-code-system" nil - "gender6-vs" {} - #_#_"gender7-cs-entire-code-system" nil - #_#_"gender7-vs" {}}}}}))) + (get (test-utils/fs-tree->tree-map profile-lib-path) + "ftr") + {"ig" + {"tags" {"v1.ndjson.gz" {}} + "vs" {"gender2-cs-entire-code-system" nil + "gender2-vs" {} + "gender3-cs-entire-code-system" nil + "gender3-vs" nil + "gender4-cs-entire-code-system" {} + "gender4-vs" nil + "gender5-cs-entire-code-system" {} + "gender5-vs" nil + "gender6-cs-entire-code-system" nil + "gender6-vs" {} + #_#_"gender7-cs-entire-code-system" nil + #_#_"gender7-vs" {}}}}))) (defn test-concept-vs-backrefs [root-path] @@ -810,7 +810,8 @@ :id "custom-gender-vs-id" :url "custom-gender-vs-url" :status "active" - :compose {:include [{:valueSet ["gender-vs-url"]} + :compose {:include [{:valueSet ["gender-vs-url"] + :system "gender-cs-url"} {:valueSet ["expanded-gender-vs-url"]}] :exclude [{:valueSet ["unknown-gender-vs-url"]}]}}] {'ftr-expansion-lib {:deps #{['zen-fhir (str (System/getProperty "user.dir") "/zen.fhir/")]}