diff --git a/deps.edn b/deps.edn index 92f0aef..b87838a 100644 --- a/deps.edn +++ b/deps.edn @@ -12,7 +12,8 @@ com.rpl/specter {:mvn/version "1.1.3"} cli-matic/cli-matic {:mvn/version "0.4.3"} http-kit/http-kit {:mvn/version "2.6.0"} - enlive/enlive {:mvn/version "1.1.6"}} + enlive/enlive {:mvn/version "1.1.6"} + clj-commons/clj-yaml {:mvn/version "1.0.26"}} :aliases {:nrepl diff --git a/src/ftr/core.clj b/src/ftr/core.clj index 51f1961..1ee7309 100644 --- a/src/ftr/core.clj +++ b/src/ftr/core.clj @@ -104,6 +104,14 @@ ::write-tag-index-hash]) +(def serialized-objects-array-pipeline + [::extract-terminology + ::write-terminology-file + ::shape-ftr-layout + :ftr.post-write-coordination.core/coordinate + ::write-tag-index-hash]) + + (defmethod u/*fn ::select-ftr-pipeline [{:as _ctx, ::keys [commit-type] {:keys [source-type]} :cfg}] @@ -118,6 +126,9 @@ [:append :snomed] snomed-pipeline + [:append :serialized-objects-array] + serialized-objects-array-pipeline + [:tag-merge nil] tag-merge-pipeline)}) diff --git a/src/ftr/extraction/core.clj b/src/ftr/extraction/core.clj index f2bd07b..e415676 100644 --- a/src/ftr/extraction/core.clj +++ b/src/ftr/extraction/core.clj @@ -1,7 +1,8 @@ (ns ftr.extraction.core (:require [ftr.extraction.flat-table :as flat-table] [ftr.extraction.ig.core] - [ftr.extraction.snomed])) + [ftr.extraction.snomed] + [ftr.extraction.serialized-objects-array])) (defn extract [cfg] (let [{:keys [source-type source-url extractor-options]} cfg @@ -12,4 +13,5 @@ :flat-table (flat-table/import-from-cfg extractor-cfg) :ig (ftr.extraction.ig.core/import-from-cfg extractor-cfg) :igs (ftr.extraction.ig.core/import-from-cfg extractor-cfg) - :snomed (ftr.extraction.snomed/import-from-cfg extractor-cfg)))) + :snomed (ftr.extraction.snomed/import-from-cfg extractor-cfg) + :serialized-objects-array (ftr.extraction.serialized-objects-array/import-from-cfg extractor-cfg)))) diff --git a/src/ftr/extraction/flat_table.clj b/src/ftr/extraction/flat_table.clj index 3a0c4c2..97f7c7a 100644 --- a/src/ftr/extraction/flat_table.clj +++ b/src/ftr/extraction/flat_table.clj @@ -3,8 +3,7 @@ [ftr.utils.unifn.core :as u] [ftr.utils.core] [clojure.data.csv :as csv] - [clojure.string :as str] - [cheshire.core])) + [clojure.string :as str])) (defmethod u/*fn ::create-value-set [cfg] {::result {:value-set diff --git a/src/ftr/extraction/serialized_objects_array.clj b/src/ftr/extraction/serialized_objects_array.clj new file mode 100644 index 0000000..cc90422 --- /dev/null +++ b/src/ftr/extraction/serialized_objects_array.clj @@ -0,0 +1,99 @@ +(ns ftr.extraction.serialized-objects-array + (:require [clojure.java.io :as io] + [ftr.utils.unifn.core :as u] + [ftr.utils.core] + [clojure.string :as str] + [clj-yaml.core])) + + +(defmethod u/*fn ::create-value-set [cfg] + {::result {:value-set + (-> (:value-set cfg) + (assoc :resourceType "ValueSet") + (->> (merge {:status "unknown" + :compose {:include [{:system (get-in cfg [:code-system :url])}]}})))}}) + + +(defmethod u/*fn ::create-code-system [cfg] + {::result {:code-system (-> (:code-system cfg) + (assoc :resourceType "CodeSystem") + (->> (merge {:status "unknown" + :content "not-present" + :valueSet (get-in cfg [:value-set :url])}) + (conj #{})))}}) + + +(defn extract-mapped-value [mapping source] + (update-vals mapping + (fn [el] + (let [v (get-in source (:path el))] + (cond-> el + (not (str/blank? v)) + (assoc :value v)))))) + + +(defn extract-mapping [deserialized-object mapping] + (update-vals mapping #(extract-mapped-value % deserialized-object))) + +(ftr.utils.core/strip-nils {}) + +(defn format-concept-resource [extracted-mapping {:keys [system valueset]}] + (let [code (get-in extracted-mapping [:concept :code :value]) + display (get-in extracted-mapping [:concept :display :value]) + status (get-in extracted-mapping [:concept :deprecated? :value]) + deprecation-mark (get-in extracted-mapping [:concept :deprecated? :true-values]) + parent-id (get-in extracted-mapping [:concept :parent-id :value]) + hierarchy-id (get-in extracted-mapping [:concept :hierarchy-id :value]) + concept (ftr.utils.core/strip-nils + {:resourceType "Concept" + :system system + :valueset [valueset] + :code code + :deprecated (if (some #(= status %) deprecation-mark) true nil) + :display display + :parent-id parent-id + :hierarchy-id hierarchy-id + :property (-> (:property extracted-mapping) + (update-vals :value) + ftr.utils.core/strip-nils + not-empty)})] + (when (:code concept) + concept))) + + +(defn process-deserialized-objects [deserialized-objects + {:as cfg, :keys [mapping]}] + (->> deserialized-objects + (map (fn process-deserialized-object [do] + (-> do + (extract-mapping mapping) + (format-concept-resource cfg)))) + (filter identity))) + + +(defmethod u/*fn ::import [{:as cfg, + :keys [source-url]}] + (let [deserialized-objects + (clj-yaml.core/parse-stream (io/reader source-url)) + + deserialized-objects-seq + (process-deserialized-objects + deserialized-objects + (merge cfg + {:system (get-in cfg [:code-system :url]) + :valueset (get-in cfg [:value-set :url]) + :mapping (:mapping cfg)}))] + {::result {:concepts deserialized-objects-seq}})) + + +(defn import-from-cfg [cfg] + (::result (u/*apply [::create-value-set + ::create-code-system + ::import] cfg))) + +(comment + + (clj-yaml.core/parse-stream (io/reader "/Users/ghrp/Downloads/external-care-team-relationship.yaml") ) + + + ) diff --git a/test/fixture/yaml/external-care-team-relationship.yaml b/test/fixture/yaml/external-care-team-relationship.yaml new file mode 100644 index 0000000..0fa160f --- /dev/null +++ b/test/fixture/yaml/external-care-team-relationship.yaml @@ -0,0 +1,24 @@ +- code: "SPS" + display: "Spouse" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- code: "CAREGIVER" + display: "Caregiver" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- code: "CHILD" + display: "Children" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- code: "NMTH" + display: "Parents" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- code: "SIGOTHR" + display: "Significant Other" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- code: "HOMPART" + display: "Partner(s)" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- Healthcare Proxy +- Relative +- code: "STPPRN" + display: "Step-Parent(s)" + system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode" +- Other diff --git a/test/ftr/core_test.clj b/test/ftr/core_test.clj index 903f954..15444b7 100644 --- a/test/ftr/core_test.clj +++ b/test/ftr/core_test.clj @@ -970,3 +970,115 @@ (clean-up-test-env!)) ) + + +(t/deftest serialized-objects-array-source-type-test + (def soa-test-env-cfg {:ftr-path "/tmp/soa/ftr" + :soa-source "/Users/ghrp/dev/ftr/test/fixture/yaml/external-care-team-relationship.yaml"}) + + + (def soa-user-cfg {:module "soa" + :source-url (:soa-source soa-test-env-cfg) + :ftr-path (:ftr-path soa-test-env-cfg) + :tag "prod" + :source-type :serialized-objects-array + :extractor-options {:format "yaml" + :mapping {:concept {:code {:path [:code]} + :display {:path [:display]}}} + :code-system {:id "soa" + :url "http://soa/cs"} + :value-set {:id "soa" + :url "http://soa/vs"}}}) + (t/testing "Clean up test-env" + (ftr.utils.core/rmrf (:ftr-path soa-test-env-cfg))) + + (t/testing "User provides config for SOA" + + + (let [{:as user-cfg, :keys [module ftr-path tag] + {{value-set-name :url} :value-set} :extractor-options} + soa-user-cfg + + value-set-name (ftr.utils.core/escape-url value-set-name) + + _ + (sut/apply-cfg {:cfg user-cfg}) + + ftr-tree + (get-in (fs-tree->tree-map ftr-path) (str/split (subs ftr-path 1) #"/"))] + + (t/testing "sees generated repository layout, tf sha is correct" + (matcho/match + ftr-tree + {"soa" + {"vs" + {"http:--soa-vs" + {"tf.92be9f413299bc56f798f6855a3f018b73f676e353e5faae6b51ebe8d946777b.ndjson.gz" + {}, + "tag.prod.ndjson.gz" {}}}, + "tags" {"prod.ndjson.gz" {}, "prod.hash" {}}}})) + + (t/testing "sees terminology tag file" + (matcho/match + (ftr.utils.core/parse-ndjson-gz + "/tmp/soa/ftr/soa/tags/prod.ndjson.gz") + [{:hash + "92be9f413299bc56f798f6855a3f018b73f676e353e5faae6b51ebe8d946777b" + :name "soa.http:--soa-vs"} + nil?])) + + (t/testing "sees terminology file" + (matcho/match + (ftr.utils.core/parse-ndjson-gz + "/tmp/soa/ftr/soa/vs/http:--soa-vs/tf.92be9f413299bc56f798f6855a3f018b73f676e353e5faae6b51ebe8d946777b.ndjson.gz") + [{:resourceType "CodeSystem" + :url "http://soa/cs" + :valueSet "http://soa/vs"} + {:resourceType "ValueSet" + :url "http://soa/vs"} + {:code "CAREGIVER" + :display "Caregiver" + :id "http:--soa-cs-http:--soa-vs-CAREGIVER" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + {:code "CHILD" + :display "Children" + :id "http:--soa-cs-http:--soa-vs-CHILD" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + {:code "HOMPART" + :display "Partner(s)" + :id "http:--soa-cs-http:--soa-vs-HOMPART" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + {:code "NMTH" + :display "Parents" + :id "http:--soa-cs-http:--soa-vs-NMTH" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + {:code "SIGOTHR" + :display "Significant Other" + :id "http:--soa-cs-http:--soa-vs-SIGOTHR" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + {:code "SPS" + :display "Spouse" + :id "http:--soa-cs-http:--soa-vs-SPS" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + {:code "STPPRN" + :display "Step-Parent(s)" + :id "http:--soa-cs-http:--soa-vs-STPPRN" + :resourceType "Concept" + :system "http://soa/cs" + :valueset ["http://soa/vs"]} + nil?])))) + + (t/testing "Clean up test-env" + (ftr.utils.core/rmrf (:ftr-path soa-test-env-cfg))))