diff --git a/Makefile b/Makefile index 5f6468e..43766a5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ build-jar: clj -T:build uberjar -test: +run-tests: clj -M:test -m kaocha.runner PATH_TO_JAR := $(project_dir)/$(jar_path) diff --git a/deps.edn b/deps.edn index 6b03082..ba4bb2a 100644 --- a/deps.edn +++ b/deps.edn @@ -8,10 +8,11 @@ :aliases {:dev {:extra-paths ["dev"]} - :test {:extra-paths ["tests"] + :test {:extra-paths ["test"] :exec-fn kaocha.runner/exec-fn - :extra-deps {com.health-samurai/matcho {:mvn/version "0.3.11"} - lambdaisland/kaocha {:mvn/version "1.91.1392"}}} + :extra-deps {com.health-samurai/matcho {:mvn/version "0.3.11"} + lambdaisland/kaocha {:mvn/version "1.91.1392"} + nubank/matcher-combinators {:mvn/version "3.9.1"}}} :build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.5"}} :ns-default build}}} diff --git a/dev/user.clj b/dev/user.clj index 8ea2b9a..2388381 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -1,17 +1,19 @@ (ns user (:require [aidbox-sdk.generator :as gen] + [aidbox-sdk.schema.verify :refer [fhir-version-pattern]] [clojure.data] - [clojure.java.io :as io])) + [clojure.java.io :as io] + [clojure.string :as str])) (def source' (io/file "resources/schemas")) -(def target (io/file "/tmp/sdk")) +(def target (io/file "out/")) (defn vector-to-map [v] (->> (map (fn [item] (hash-map (:url item) item)) v) (into {}))) -(apply merge [{:a 1 :b 2} {:a 3 :c 4}]) +;; (apply merge [{:a 1 :b 2} {:a 3 :c 4}]) (comment @@ -71,5 +73,4 @@ (gen/build-all! source' target) - ;; - ) + :rcf) diff --git a/resources/schemas/1.0.0_hl7.fhir.us.core#4.1.0_package.ndjson.gz b/resources/schemas/1.0.0_hl7.fhir.us.core#4.1.0_package.ndjson.gz deleted file mode 100644 index d23cf37..0000000 Binary files a/resources/schemas/1.0.0_hl7.fhir.us.core#4.1.0_package.ndjson.gz and /dev/null differ diff --git a/resources/schemas/1.0.0_hl7.fhir.r4.core#4.0.1_package.ndjson.gz b/resources/schemas/hl7.fhir.r4.core#4.0.1.ndjson.gz similarity index 100% rename from resources/schemas/1.0.0_hl7.fhir.r4.core#4.0.1_package.ndjson.gz rename to resources/schemas/hl7.fhir.r4.core#4.0.1.ndjson.gz diff --git a/resources/schemas/1_0_0_hl7_fhir_us_codex_radiation_therapy#1_0_0_package_ndjson.gz b/resources/schemas/hl7.fhir.us.codex.radiation.therapy#1.0.0.ndjson.gz similarity index 100% rename from resources/schemas/1_0_0_hl7_fhir_us_codex_radiation_therapy#1_0_0_package_ndjson.gz rename to resources/schemas/hl7.fhir.us.codex.radiation.therapy#1.0.0.ndjson.gz diff --git a/resources/schemas/hl7.fhir.us.core#4.0.0.ndjson.gz b/resources/schemas/hl7.fhir.us.core#4.0.0.ndjson.gz new file mode 100644 index 0000000..e6c2e7e Binary files /dev/null and b/resources/schemas/hl7.fhir.us.core#4.0.0.ndjson.gz differ diff --git a/resources/schemas/1.0.0_hl7.fhir.us.mcode#2.1.0_package.ndjson.gz b/resources/schemas/hl7.fhir.us.mcode#2.1.0.ndjson.gz similarity index 100% rename from resources/schemas/1.0.0_hl7.fhir.us.mcode#2.1.0_package.ndjson.gz rename to resources/schemas/hl7.fhir.us.mcode#2.1.0.ndjson.gz diff --git a/resources/schemas/hl7.fhir.uv.bulkdata#1.1.0.ndjson.gz b/resources/schemas/hl7.fhir.uv.bulkdata#1.1.0.ndjson.gz new file mode 100644 index 0000000..6ef32fc Binary files /dev/null and b/resources/schemas/hl7.fhir.uv.bulkdata#1.1.0.ndjson.gz differ diff --git a/resources/schemas/hl7.fhir.uv.extensions.r4#1.0.0.ndjson.gz b/resources/schemas/hl7.fhir.uv.extensions.r4#1.0.0.ndjson.gz new file mode 100644 index 0000000..5128173 Binary files /dev/null and b/resources/schemas/hl7.fhir.uv.extensions.r4#1.0.0.ndjson.gz differ diff --git a/resources/schemas/1.0.0_hl7.terminology.r4#5.0.0_package.ndjson.gz b/resources/schemas/hl7.terminology.r4#5.0.0.ndjson.gz similarity index 100% rename from resources/schemas/1.0.0_hl7.terminology.r4#5.0.0_package.ndjson.gz rename to resources/schemas/hl7.terminology.r4#5.0.0.ndjson.gz diff --git a/resources/schemas/us.nlm.vsac#0.3.0.ndjson.gz b/resources/schemas/us.nlm.vsac#0.3.0.ndjson.gz new file mode 100644 index 0000000..6094aa8 Binary files /dev/null and b/resources/schemas/us.nlm.vsac#0.3.0.ndjson.gz differ diff --git a/src/aidbox_sdk/core.clj b/src/aidbox_sdk/core.clj index 32245a5..9908dd4 100644 --- a/src/aidbox_sdk/core.clj +++ b/src/aidbox_sdk/core.clj @@ -30,6 +30,8 @@ (println "Error: please provide an output argument")) :else - (generator/build-all! - (resource input) - (io/as-file output))))) + (do + (println "Building FHIR SDK...") + (generator/build-all! + (resource input) + (io/as-file output)))))) diff --git a/src/aidbox_sdk/generator.clj b/src/aidbox_sdk/generator.clj index e6aa588..1e7c098 100644 --- a/src/aidbox_sdk/generator.clj +++ b/src/aidbox_sdk/generator.clj @@ -1,16 +1,15 @@ (ns aidbox-sdk.generator (:refer-clojure :exclude [namespace]) - (:require - [clojure.data.json :as json] - [aidbox-sdk.generator.dotnet.templates :as dotnettpl] - [aidbox-sdk.generator.helpers :refer [->pascal-case safe-conj - uppercase-first-letter vector-to-map]] - [clojure.java.io :as io] - [clojure.set :as set] - [clojure.string :as str] - [clojure.walk]) - (:import - [java.util.zip ZipEntry ZipOutputStream])) + (:require [aidbox-sdk.generator.dotnet.templates :as dotnettpl] + [aidbox-sdk.generator.helpers :refer [->pascal-case safe-conj + uppercase-first-letter + vector-to-map]] + [aidbox-sdk.schema :as schema] + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.string :as str] + [clojure.walk]) + (:import [java.util.zip ZipEntry ZipOutputStream])) ;; ;; FHIR @@ -646,6 +645,7 @@ (delete-directory! dir) (create-directory! dir)) +;; FIXME do we need it? (defn zip-dir! [path zip-name] (with-open [zip (ZipOutputStream. (io/output-stream zip-name))] (doseq [f (file-seq (io/file path)) :when (.isFile f)] @@ -654,6 +654,7 @@ (.closeEntry zip))) (io/file zip-name)) +;; FIXME do we need it? (defn copy-files! [src-dir target-dir] (doseq [file (remove #(.isDirectory %) (file-seq src-dir))] (io/copy file (io/file target-dir (.getName file))))) @@ -691,61 +692,15 @@ (conj schema {:backbone-elements (flat-backbones (:backbone-elements schema) [])}))))) -(defn get-directory-files [path] - (->> path - file-seq - (remove #(.isDirectory %)))) - -(defn fetch-packages [source-path] - (->> source-path - (get-directory-files) - (remove #(.isDirectory %)) - (filter #(str/includes? (.getName %) "hl7.fhir")))) - -(defn create-gzip-reader [path] - (-> path - (io/input-stream) - (java.util.zip.GZIPInputStream.) - (io/reader))) - -(defn parse-ndjson-gz [path] - (with-open [rdr (create-gzip-reader path)] - (->> rdr - line-seq - (mapv (fn [line] - (json/read-str line :key-fn keyword)))))) - -(defn merge-duplicates [schemas] - (->> schemas - (group-by :url) - (map (fn [[_url same-url-schemas]] - (apply merge same-url-schemas))))) - -(defmulti retrieve-schemas class) - -(defmethod retrieve-schemas java.io.File - [source] - (->> (fetch-packages source) - (map parse-ndjson-gz) - (flatten) - (remove #(nil? (:package-meta %))) - (map (fn [schema] - (assoc schema :package (get-in schema [:package-meta :name])))) - (merge-duplicates))) - -(defmethod retrieve-schemas java.net.URL - [source] (do "something")) - (defn build-all! [input output] (let [search-parameters-dir (io/file output "search") - all-schemas (retrieve-schemas input) + all-schemas (schema/retrieve input) ;; search-params-schemas (retrieve-search-params source-dir) search-params-schemas all-schemas constraints (->> all-schemas (filter #(and (constraint? %) (not (from-extension? %)))))] - (prepare-target-directory! output) ;; create base namespace (all FHIR datatypes) file diff --git a/src/aidbox_sdk/schema.clj b/src/aidbox_sdk/schema.clj new file mode 100644 index 0000000..76381de --- /dev/null +++ b/src/aidbox_sdk/schema.clj @@ -0,0 +1,70 @@ +(ns aidbox-sdk.schema + (:require [aidbox-sdk.schema.verify :as verify] + [clojure.data.json :as json] + [clojure.java.io :as io] + [clojure.string :as str])) + +(defn get-packages-from-directory + "Returns all packages in the given directory, including files in subdirectories. + NOTE: right now it'll filter out all files which do not contains hl7.fhir + in their name." + [path] + (let [packages (->> path + file-seq + (remove #(.isDirectory %)) + ;; FIXME only gzip, but there is no problem to accept unpacked ndjson + ;; see https://github.com/Aidbox/aidbox-sdk/issues/11 + (filter #(str/ends-with? (.getName %) ".gz")))] + (println "✅ Found packages:" (count packages)) + packages)) + +(defn create-gzip-reader [path] + (-> path + (io/input-stream) + (java.util.zip.GZIPInputStream.) + (io/reader))) + +;; TODO derive some criteria to determine whether it's a package (is it valid?) +;; see https://github.com/Aidbox/aidbox-sdk/issues/10 +(defn parse-package [path] + (println "Parsing package:" (str path)) + (with-open [rdr (create-gzip-reader path)] + (->> rdr + line-seq + (mapv (fn [line] + (json/read-str line :key-fn keyword)))))) + +(defn remove-invalid-schemas [schemas] + (remove #(nil? (:package-meta %)) schemas)) + +(defn merge-duplicates [schemas] + (->> schemas + (group-by :url) + (map (fn [[_url same-url-schemas]] + (apply merge same-url-schemas))))) + +(defn prepare-schemas [schemas] + (map #(->> (get-in % [:package-meta :name]) + (assoc % :package)) + schemas)) + + +(defmulti retrieve class) + +;; ! According to an example here: +;; ! https://clojuredocs.org/clojure.java.io/file +;; ! it's possible to create a File instance from url, which may lead to bugs +(defmethod retrieve java.io.File + [source] + (println "Retrieving packages from: " (str source)) + (->> (get-packages-from-directory source) + (mapv parse-package) + (verify/check-compatibility!) + (flatten) + (remove-invalid-schemas) + (prepare-schemas) + (merge-duplicates))) + +(defmethod retrieve java.net.URL + [source] + (do "something")) diff --git a/src/aidbox_sdk/schema/verify.clj b/src/aidbox_sdk/schema/verify.clj new file mode 100644 index 0000000..11454b3 --- /dev/null +++ b/src/aidbox_sdk/schema/verify.clj @@ -0,0 +1,144 @@ +(ns aidbox-sdk.schema.verify + (:require [clojure.string :as str])) + +(def fhir-version-pattern + #"^(0|[1-9]\d*)\.(0|[1-9]\d*)(?:\.(0|[1-9]\d*))?(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))?)?$") + +(defn versions-match? [v1 v2] + (let [->groups (fn [v] (->> v (re-matcher fhir-version-pattern) re-find)) + ->map (fn [[_ major minor patch label]] + {:major major, :minor minor, :patch patch, :label label}) + + v1-groups (-> v1 ->groups ->map) + v2-groups (-> v2 ->groups ->map)] + + (if (and v1-groups v2-groups) + (and (= (:major v1-groups) (:major v2-groups)) + (= (:minor v1-groups) (:minor v2-groups))) + false))) + +(defn simplify-package-meta [package] + (select-keys package [:name :version])) + + +;; FIXME: is it reliable to use first element of the list? +;; ! seems like it's not for original packages (without Aidbox processing). +(defn extract-meta-from-package + "Extracts meta information from the package." + [package] + (first package)) + +(defn find-core-package + "Finds core package in the list of packages. + Throws an exception if there are more then one core package." + [packages] + (let [cores (filter #(= "fhir.core" (:type %)) packages) + core (first cores)] + (cond + (= (count cores) 0) + (throw (ex-info "No core package found" {})) + + (> (count cores) 1) + (throw (ex-info "Found more then one core package" + {:packages (mapv #(simplify-package-meta %) cores)})) + + :else + core))) + +(defn find-extra-packages + "Finds extra packages in the list of packages. + Throws an exception if there are a few packages with same name." + [packages] + (let [extra (remove #(= "fhir.core" (:type %)) packages) + + duplicates (reduce (fn [duplicates [k v]] + (if (= (count v) 1) + duplicates + (assoc duplicates k v))) + {} (group-by :name extra))] + (cond + (> (count duplicates) 0) + (throw (ex-info "Found more then one package with same name" + {:packages (keys duplicates)})) + + :else + extra)) + + (remove #(= "fhir.core" (:type %)) packages)) + +(defn find-core-package-mismatch + "Finds packages which do not support a core package version." + [version packages] + (->> packages + (mapv (fn [package] + (assoc package :match-with-core? + (if (> (count (:fhirVersions package)) 0) + (->> (:fhirVersions package) + (map #(versions-match? version %)) + (some #{true}) + (boolean)) + true)))) + + (filterv #(not (:match-with-core? %))))) + +(defn- find-failed-dependencies + "Finds failed dependencies for the package looking up in all packages. + In case of failure will return a vec of failed dependencies." + [packages {:keys [dependencies]}] + (reduce (fn [mismatch dependency] + (let [dependency + (if (str/starts-with? dependency ":") + (subs dependency 1) + dependency) + + [dep-name dep-version] + (str/split dependency #"#") + + found + (->> packages + (filterv #(= (:name %) dep-name)) + (mapv #(simplify-package-meta %)))] + + (if (and (= (count found) 1) + (every? #(= (:version %) dep-version) found)) + mismatch + (conj mismatch {:required {:name dep-name + :version dep-version} + :found found})))) + [] dependencies)) + +(defn find-dependencies-mismatch + "Finds packages with failed dependencies check." + [packages] + (reduce (fn [mismatches package] + (let [failed-dependencies (find-failed-dependencies packages package)] + (if-not (empty? failed-dependencies) + (->> failed-dependencies + (assoc package :failed-dependencies) + (conj mismatches)) + mismatches))) + [] packages)) + +(defn check-compatibility! [packages] + (println "Verifying compatibility...") + (let [all (map extract-meta-from-package packages) + core (find-core-package all) + extra (find-extra-packages all)] + (println "✅ Core package found:" (simplify-package-meta core)) + + (println "Checking core version match...") + (let [core-mismatch (find-core-package-mismatch (:version core) extra)] + (when-not (empty? core-mismatch) + (throw (ex-info "Some packages do not match with core version" + {:version (:version core) + :packages (mapv :name core-mismatch)})))) + (println "✅ Core version match check passed") + + (println "Checking dependencies match...") + (let [deps-mismatch (find-dependencies-mismatch all)] + (when-not (empty? deps-mismatch) + (throw (ex-info "Some packages failed dependencies check" + {:packages (mapv #(select-keys % [:name :failed-dependencies]) + deps-mismatch)})))) + (println "✅ Dependencies match check passed")) + packages) diff --git a/tests/aidbox_sdk/generator_test.clj b/test/aidbox_sdk/generator_test.clj similarity index 100% rename from tests/aidbox_sdk/generator_test.clj rename to test/aidbox_sdk/generator_test.clj diff --git a/test/aidbox_sdk/schema/verify_test.clj b/test/aidbox_sdk/schema/verify_test.clj new file mode 100644 index 0000000..110a644 --- /dev/null +++ b/test/aidbox_sdk/schema/verify_test.clj @@ -0,0 +1,153 @@ +(ns aidbox-sdk.schema.verify-test + (:require [aidbox-sdk.schema.verify :as sut :refer [simplify-package-meta]] + [matcher-combinators.test :refer [match? thrown-match?]] + [clojure.test :refer [deftest is testing]])) + +(def r4-core-package + {:name "hl7.fhir.r4.core" + :version "4.0.1" + :fhirVersions ["4.0.1"] + :type "fhir.core" + :dependencies []}) + +(def us-core-package + {:name "hl7.fhir.us.core" + :version "4.1.0" + :fhirVersions ["4.0.1"] + :type "fhir.ig" + :dependencies [":hl7.fhir.r4.core#4.0.1" + ":us.nlm.vsac#0.3.0" + ":hl7.fhir.uv.sdc#2.7.0"]}) + +(def us-mcode-package + {:name "hl7.fhir.us.mcode" + :version "2.1.0" + :fhirVersions ["4.0.1"] + :type "fhir.ig" + :dependencies [":hl7.fhir.r4.core#4.0.1" + ":hl7.terminology.r4#5.0.0"]}) + +(deftest versions-match?-test + (testing "version match according to semver" + (is (sut/versions-match? "1.0.0" "1.0.0")) + (is (sut/versions-match? "1.0.1" "1.0")) + + (is (not (sut/versions-match? "1.1" "1.0"))) + (is (not (sut/versions-match? "2.0" "1.0"))) + + (is (not (sut/versions-match? "2" "2.0.0"))))) + +(deftest extract-meta-from-package-test + (testing "TODO" + (is true))) + +(deftest find-core-package-test + (testing "throws an exception if no core package found" + (is (thrown-match? clojure.lang.ExceptionInfo + {} + (sut/find-core-package [us-core-package])))) + + (testing "throws an exception if more then one core package found" + (let [r5-core-package {:name "hl7.fhir.r5.core" + :version "5.0.0" + :type "fhir.core"}] + (is (thrown-match? clojure.lang.ExceptionInfo + {:packages [(simplify-package-meta r4-core-package) + (simplify-package-meta r5-core-package)]} + (sut/find-core-package [r4-core-package + r5-core-package]))))) + + (testing "returns a core package iff there is only one" + (is (match? (sut/find-core-package [r4-core-package + us-core-package]) + r4-core-package)))) + +(deftest find-extra-packages-test + (testing "throws an exception if there is duplicate" + (let [us-core-package' (assoc us-core-package :version "5.1.0")] + (is (thrown-match? clojure.lang.ExceptionInfo + {:packages [(:name us-core-package)]} + (sut/find-extra-packages [r4-core-package + us-core-package + us-core-package']))))) + + (testing "returns extra packages if there is no duplicates" + (is (match? (sut/find-extra-packages [r4-core-package + us-core-package]) + [us-core-package])))) + +(deftest find-core-package-mismatch-test + (testing "returns a vec with mismatched names" + (let [us-mcode-package (assoc us-mcode-package :fhirVersions ["4.1.0"])] + (is (match? (sut/find-core-package-mismatch "4.0.1" + [us-core-package + us-mcode-package]) + [(assoc us-mcode-package :match-with-core? false)])))) + + (testing "returns an empty vec when fhirVersions is not specified" + (let [us-mcode-package (assoc us-mcode-package :fhirVersions [])] + (is (match? (sut/find-core-package-mismatch "4.0.1" + [us-core-package + us-mcode-package]) + [])))) + + (testing "returns an empty vec when all matched" + (is (match? (sut/find-core-package-mismatch "4.0.1" + [us-core-package + us-mcode-package]) + []))) + + (testing "returns an empty vec when major and minor match" + (is (match? (sut/find-core-package-mismatch "4.0" + [us-core-package + us-mcode-package]) + [])))) + +(deftest find-dependencies-mismatch-test + (let [us-vsac-package {:name "us.nlm.vsac", :version "0.3.0"} + uv-sdc-package {:name "hl7.fhir.uv.sdc", :version "2.7.0"} + term-package {:name "hl7.terminology.r4", :version "5.0.0"}] + (testing "returns a vec with mismatched names and version" + (testing "when required package was not found" + (is (match? (sut/find-dependencies-mismatch [r4-core-package + us-core-package + us-mcode-package]) + [(assoc us-core-package :failed-dependencies + [{:required (simplify-package-meta us-vsac-package) + :found []} + {:required (simplify-package-meta uv-sdc-package) + :found []}]) + + (assoc us-mcode-package :failed-dependencies + [{:required (simplify-package-meta term-package) + :found []}])]))) + + (testing "when required package is found, but version do not match" + (let [uv-sdc-package' (assoc uv-sdc-package :version "2.8.0") + term-package' (assoc term-package :version "5.1.0")] + (is (match? (sut/find-dependencies-mismatch [r4-core-package + us-core-package + us-mcode-package + us-vsac-package + uv-sdc-package' + term-package']) + [(assoc us-core-package :failed-dependencies + [{:required (simplify-package-meta uv-sdc-package) + :found [(simplify-package-meta uv-sdc-package')]}]) + + (assoc us-mcode-package :failed-dependencies + [{:required (simplify-package-meta term-package) + :found [(simplify-package-meta term-package')]}])]))))) + + (testing "returns an empty vec when no dependencies" + (is (match? (sut/find-dependencies-mismatch [r4-core-package]) + []))) + + (testing "returns an empty vec when all matched" + (is (match? (sut/find-dependencies-mismatch [r4-core-package + us-core-package + us-mcode-package + us-vsac-package + uv-sdc-package + term-package]) + []))))) diff --git a/tests.edn b/tests.edn index c85a956..d48a309 100644 --- a/tests.edn +++ b/tests.edn @@ -2,7 +2,7 @@ :kaocha.testable/id :unit :kaocha/ns-patterns ["-test$"] :kaocha/source-paths ["src"] - :kaocha/test-paths ["tests"]}] + :kaocha/test-paths ["test"]}] :kaocha/fail-fast? false :kaocha/color? true :kaocha/reporter [#_kaocha.report/dots