Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#7] Verify dependencies #9

Merged
merged 10 commits into from
Jul 22, 2024
Prev Previous commit
Next Next commit
feat(#7): check dependencies
worm2fed committed Jul 22, 2024

Verified

This commit was signed with the committer’s verified signature.
worm2fed Andrii Demydenko
commit aff4f2683de1072681bcba7ffebbebbfcf00210f
5 changes: 3 additions & 2 deletions deps.edn
Original file line number Diff line number Diff line change
@@ -10,8 +10,9 @@

:test {:extra-paths ["test"]
:exec-fn kaocha.runner/exec-fn
:extra-deps {com.health-samurai/matcho {:mvn/version "0.3.11"}
lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
:extra-deps {com.health-samurai/matcho {:mvn/version "0.3.11"}
lambdaisland/kaocha {:mvn/version "1.91.1392"}
nubank/matcher-combinators {:mvn/version "3.9.1"}}}

:build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.5"}}
:ns-default build}}}
8 changes: 5 additions & 3 deletions src/aidbox_sdk/core.clj
Original file line number Diff line number Diff line change
@@ -30,6 +30,8 @@
(println "Error: please provide an output argument"))

:else
(generator/build-all!
(resource input)
(io/as-file output)))))
(do
(println "Building FHIR SDK...")
(generator/build-all!
(resource input)
(io/as-file output))))))
24 changes: 15 additions & 9 deletions src/aidbox_sdk/schema.clj
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
(ns aidbox-sdk.schema
(:require [clojure.data.json :as json]
(:require [aidbox-sdk.schema.verify :as verify]
[clojure.data.json :as json]
[clojure.java.io :as io]
[clojure.string :as str]))

@@ -8,13 +9,15 @@
NOTE: right now it'll filter out all files which do not contains hl7.fhir
in their name."
[path]
(->> path
file-seq
(remove #(.isDirectory %))
;; FIXME is this really good approach to determine packages?
(filter #(str/includes? (.getName %) "hl7.fhir"))
;; FIXME only gzip, but there is no problem to accept unpacked ndjson
(filter #(str/ends-with? (.getName %) ".gz"))))
(let [packages (->> path
file-seq
(remove #(.isDirectory %))
;; FIXME is this really good approach to determine packages?
(filter #(str/includes? (.getName %) "hl7.fhir"))
;; FIXME only gzip, but there is no problem to accept unpacked ndjson
(filter #(str/ends-with? (.getName %) ".gz")))]
(println "✅ Found packages:" (count packages))
packages))

(defn create-gzip-reader [path]
(-> path
@@ -23,6 +26,7 @@
(io/reader)))

(defn parse-package [path]
(println "Parsing package:" (str path))
(with-open [rdr (create-gzip-reader path)]
(->> rdr
line-seq
@@ -51,8 +55,10 @@
;; ! it's possible to create a File instance from url, which may lead to bugs
(defmethod retrieve java.io.File
[source]
(println "Retrieving packages from: " (str source))
(->> (get-packages-from-directory source)
(map parse-package)
(mapv parse-package)
(verify/check-compatibility!)
(flatten)
(remove-invalid-schemas)
(prepare-schemas)
98 changes: 98 additions & 0 deletions src/aidbox_sdk/schema/verify.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
(ns aidbox-sdk.schema.verify
(:require [clojure.string :as str]))

;; FIXME: is it reliable to use first element of the list?
;; ! seems like it's not for original packages (without Aidbox processing).
(defn extract-meta-from-package
"Extracts meta information from the package."
[package]
(first package))

(defn find-core-package
"Finds core package in the list of packages.
Throws an exception if there are more then one core package."
[packages]
(let [cores (filter #(= "fhir.core" (:type %)) packages)
core (first cores)]
(cond
(= (count cores) 0)
(throw (ex-info "No core package found" {}))

(> (count cores) 1)
(throw (ex-info "Found more then one core package"
{:packages (mapv :name cores)}))

:else
core)))

(defn find-extra-packages
"Finds extra packages in the list of packages."
[packages]
(remove #(= "fhir.core" (:type %)) packages))

(defn find-core-package-mismatch
"Finds packages which do not support a core package version."
[version packages]
(->> packages
(mapv #(assoc % :match-with-core? (-> (hash-set version)
(some (:fhirVersions %))
(boolean))))
(filterv #(not (:match-with-core? %)))))

(defn- find-failed-dependencies
"Finds failed dependencies for the package looking up in all packages.
In case of failure will return a vec of failed dependencies."
[packages {:keys [dependencies]}]
(reduce (fn [mismatch dependency]
(let [[dep-name dep-version]
(-> (subs dependency 1)
(str/split #"#"))

found
(->> packages
(filterv #(= (:name %) dep-name))
(mapv #(select-keys % [:name :version])))]

(if (and (= (count found) 1)
(every? #(= (:version %) dep-version) found))
mismatch
(conj mismatch {:required {:name dep-name
:version dep-version}
:found found}))))
[] dependencies))

(defn find-dependencies-mismatch
"Finds packages with failed dependencies check."
[packages]
(reduce (fn [mismatches package]
(let [failed-dependencies (find-failed-dependencies packages package)]
(if-not (empty? failed-dependencies)
(->> failed-dependencies
(assoc package :failed-dependencies)
(conj mismatches))
mismatches)))
[] packages))

(defn check-compatibility! [packages]
(println "Verifying compatibility...")
(let [all (map extract-meta-from-package packages)
core (find-core-package all)
extra (find-extra-packages all)]
(println "✅ Core package found:" (:name core))

(println "Checking core version match...")
(let [core-mismatch (find-core-package-mismatch (:version core) extra)]
(when-not (empty? core-mismatch)
(throw (ex-info "Some packages do not match with core version"
{:version (:version core)
:packages (mapv :name core-mismatch)}))))
(println "✅ Core version match check passed")

(println "Checking dependencies match...")
(let [deps-mismatch (find-dependencies-mismatch all)]
(when-not (empty? deps-mismatch)
(throw (ex-info "Some packages failed dependencies check"
{:packages (mapv #(select-keys % [:name :failed-dependencies])
deps-mismatch)}))))
(println "✅ Dependencies match check passed"))
packages)
116 changes: 116 additions & 0 deletions test/aidbox_sdk/schema/verify_test.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
(ns aidbox-sdk.schema.verify-test
(:require [aidbox-sdk.schema.verify :as sut]
[matcher-combinators.test :refer [match? thrown-match?]]
[clojure.test :refer [deftest is testing]]))

(def r4-core-package
{:name "hl7.fhir.r4.core"
:version "4.0.1"
:fhirVersions ["4.0.1"]
:type "fhir.core"
:dependencies []})

(def us-core-package
{:name "hl7.fhir.us.core"
:version "4.1.0"
:fhirVersions ["4.0.1"]
:type "fhir.ig"
:dependencies [":hl7.fhir.r4.core#4.0.1"
":us.nlm.vsac#0.3.0"
":hl7.fhir.uv.sdc#2.7.0"]})

(def us-mcode-package
{:name "hl7.fhir.us.mcode"
:version "2.1.0"
:fhirVersions ["4.0.1"]
:type "fhir.ig"
:dependencies [":hl7.fhir.r4.core#4.0.1"
":hl7.terminology.r4#5.0.0"]})

(deftest extract-meta-from-package
(testing "TODO"
(is true)))

(deftest find-core-package-test
(testing "throws an exception if no core package found"
(is (thrown-match? clojure.lang.ExceptionInfo
{}
(sut/find-core-package [us-core-package]))))

(testing "throws an exception if more then one core package found"
(is (thrown-match? clojure.lang.ExceptionInfo
{:packages [(:name r4-core-package) "hl7.fhir.r5.core"]}
(sut/find-core-package [r4-core-package
{:name "hl7.fhir.r5.core"
:version "5.0.0"
:type "fhir.core"}]))))

(testing "returns a core package iff there is only one"
(is (match? (sut/find-core-package [r4-core-package
us-core-package])
r4-core-package))))

(deftest find-core-package-mismatch-test
(testing "returns a vec with mismatched names"
(let [us-mcode-package (assoc us-mcode-package :fhirVersions ["4.1.0"])]
(is (match? (sut/find-core-package-mismatch "4.0.1"
[us-core-package
us-mcode-package])
[(assoc us-mcode-package :match-with-core? false)]))))

(testing "returns an empty vec when all matched"
(is (match? (sut/find-core-package-mismatch "4.0.1"
[us-core-package
us-mcode-package])
[]))))

(deftest find-dependencies-mismatch-test
(let [format-dep-for-check #(select-keys % [:name :version])

us-vsac-package {:name "us.nlm.vsac", :version "0.3.0"}
uv-sdc-package {:name "hl7.fhir.uv.sdc", :version "2.7.0"}
term-package {:name "hl7.terminology.r4", :version "5.0.0"}]
(testing "returns a vec with mismatched names and version"
(testing "when required package was not found"
(is (match? (sut/find-dependencies-mismatch [r4-core-package
us-core-package
us-mcode-package])
[(assoc us-core-package :failed-dependencies
[{:required (format-dep-for-check us-vsac-package)
:found []}
{:required (format-dep-for-check uv-sdc-package)
:found []}])

(assoc us-mcode-package :failed-dependencies
[{:required (format-dep-for-check term-package)
:found []}])])))

(testing "when required package is found, but version do not match"
(let [uv-sdc-package' (assoc uv-sdc-package :version "2.8.0")
term-package' (assoc term-package :version "5.1.0")]
(is (match? (sut/find-dependencies-mismatch [r4-core-package
us-core-package
us-mcode-package
us-vsac-package
uv-sdc-package'
term-package'])
[(assoc us-core-package :failed-dependencies
[{:required (format-dep-for-check uv-sdc-package)
:found [(format-dep-for-check uv-sdc-package')]}])

(assoc us-mcode-package :failed-dependencies
[{:required (format-dep-for-check term-package)
:found [(format-dep-for-check term-package')]}])])))))

(testing "returns an empty vec when no dependencies"
(is (match? (sut/find-dependencies-mismatch [r4-core-package])
[])))

(testing "returns an empty vec when all matched"
(is (match? (sut/find-dependencies-mismatch [r4-core-package
us-core-package
us-mcode-package
us-vsac-package
uv-sdc-package
term-package])
[])))))