From 582227ea40169a5bcc5882bcbfb62f4e535c0792 Mon Sep 17 00:00:00 2001 From: Vitaly Kravtsov Date: Tue, 23 Jul 2024 17:23:43 +0600 Subject: [PATCH] feat(#3): add schemas downloading from aidbox url --- deps.edn | 3 +- dev/user.clj | 3 +- src/aidbox_sdk/core.clj | 4 +- src/aidbox_sdk/generator.clj | 45 +++++++++----------- src/aidbox_sdk/generator/helpers.clj | 7 ++++ src/aidbox_sdk/schema.clj | 61 +++++++++++++++++++++++----- 6 files changed, 83 insertions(+), 40 deletions(-) diff --git a/deps.edn b/deps.edn index ba4bb2a..42ed1a9 100644 --- a/deps.edn +++ b/deps.edn @@ -1,6 +1,5 @@ {:paths ["src"] - :deps {org.clojure/data.json {:mvn/version "2.5.0"} - + :deps {clj-http/clj-http {:mvn/version "3.13.0"} com.github.clj-easy/graal-build-time {:mvn/version "1.0.5"} org.clojure/clojure {:mvn/version "1.11.3"} org.clojure/tools.cli {:mvn/version "1.1.230"}} diff --git a/dev/user.clj b/dev/user.clj index 2388381..b7d8b31 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -61,7 +61,6 @@ (count (gen/apply-constraints mcodes bases)) - (vec (gen/retrieve-schemas' source')) (contains? (->> (gen/retrieve-schemas' source') (filter gen/base-schema?) (gen/prepared-schemas) @@ -71,6 +70,6 @@ (vector-to-map)) "http://hl7.org/fhir/StructureDefinition/Observation") - (gen/build-all! source' target) + (gen/build-all! (io/as-url "http://localhost:8765/sdk/fhir-packages") target) :rcf) diff --git a/src/aidbox_sdk/core.clj b/src/aidbox_sdk/core.clj index 9908dd4..f350d85 100644 --- a/src/aidbox_sdk/core.clj +++ b/src/aidbox_sdk/core.clj @@ -34,4 +34,6 @@ (println "Building FHIR SDK...") (generator/build-all! (resource input) - (io/as-file output)))))) + (io/as-file output)) + (println "Finished succesfully!") + (System/exit 0))))) diff --git a/src/aidbox_sdk/generator.clj b/src/aidbox_sdk/generator.clj index 1e7c098..d0a7ee1 100644 --- a/src/aidbox_sdk/generator.clj +++ b/src/aidbox_sdk/generator.clj @@ -1,15 +1,16 @@ (ns aidbox-sdk.generator (:refer-clojure :exclude [namespace]) - (:require [aidbox-sdk.generator.dotnet.templates :as dotnettpl] - [aidbox-sdk.generator.helpers :refer [->pascal-case safe-conj - uppercase-first-letter - vector-to-map]] - [aidbox-sdk.schema :as schema] - [clojure.java.io :as io] - [clojure.set :as set] - [clojure.string :as str] - [clojure.walk]) - (:import [java.util.zip ZipEntry ZipOutputStream])) + (:require + [aidbox-sdk.generator.dotnet.templates :as dotnettpl] + [aidbox-sdk.generator.helpers :refer [->pascal-case + safe-conj + uppercase-first-letter + vector-to-map]] + [aidbox-sdk.schema :as schema] + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.string :as str] + [clojure.walk])) ;; ;; FHIR @@ -645,20 +646,6 @@ (delete-directory! dir) (create-directory! dir)) -;; FIXME do we need it? -(defn zip-dir! [path zip-name] - (with-open [zip (ZipOutputStream. (io/output-stream zip-name))] - (doseq [f (file-seq (io/file path)) :when (.isFile f)] - (.putNextEntry zip (ZipEntry. (str/replace-first (.getPath f) path ""))) - (io/copy f zip) - (.closeEntry zip))) - (io/file zip-name)) - -;; FIXME do we need it? -(defn copy-files! [src-dir target-dir] - (doseq [file (remove #(.isDirectory %) (file-seq src-dir))] - (io/copy file (io/file target-dir (.getName file))))) - ;; ;; main ;; @@ -703,7 +690,9 @@ (not (from-extension? %)))))] (prepare-target-directory! output) + (println "---") ;; create base namespace (all FHIR datatypes) file + (println "Generating base namespace") (->> all-schemas (filter base-schema?) (prepared-schemas) @@ -713,6 +702,7 @@ (io/file output "Base.cs"))) ;; create spezialization files + (println "Generating resource classes") (doseq [item (->> all-schemas (filter base-schema?) (filter domain-resource?) @@ -727,6 +717,7 @@ (generate-resource-namespace item))) ;; create resource map file + (println "Generating resource map") (->> all-schemas (filter base-schema?) (filter domain-resource?) @@ -751,6 +742,7 @@ (:class-file-content item))) ;; create constraints + (println "Generating constraints classes") (doseq [{:keys [name schema file-content]} (->> (apply-constraints constraints @@ -767,8 +759,11 @@ (assoc schema :url name'))})))] (save-to-file! - (io/file output (package->directory (:package schema)) (str (->pascal-case (url->resource-type name)) ".cs")) + (io/file output + (package->directory (:package schema)) + (str (->pascal-case (url->resource-type name)) ".cs")) file-content)) + (println "Generating common SDK files") (doseq [file dotnettpl/files] (spit (io/file output (:name file)) (:content file))))) diff --git a/src/aidbox_sdk/generator/helpers.clj b/src/aidbox_sdk/generator/helpers.clj index e9c2dc1..3860776 100644 --- a/src/aidbox_sdk/generator/helpers.clj +++ b/src/aidbox_sdk/generator/helpers.clj @@ -1,5 +1,6 @@ (ns aidbox-sdk.generator.helpers (:require + [clojure.data.json :as json] [clojure.string :as str])) (defn words @@ -24,3 +25,9 @@ (into {}))) (defn safe-conj [a b] (conj a (or b {}))) + +(defn rand-int-between [min max] + (int (+ min (Math/floor (rand (- max min)))))) + +(defn parse-json [s] + (json/read-str s :key-fn keyword)) diff --git a/src/aidbox_sdk/schema.clj b/src/aidbox_sdk/schema.clj index 76381de..462e50b 100644 --- a/src/aidbox_sdk/schema.clj +++ b/src/aidbox_sdk/schema.clj @@ -1,13 +1,12 @@ (ns aidbox-sdk.schema (:require [aidbox-sdk.schema.verify :as verify] - [clojure.data.json :as json] + [aidbox-sdk.generator.helpers :refer [rand-int-between parse-json]] + [clj-http.client :as http.client] [clojure.java.io :as io] [clojure.string :as str])) (defn get-packages-from-directory - "Returns all packages in the given directory, including files in subdirectories. - NOTE: right now it'll filter out all files which do not contains hl7.fhir - in their name." + "Returns all packages in the given directory, including files in subdirectories. " [path] (let [packages (->> path file-seq @@ -28,11 +27,10 @@ ;; see https://github.com/Aidbox/aidbox-sdk/issues/10 (defn parse-package [path] (println "Parsing package:" (str path)) - (with-open [rdr (create-gzip-reader path)] - (->> rdr + (with-open [reader (create-gzip-reader path)] + (->> reader line-seq - (mapv (fn [line] - (json/read-str line :key-fn keyword)))))) + (mapv parse-json)))) (defn remove-invalid-schemas [schemas] (remove #(nil? (:package-meta %)) schemas)) @@ -48,7 +46,6 @@ (assoc % :package)) schemas)) - (defmulti retrieve class) ;; ! According to an example here: @@ -65,6 +62,50 @@ (prepare-schemas) (merge-duplicates))) +(defn- next-timeout + "Timeout calculation for retrying like in kafka. + https://kafka.js.org/docs/retry-detailed" + [timeout] + (let [factor 0.2 + multiplier 2] + (* (rand-int-between + (* timeout (- 1 factor)) + (* timeout (+ 1 factor))) + multiplier))) + +(defn- retry [f & {:keys [timeout trials] + :or {timeout 3000 + trials 3}}] + (if (zero? (dec trials)) + (f) + (try + (f) + (catch Throwable _ + (Thread/sleep timeout) + (retry f {:timeout (next-timeout timeout) + :trials (dec trials)}))))) + +(defn fetch-n-parse [url] + (let [url-string (if (instance? java.net.URL url) + (.toString url) + url) + result (retry #(http.client/get url-string))] + (some-> result :body parse-json))) + +(defn skip-root-package [packages] + (rest packages)) + (defmethod retrieve java.net.URL [source] - (do "something")) + (let [extract-link (fn [package] (-> package :href io/as-url)) + extract-name (fn [package] (str (:name package) "#" (:version package))) + fhir-packages (do + (println "Downloading list of dependencies from:" (.toString source)) + (-> (fetch-n-parse source) + (skip-root-package)))] + + (->> fhir-packages + (pmap (fn [package] + (println "Downloading schemas for:" (extract-name package)) + (fetch-n-parse (extract-link package)))) + (flatten))))