feat(#3): add schemas downloading from aidbox url

deps.edn

@@ -1,6 +1,5 @@
 {:paths ["src"]
- :deps {org.clojure/data.json                {:mvn/version "2.5.0"}
-
+ :deps {clj-http/clj-http                    {:mvn/version "3.13.0"}
         com.github.clj-easy/graal-build-time {:mvn/version "1.0.5"}
         org.clojure/clojure                  {:mvn/version "1.11.3"}
         org.clojure/tools.cli                {:mvn/version "1.1.230"}}

dev/user.clj

@@ -61,7 +61,6 @@
   (count (gen/apply-constraints mcodes
                                 bases))
 
-  (vec (gen/retrieve-schemas' source'))
   (contains? (->> (gen/retrieve-schemas' source')
                   (filter gen/base-schema?)
                   (gen/prepared-schemas)
@@ -71,6 +70,6 @@
                   (vector-to-map))
              "http://hl7.org/fhir/StructureDefinition/Observation")
 
-  (gen/build-all! source' target)
+  (gen/build-all! (io/as-url "http://localhost:8765/sdk/fhir-packages")  target)
 
   :rcf)

src/aidbox_sdk/core.clj

@@ -34,4 +34,6 @@
         (println "Building FHIR SDK...")
         (generator/build-all!
          (resource input)
-         (io/as-file output))))))
+         (io/as-file output))
+        (println "Finished succesfully!")
+        (System/exit 0)))))

src/aidbox_sdk/generator.clj

@@ -1,15 +1,16 @@
 (ns aidbox-sdk.generator
   (:refer-clojure :exclude [namespace])
-  (:require [aidbox-sdk.generator.dotnet.templates :as dotnettpl]
-            [aidbox-sdk.generator.helpers :refer [->pascal-case safe-conj
-                                                  uppercase-first-letter
-                                                  vector-to-map]]
-            [aidbox-sdk.schema :as schema]
-            [clojure.java.io :as io]
-            [clojure.set :as set]
-            [clojure.string :as str]
-            [clojure.walk])
-  (:import [java.util.zip ZipEntry ZipOutputStream]))
+  (:require
+   [aidbox-sdk.generator.dotnet.templates :as dotnettpl]
+   [aidbox-sdk.generator.helpers :refer [->pascal-case
+                                         safe-conj
+                                         uppercase-first-letter
+                                         vector-to-map]]
+   [aidbox-sdk.schema :as schema]
+   [clojure.java.io :as io]
+   [clojure.set :as set]
+   [clojure.string :as str]
+   [clojure.walk]))
 
 ;;
 ;; FHIR
@@ -645,20 +646,6 @@
   (delete-directory! dir)
   (create-directory! dir))
 
-;; FIXME do we need it?
-(defn zip-dir! [path zip-name]
-  (with-open [zip (ZipOutputStream. (io/output-stream zip-name))]
-    (doseq [f (file-seq (io/file path)) :when (.isFile f)]
-      (.putNextEntry zip (ZipEntry. (str/replace-first (.getPath f) path "")))
-      (io/copy f zip)
-      (.closeEntry zip)))
-  (io/file zip-name))
-
-;; FIXME do we need it?
-(defn copy-files! [src-dir target-dir]
-  (doseq [file (remove #(.isDirectory %) (file-seq src-dir))]
-    (io/copy file (io/file target-dir (.getName file)))))
-
 ;;
 ;; main
 ;;
@@ -703,7 +690,9 @@
                                              (not (from-extension? %)))))]
     (prepare-target-directory! output)
 
+    (println "---")
     ;; create base namespace (all FHIR datatypes) file
+    (println "Generating base namespace")
     (->> all-schemas
          (filter base-schema?)
          (prepared-schemas)
@@ -713,6 +702,7 @@
           (io/file output "Base.cs")))
 
     ;; create spezialization files
+    (println "Generating resource classes")
     (doseq [item (->> all-schemas
                       (filter base-schema?)
                       (filter domain-resource?)
@@ -727,6 +717,7 @@
        (generate-resource-namespace item)))
 
     ;; create resource map file
+    (println "Generating resource map")
     (->> all-schemas
          (filter base-schema?)
          (filter domain-resource?)
@@ -751,6 +742,7 @@
        (:class-file-content item)))
 
     ;; create constraints
+    (println "Generating constraints classes")
     (doseq [{:keys [name schema file-content]}
             (->> (apply-constraints
                   constraints
@@ -767,8 +759,11 @@
                                          (assoc schema
                                                 :url name'))})))]
       (save-to-file!
-       (io/file output (package->directory (:package schema)) (str (->pascal-case (url->resource-type name)) ".cs"))
+       (io/file output
+                (package->directory (:package schema))
+                (str (->pascal-case (url->resource-type name)) ".cs"))
        file-content))
 
+    (println "Generating common SDK files")
     (doseq [file dotnettpl/files]
       (spit (io/file output (:name file)) (:content file)))))

src/aidbox_sdk/generator/helpers.clj

@@ -1,5 +1,6 @@
 (ns aidbox-sdk.generator.helpers
   (:require
+   [clojure.data.json :as json]
    [clojure.string :as str]))
 
 (defn words
@@ -24,3 +25,9 @@
        (into {})))
 
 (defn safe-conj [a b] (conj a (or b {})))
+
+(defn rand-int-between [min max]
+  (int (+ min (Math/floor (rand (- max min))))))
+
+(defn parse-json [s]
+  (json/read-str s :key-fn keyword))

src/aidbox_sdk/schema.clj

@@ -1,13 +1,12 @@
 (ns aidbox-sdk.schema
   (:require [aidbox-sdk.schema.verify :as verify]
-            [clojure.data.json :as json]
+            [aidbox-sdk.generator.helpers :refer [rand-int-between parse-json]]
+            [clj-http.client :as http.client]
             [clojure.java.io :as io]
             [clojure.string :as str]))
 
 (defn get-packages-from-directory
-  "Returns all packages in the given directory, including files in subdirectories.
-   NOTE: right now it'll filter out all files which do not contains hl7.fhir
-   in their name."
+  "Returns all packages in the given directory, including files in subdirectories. "
   [path]
   (let [packages (->> path
                       file-seq
@@ -28,11 +27,10 @@
 ;; see https://github.com/Aidbox/aidbox-sdk/issues/10
 (defn parse-package [path]
   (println "Parsing package:" (str path))
-  (with-open [rdr (create-gzip-reader path)]
-    (->> rdr
+  (with-open [reader (create-gzip-reader path)]
+    (->> reader
          line-seq
-         (mapv (fn [line]
-                 (json/read-str line :key-fn keyword))))))
+         (mapv parse-json))))
 
 (defn remove-invalid-schemas [schemas]
   (remove #(nil? (:package-meta %)) schemas))
@@ -48,7 +46,6 @@
              (assoc  %  :package))
        schemas))
 
-
 (defmulti retrieve class)
 
 ;; ! According to an example here:
@@ -65,6 +62,50 @@
        (prepare-schemas)
        (merge-duplicates)))
 
+(defn- next-timeout
+  "Timeout calculation for retrying like in kafka.
+  https://kafka.js.org/docs/retry-detailed"
+  [timeout]
+  (let [factor 0.2
+        multiplier 2]
+    (* (rand-int-between
+        (* timeout (- 1 factor))
+        (* timeout (+ 1 factor)))
+       multiplier)))
+
+(defn- retry [f & {:keys [timeout trials]
+                   :or   {timeout 3000
+                          trials  3}}]
+  (if (zero? (dec trials))
+    (f)
+    (try
+      (f)
+      (catch Throwable _
+        (Thread/sleep timeout)
+        (retry f {:timeout (next-timeout timeout)
+                  :trials  (dec trials)})))))
+
+(defn fetch-n-parse [url]
+  (let [url-string (if (instance? java.net.URL url)
+                     (.toString url)
+                     url)
+        result     (retry #(http.client/get url-string))]
+    (some-> result :body parse-json)))
+
+(defn skip-root-package [packages]
+  (rest packages))
+
 (defmethod retrieve java.net.URL
   [source]
-  (do "something"))
+  (let [extract-link (fn [package] (-> package :href io/as-url))
+        extract-name (fn [package] (str (:name package) "#" (:version package)))
+        fhir-packages (do
+                        (println "Downloading list of dependencies from:" (.toString source))
+                        (-> (fetch-n-parse source)
+                            (skip-root-package)))]
+
+    (->> fhir-packages
+         (pmap (fn [package]
+                 (println "Downloading schemas for:" (extract-name package))
+                 (fetch-n-parse (extract-link package))))
+         (flatten))))