Artifacts support (#172)

* add Julia artifacts

Co-authored-by: matcox <[email protected]>

.github/workflows/run_windows.yml

@@ -102,6 +102,11 @@ jobs:
       with:
         version: 1.3.1
     # BoardController testing
+    - name: Run Julia Tests
+      run: |
+        cd %GITHUB_WORKSPACE%\julia-package\brainflow
+        julia --project=. -e "import Pkg; Pkg.test()"
+      shell: cmd
     - name: Cyton Windows Python Test
       run: python %GITHUB_WORKSPACE%\emulator\brainflow_emulator\cyton_windows.py python %GITHUB_WORKSPACE%\tests\python\brainflow_get_data.py --log --board-id 0 --serial-port
       shell: cmd
@@ -214,11 +219,6 @@ jobs:
     - name: EEG Metrics CI C# Test
       run: .\csharp-package\brainflow\eeg_metrics\bin\Release\test.exe
       shell: cmd
-    - name: Run Julia Tests
-      run: |
-        cd %GITHUB_WORKSPACE%\julia-package\brainflow
-        julia --project=. -e "import Pkg; Pkg.test()"
-      shell: cmd
     # Start Deploy Stage
     - name: Install Python AWS Tools
       run: |

docs/BuildBrainFlow.rst

@@ -94,24 +94,16 @@ Steps to setup Matlab binding for BrainFlow:
 Julia
 --------
 
-Steps to setup Julia binding for BrainFlow:
-
-- Compile Core Module, using instructions below
-- Set PATH(on Windows) or LD_LIBRARY_PATH(on Unix) env variables to ensure that compiled libraries are in search path
-- Install BrainFlow package locally
+brainflow is a registered package in the Julia general registry, so it can be installed via the Pkg manager:
 
 .. compound::
 
     Example: ::
 
-        # compile core module first
-        # set env variable
-        export LD_LIBRARY_PATH=/home/andreyparfenov/brainflow/installed_linux/lib/:$LD_LIBRARY_PATH
-        cd julia-package/brainflow
-        julia
-        # type ']' to switch to pkg terminal
-        activate . # activate BrainFlow's env
+        import Pkg
+		Pkg.add("brainflow")
 
+When using brainflow for the first time in Julia, the brainflow artifact containing the compiled brainflow libraries will be downloaded.
 
 Docker Image
 --------------

julia-package/brainflow/Artifacts.toml

@@ -0,0 +1,7 @@
+[brainflow]
+git-tree-sha1 = "1681bc5c836c0465e02c42dce71db9572d723b6e"
+lazy = true
+
+    [[brainflow.download]]
+    sha256 = "92bd7485de8ccbd19d27a9b203e8176806269415b1087c767bb36539efe063d9"
+    url = "https://github.com/brainflow-dev/brainflow/releases/download/3.7.2/compiled_libs.tar"

julia-package/brainflow/Project.toml

@@ -1,11 +1,14 @@
 name = "brainflow"
 uuid = "abd5acf9-0fc9-4730-984d-e27a37823580"
 authors = ["Andrey1994 <[email protected]>"]
-version = "0.1.0"
+version = "3.7.2"
 
 [deps]
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 
 [compat]
-julia = "1.2"
+julia = "1.3"

julia-package/brainflow/generate_brainflow_artifact.jl

@@ -0,0 +1,45 @@
+# when new brainflow binaries are released, then this script needs to be re-executed.
+
+using SHA
+using Pkg
+using Pkg.Artifacts
+using Tar
+
+function sha256sum(tarball_path)
+    return open(tarball_path, "r") do io
+        return bytes2hex(sha256(io))
+    end
+end
+
+function add_brainflow_artifact!(
+    url,
+    artifact_toml_path = "Artifacts.toml",
+    artifact_name = "brainflow",
+    )
+
+    download_path = "$(tempname())-download.tar"
+    download(url, download_path)
+    tar_hash_sha256 = sha256sum(download_path)
+
+    brainflow_hash = create_artifact() do artifact_dir 
+        # Pkg.PlatformEngines.unpack() gives errors for some users on Windows
+        Tar.extract(download_path, artifact_dir)
+    end
+
+    rm(download_path)
+
+    Pkg.Artifacts.bind_artifact!(
+        artifact_toml_path, 
+        artifact_name, 
+        brainflow_hash; 
+        download_info=[(url, tar_hash_sha256)], 
+        force=true,
+        lazy=true,
+    )
+
+    return brainflow_hash
+end
+
+cd(@__DIR__)
+include("src/brainflow_url.jl")
+add_brainflow_artifact!(brainflow_url())

julia-package/brainflow/src/brainflow.jl

@@ -1,6 +1,7 @@
 module brainflow
 
 include("errors.jl")
+include("brainflow_url.jl")
 include("c_interfaces.jl")
 include("brainflow_logs.jl")
 include("board_shim.jl")

julia-package/brainflow/src/brainflow_url.jl

@@ -0,0 +1,5 @@
+# TODO: automatically generate by brainflow CICD
+function brainflow_url()
+    url = "https://github.com/brainflow-dev/brainflow/releases/download/3.7.2/compiled_libs.tar"
+    return url
+end

julia-package/brainflow/src/c_interfaces.jl

@@ -1,3 +1,49 @@
+using Pkg
+using Pkg.Artifacts
+using SHA
+using Tar
+
+# we have an issue with unpack(), so wrote a custom download
+# https://discourse.julialang.org/t/unable-to-automatically-install-artifact/51984/2
+function download_brainflow_artifact()
+
+    url = brainflow_url()
+
+    download_path = "$(tempname())-download.tar"
+    download(url, download_path)
+
+    brainflow_hash = create_artifact() do artifact_dir 
+        # Pkg.PlatformEngines.unpack() gives errors for some users on Windows
+        Tar.extract(download_path, artifact_dir)
+    end
+
+    rm(download_path)
+
+    return brainflow_hash
+end
+
+# library path you can optionally use while developing
+# please then copy the compiled libraries into julia-package/brainflow/lib
+function dev_library_path()
+    return abspath(joinpath(@__DIR__, "../lib"))
+end
+
+function get_brainflow_artifact_path()
+    artifacts_toml = find_artifacts_toml(@__DIR__) # is there a better way? @__DIR__ makes brainflow non-relocatable I believe.
+    brainflow_hash = artifact_hash("brainflow", artifacts_toml)
+    if isdir(dev_library_path())
+        # developer library takes precedence
+        return dev_library_path()
+    elseif artifact_exists(brainflow_hash)
+        # libraries are automatically stored here for users of brainflow
+        return artifact_path(brainflow_hash)
+    else
+        println("Downloading artifact: brainflow")
+        brainflow_hash = download_brainflow_artifact()
+        return artifact_path(brainflow_hash)
+    end
+end
+
 function interface_path(library::AbstractString)
     return abspath(INTERFACE_PATH, interface_name(library))
 end
@@ -12,13 +58,8 @@ function interface_name(library::AbstractString)
     end
 end
 
-# We can later replace this with:
-# using Pkg.Artifacts
-# const INTERFACE_PATH = artifact"brainflow"
-# Right now using a hardcoded path during refactoring, with manual /lib location
-const INTERFACE_PATH = abspath(joinpath(@__DIR__, "../lib"))
+const INTERFACE_PATH = get_brainflow_artifact_path()
 
-# we should instead use dlopen(interface_path())?
 const DATA_HANDLER_INTERFACE = interface_path("DataHandler")
 const BOARD_CONTROLLER_INTERFACE = interface_path("BoardController")
 const ML_MODULE_INTERFACE = interface_path("MLModule")