BOSS backend (#59)

* WIP, v1

* update Pipfile hashes

* use dataset info from BOSS, support resolutions

* typing & package versions

* update Pipfile.lock, but not dev-dependencies

* use datasets.json

* refactoring: extract Boss Client, models & token repo in extra files

* Update Pipfile

* extract handle_new_channel, minor refactorings

* use env-vars for neurodata.io credentials

* pretty

* pretty

* pretty

Pipfile

@@ -15,6 +15,7 @@ pylint = "*"
 [packages]
 aiohttp = "*"
 async-lru = "*"
+blosc = "*"
 compressed-segmentation = ">=1.0.0"
 jpeg4py = "*"
 numpy = "*"

README.md

@@ -10,8 +10,13 @@ A [webKnossos](https://github.com/scalableminds/webknossos) compatible data conn
   INSERT INTO "webknossos"."datastores"("name","url","key","isscratch","isdeleted","isforeign") 
   VALUES (E'connect',E'http://localhost:8000',E'k',FALSE,FALSE,FALSE);
   ```
-2. `docker-compose up --build webknossos-connect`
-3. By default, some public datasets are reported. Add more datasets from the webKnossos user interface.
+2. To use the initial datasets from [neurodata.io](https://neurodata.io/ndcloud/#data), create a `.env` file:
+  ```
+  NEURODATA_IO_USER="<your username>"
+  NEURODATA_IO_PW="<your password>"
+  ```
+3. `docker-compose up --build webknossos-connect`
+4. By default, some public datasets are reported. Add more datasets from the webKnossos user interface.
 
 ## Development
 ### In Docker :whale:

benchmarks/decode_jpg.py

@@ -3,18 +3,18 @@
 import numpy as np
 from aiohttp import ClientSession
 
-from wkconnect.backends.neuroglancer.backend import NeuroglancerBackend
+from wkconnect.backends.neuroglancer.backend import Neuroglancer
 from wkconnect.utils.types import Vec3D
 
-neuroglancer: NeuroglancerBackend
+neuroglancer: Neuroglancer
 data: np.ndarray
 
 
 async def setup() -> None:
     global neuroglancer
     global data
     http_client = await ClientSession().__aenter__()
-    neuroglancer = NeuroglancerBackend({}, http_client)
+    neuroglancer = Neuroglancer({}, http_client)
 
     data_url = "https://storage.googleapis.com/neuroglancer-public-data/kasthuri2011/image/24_24_30/896-960_1152-1216_1472-1536"
 

data/datasets.json

@@ -1,4 +1,22 @@
 {
+    "boss": {
+        "Connectomics_Department": {
+            "ara": {
+                "domain": "https://api.boss.neurodata.io",
+                "collection": "ara_2016",
+                "experiment": "sagittal_50um",
+                "username": "$NEURODATA_IO_USER",
+                "password": "$NEURODATA_IO_PW"
+            },
+            "kasthuri2011-neurodata-io": {
+                "domain": "https://api.boss.neurodata.io",
+                "collection": "kasthuri",
+                "experiment": "kasthuri11",
+                "username": "$NEURODATA_IO_USER",
+                "password": "$NEURODATA_IO_PW"
+            }
+        }
+    },
     "neuroglancer": {
         "Connectomics_Department": {
             "fafb_v14": {

stubs/compressed_segmentation.pyi

@@ -1,6 +1,7 @@
-import numpy as np
 from typing import Tuple
 
+import numpy as np
+
 def decompress(
     encoded: bytes,
     volume_size: Tuple[int, int, int],

wkconnect/__main__.py

@@ -1,8 +1,9 @@
 import asyncio
 import json
 import logging
+import os
 from copy import deepcopy
-from typing import Dict, List, Tuple, Type
+from typing import Any, Dict, List, Tuple, Type
 
 from aiohttp import ClientSession
 from aiohttp.client_exceptions import ClientConnectorError
@@ -12,7 +13,8 @@
 from uvloop import Loop
 
 from .backends.backend import Backend
-from .backends.neuroglancer.backend import NeuroglancerBackend as Neuroglancer
+from .backends.boss.backend import Boss
+from .backends.neuroglancer.backend import Neuroglancer
 from .repository import Repository
 from .routes import routes
 from .utils.scheduler import repeat_every_seconds
@@ -29,7 +31,7 @@ def __init__(self) -> None:
         self.repository: Repository
         self.webknossos: WebKnossos
         self.backends: Dict[str, Backend]
-        self.available_backends: List[Type[Backend]] = [Neuroglancer]
+        self.available_backends: List[Type[Backend]] = [Boss, Neuroglancer]
 
     async def add_dataset(
         self,
@@ -46,9 +48,15 @@ async def add_dataset(
         await self.webknossos.report_dataset(dataset.to_webknossos())
 
     async def load_persisted_datasets(self) -> None:
+        def expandvars_hook(dict: Dict[str, Any]) -> Dict[str, Any]:
+            for key, val in dict.items():
+                if isinstance(val, str):
+                    dict[key] = os.path.expandvars(val)
+            return dict
+
         try:
             with open(self.config["datasets_path"]) as datasets_file:
-                datasets = json.load(datasets_file)
+                datasets = json.load(datasets_file, object_hook=expandvars_hook)
         except FileNotFoundError:
             datasets = {}
         await asyncio.gather(
@@ -80,14 +88,13 @@ async def load_persisted_datasets(self) -> None:
 async def setup(app: Server, loop: Loop) -> None:
     def instanciate_backend(backend_class: Type[Backend]) -> Tuple[str, Backend]:
         backend_name = backend_class.name()
-        config = app.config["backends"][backend_name]
+        config = app.config["backends"].get(backend_name, {})
         return (backend_name, backend_class(config, app.http_client))
 
     app.http_client = await ClientSession(raise_for_status=True).__aenter__()
     app.repository = Repository()
     app.webknossos = WebKnossos(app.config, app.http_client)
     app.backends = dict(map(instanciate_backend, app.available_backends))
-    # await app.load_persisted_datasets()
 
 
 @app.listener("before_server_stop")

wkconnect/backends/backend.py

@@ -19,9 +19,8 @@ def to_webknossos(self) -> WKDataSource:
 
 class Backend(metaclass=ABCMeta):
     @classmethod
-    @abstractmethod
     def name(cls) -> str:
-        pass
+        return cls.__name__.lower()
 
     @abstractmethod
     def __init__(self, config: Dict, http_client: ClientSession) -> None: