Merge pull request #27 from elephant-track/dev

ELEPHANT server v0.5.1 -> v0.5.2
elephant-track · Nov 5, 2023 · b8f3367 · b8f3367
2 parents a60afd6 + 933f59d
commit b8f3367
Show file tree

Hide file tree

Showing 8 changed files with 299 additions and 70 deletions.
diff --git a/Dockerfile-test b/Dockerfile-test
@@ -1,4 +1,4 @@
-FROM elephant-server:0.5.1
+FROM elephant-server:0.5.2
 
 LABEL maintainer="Ko Sugawara <[email protected]>"
 

diff --git a/Makefile b/Makefile
@@ -5,7 +5,7 @@ help:
 
 ELEPHANT_GPU?=all
 ELEPHANT_WORKSPACE?=${PWD}/workspace
-ELEPHANT_IMAGE_NAME?=elephant-server:0.5.1
+ELEPHANT_IMAGE_NAME?=elephant-server:0.5.2
 ELEPHANT_NVIDIA_GID?=$$(ls -n /dev/nvidia0 2>/dev/null | awk '{print $$4}')
 ELEPHANT_DOCKER?=docker
 ELEPHANT_RABBITMQ_NODENAME?=rabbit@localhost

diff --git a/elephant-core/elephant/datasets.py b/elephant-core/elephant/datasets.py
@@ -280,7 +280,7 @@ def _generate_item(self, img_input, img_label, crop_size):
                     for i in range(self.n_dims)
                 ]
 
-            if isinstance(self, SegmentationDatasetZarr):
+            if RUN_ON_FLASK and isinstance(self, SegmentationDatasetZarr):
                 if not self.is_ae:
                     za_label_a = zarr.open(self.zpath_seg_label, mode='a')
                     index_pool = za_label_a.attrs.get(
@@ -556,10 +556,11 @@ def _get_memmap_or_load_label(self, timepoint, img_size=None):
 
     def _get_label_at(self, ind, img_size=None):
         if self.use_cache:
-            za_label_a = zarr.open(self.zpath_seg_label, mode='a')
-            if za_label_a.attrs.get('updated', False):
-                self.cache_dict_label.clear()
-                za_label_a.attrs['updated'] = False
+            if RUN_ON_FLASK:
+                za_label_a = zarr.open(self.zpath_seg_label, mode='a')
+                if za_label_a.attrs.get('updated', False):
+                    self.cache_dict_label.clear()
+                    za_label_a.attrs['updated'] = False
             key = f'{self.za_label.store.path}-t{ind}'
             if img_size is not None:
                 key += '-' + '-'.join(map(str, img_size))

diff --git a/elephant-core/elephant/version.py b/elephant-core/elephant/version.py
@@ -24,4 +24,4 @@
 # ==============================================================================
 """Version definition."""
 
-__version__ = '0.5.1'
+__version__ = "0.5.2"
diff --git a/elephant_server.ipynb b/elephant_server.ipynb
@@ -1,18 +1,4 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "elephant_server.ipynb",
-      "provenance": [],
-      "collapsed_sections": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "accelerator": "GPU"
-  },
   "cells": [
     {
       "cell_type": "markdown",
@@ -25,16 +11,16 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "Fx1aRfKREJLd"
       },
+      "outputs": [],
       "source": [
         "# from google.colab import drive\n",
         "# drive.mount('/content/drive')\n",
         "# !ln -sfn /content/drive/MyDrive/elephant_workspace /workspace"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -47,28 +33,28 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "CpJPKG_ATv39"
       },
+      "outputs": [],
       "source": [
-        "%env ELEPHANT_SERVER_VERSION = v0.5.1"
-      ],
-      "execution_count": null,
-      "outputs": []
+        "%env ELEPHANT_SERVER_VERSION = v0.5.2"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "X0Hdn56eMre2"
       },
+      "outputs": [],
       "source": [
         "!wget -q -c -nc https://github.com/elephant-track/elephant-server/archive/refs/tags/$ELEPHANT_SERVER_VERSION.tar.gz\n",
         "!mkdir -p /opt/elephant\n",
         "!tar -zxf $ELEPHANT_SERVER_VERSION.tar.gz --strip-components=1 -C /opt/elephant\n",
         "!rm $ELEPHANT_SERVER_VERSION.tar.gz"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -81,20 +67,22 @@
     },
     {
       "cell_type": "code",
-      "source": [
-        "%env CONDA_VERSION = py37_4.11.0"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "-42Y4tDCXTTQ"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "%env CONDA_VERSION = py37_4.11.0"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "CrfCHDcdh3mF"
       },
+      "outputs": [],
       "source": [
         "# Acknowledgements:\n",
         "#     https://donaldsrepo.github.io/Notebooks/GoogleColabCondaCreateEnv.html\n",
@@ -117,9 +105,7 @@
         "    print('found miniconda')\n",
         "\n",
         "!mamba env update -f /opt/elephant/environment.yml\n"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -132,9 +118,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "9a8LS0q0h94r"
       },
+      "outputs": [],
       "source": [
         "!apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \\\n",
         "    nginx \\\n",
@@ -146,9 +134,7 @@
         "    gosu \\\n",
         "    openssh-server \\\n",
         "    pwgen"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -161,9 +147,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "lDzE0j1DiAIu"
       },
+      "outputs": [],
       "source": [
         "%%bash\n",
         "apt-get install curl gnupg debian-keyring debian-archive-keyring apt-transport-https --no-install-recommends --no-install-suggests -y\n",
@@ -202,9 +190,7 @@
         "\n",
         "## Install rabbitmq-server and its dependencies\n",
         "apt-get install rabbitmq-server --no-install-recommends --no-install-suggests -y --fix-missing"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -217,9 +203,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "sWDwbcuhiEBt"
       },
+      "outputs": [],
       "source": [
         "# Set up RabbitMQ\n",
         "%env RABBITMQ_USER=user\n",
@@ -258,9 +246,7 @@
         "!mkdir -p /var/run/sshd\n",
         "!grep -qxF \"PermitRootLogin yes\" /etc/ssh/sshd_config || echo \"PermitRootLogin yes\" >> /etc/ssh/sshd_config\n",
         "!grep -qxF \"PasswordAuthentication yes\" /etc/ssh/sshd_config || echo \"PasswordAuthentication yes\" >> /etc/ssh/sshd_config"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -273,16 +259,16 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "RdzQk8ai5p-K"
       },
+      "outputs": [],
       "source": [
         "!wget -q -c -nc https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
         "!unzip -qq -n ngrok-stable-linux-amd64.zip -d /opt/ngrok\n",
         "!rm ngrok-stable-linux-amd64.zip"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -295,15 +281,15 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "JoRljMTaNArk"
       },
+      "outputs": [],
       "source": [
         "%load_ext tensorboard\n",
         "%tensorboard --logdir /workspace/logs"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
@@ -316,9 +302,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "WqDysO6biGfc"
       },
+      "outputs": [],
       "source": [
         "# Acknowledgements:\n",
         "# @Tamlyn https://stackoverflow.com/a/53252985\n",
@@ -366,20 +354,30 @@
         "for pname in ['rabbitmq', 'epmd', 'uwsgi', 'nginx', 'redis-server', 'sshd']:\n",
         "  !pkill -9 $pname\n",
         "!/usr/bin/supervisord"
-      ],
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "id": "tn6kRwoA6kTw"
       },
-      "source": [
-        ""
-      ],
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "elephant_server.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-  ]
-}
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/script/check_image_shapes.py b/script/check_image_shapes.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import argparse
+from pathlib import Path
+
+import skimage.io
+from tqdm import tqdm
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('dir', help='root path of CTC dataset')
+    parser.add_argument(
+        '--3d',
+        dest='is_3d',
+        action='store_true',
+        help=('specify if generate 3D labels (only used for 3D+time datasets)')
+    )
+    parser.add_argument(
+        '--gt',
+        dest='gt',
+        action='store_true',
+        help=('specify if process GT')
+    )
+    parser.add_argument(
+        '--st',
+        dest='st',
+        action='store_true',
+        help=('specify if process ST')
+    )
+    parser.add_argument(
+        '--gtst',
+        dest='gtst',
+        action='store_true',
+        help=('specify if process GT+ST')
+    )
+    args = parser.parse_args()
+    ref_types_list = []
+    if args.gt:
+        ref_types_list.append(('GT',))
+    if args.st:
+        ref_types_list.append(('ST',))
+    if args.gtst:
+        ref_types_list.append(('GT', 'ST'))
+    if len(ref_types_list) == 0:
+        ref_types_list = [('GT',), ('ST',), ('GT', 'ST')]
+    p = Path(args.dir)
+    is_3d = args.is_3d
+    seg_dir_name = f'SEG{"_3d" if is_3d else ""}'
+    for i in range(2):
+        p_img = p / f'0{i+1}'
+        shape = None
+        dtype = None
+        for f in tqdm(sorted(p_img.glob('*.tif'))):
+            img = skimage.io.imread(str(f))
+            if shape is None:
+                shape = img.shape
+                dtype = img.dtype
+            else:
+                if shape != img.shape:
+                    print(f'image {f.name}: shape expected {shape} but got {img.shape}.')
+                if dtype != img.dtype:
+                    print(f'image {f.name}: dtype expected {dtype} but got {img.dtype}.')
+        for ref_types in ref_types_list:
+            for ref_type in ref_types:
+                p_seg = p / f'0{i+1}_{ref_type}' / seg_dir_name
+                if p_seg.exists():
+                    for f in tqdm(sorted(p_seg.glob('*.tif'))):
+                        img = skimage.io.imread(str(f))
+                        if shape != img.shape:
+                            print(f'image {f.name}: shape expected {shape} but got {img.shape}.')
+
+if __name__ == "__main__":
+    main()