From 165d349678db0aecd31677291d1f63745be53927 Mon Sep 17 00:00:00 2001 From: Ko Sugawara Date: Sun, 5 Nov 2023 23:10:31 +0900 Subject: [PATCH 1/6] Update version to 0.5.2-dev --- Dockerfile-test | 2 +- Makefile | 2 +- elephant-core/elephant/version.py | 2 +- elephant_server.ipynb | 112 +++++++++++++++--------------- 4 files changed, 58 insertions(+), 60 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 47573ee..9aafa18 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,4 +1,4 @@ -FROM elephant-server:0.5.1 +FROM elephant-server:0.5.2-dev LABEL maintainer="Ko Sugawara " diff --git a/Makefile b/Makefile index 13faa8a..0f3f22c 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ help: ELEPHANT_GPU?=all ELEPHANT_WORKSPACE?=${PWD}/workspace -ELEPHANT_IMAGE_NAME?=elephant-server:0.5.1 +ELEPHANT_IMAGE_NAME?=elephant-server:0.5.2-dev ELEPHANT_NVIDIA_GID?=$$(ls -n /dev/nvidia0 2>/dev/null | awk '{print $$4}') ELEPHANT_DOCKER?=docker ELEPHANT_RABBITMQ_NODENAME?=rabbit@localhost diff --git a/elephant-core/elephant/version.py b/elephant-core/elephant/version.py index 867343f..63d67f9 100644 --- a/elephant-core/elephant/version.py +++ b/elephant-core/elephant/version.py @@ -24,4 +24,4 @@ # ============================================================================== """Version definition.""" -__version__ = '0.5.1' +__version__ = "0.5.2-dev" diff --git a/elephant_server.ipynb b/elephant_server.ipynb index da5343c..8ead723 100644 --- a/elephant_server.ipynb +++ b/elephant_server.ipynb @@ -1,18 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "elephant_server.ipynb", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, "cells": [ { "cell_type": "markdown", @@ -25,16 +11,16 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "Fx1aRfKREJLd" }, + "outputs": [], "source": [ "# from google.colab import drive\n", "# drive.mount('/content/drive')\n", "# !ln -sfn /content/drive/MyDrive/elephant_workspace /workspace" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -47,28 +33,28 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "CpJPKG_ATv39" }, + "outputs": [], "source": [ - "%env ELEPHANT_SERVER_VERSION = v0.5.1" - ], - "execution_count": null, - "outputs": [] + "%env ELEPHANT_SERVER_VERSION = v0.5.2-dev" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "X0Hdn56eMre2" }, + "outputs": [], "source": [ "!wget -q -c -nc https://github.com/elephant-track/elephant-server/archive/refs/tags/$ELEPHANT_SERVER_VERSION.tar.gz\n", "!mkdir -p /opt/elephant\n", "!tar -zxf $ELEPHANT_SERVER_VERSION.tar.gz --strip-components=1 -C /opt/elephant\n", "!rm $ELEPHANT_SERVER_VERSION.tar.gz" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -81,20 +67,22 @@ }, { "cell_type": "code", - "source": [ - "%env CONDA_VERSION = py37_4.11.0" - ], + "execution_count": null, "metadata": { "id": "-42Y4tDCXTTQ" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "%env CONDA_VERSION = py37_4.11.0" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "CrfCHDcdh3mF" }, + "outputs": [], "source": [ "# Acknowledgements:\n", "# https://donaldsrepo.github.io/Notebooks/GoogleColabCondaCreateEnv.html\n", @@ -117,9 +105,7 @@ " print('found miniconda')\n", "\n", "!mamba env update -f /opt/elephant/environment.yml\n" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -132,9 +118,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "9a8LS0q0h94r" }, + "outputs": [], "source": [ "!apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \\\n", " nginx \\\n", @@ -146,9 +134,7 @@ " gosu \\\n", " openssh-server \\\n", " pwgen" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -161,9 +147,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "lDzE0j1DiAIu" }, + "outputs": [], "source": [ "%%bash\n", "apt-get install curl gnupg debian-keyring debian-archive-keyring apt-transport-https --no-install-recommends --no-install-suggests -y\n", @@ -202,9 +190,7 @@ "\n", "## Install rabbitmq-server and its dependencies\n", "apt-get install rabbitmq-server --no-install-recommends --no-install-suggests -y --fix-missing" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -217,9 +203,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "sWDwbcuhiEBt" }, + "outputs": [], "source": [ "# Set up RabbitMQ\n", "%env RABBITMQ_USER=user\n", @@ -258,9 +246,7 @@ "!mkdir -p /var/run/sshd\n", "!grep -qxF \"PermitRootLogin yes\" /etc/ssh/sshd_config || echo \"PermitRootLogin yes\" >> /etc/ssh/sshd_config\n", "!grep -qxF \"PasswordAuthentication yes\" /etc/ssh/sshd_config || echo \"PasswordAuthentication yes\" >> /etc/ssh/sshd_config" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -273,16 +259,16 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "RdzQk8ai5p-K" }, + "outputs": [], "source": [ "!wget -q -c -nc https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n", "!unzip -qq -n ngrok-stable-linux-amd64.zip -d /opt/ngrok\n", "!rm ngrok-stable-linux-amd64.zip" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -295,15 +281,15 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "JoRljMTaNArk" }, + "outputs": [], "source": [ "%load_ext tensorboard\n", "%tensorboard --logdir /workspace/logs" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -316,9 +302,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "WqDysO6biGfc" }, + "outputs": [], "source": [ "# Acknowledgements:\n", "# @Tamlyn https://stackoverflow.com/a/53252985\n", @@ -366,20 +354,30 @@ "for pname in ['rabbitmq', 'epmd', 'uwsgi', 'nginx', 'redis-server', 'sshd']:\n", " !pkill -9 $pname\n", "!/usr/bin/supervisord" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "tn6kRwoA6kTw" }, - "source": [ - "" - ], - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "elephant_server.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} From f99635d07e8504b80d36b5d303848d6afd8dcf1a Mon Sep 17 00:00:00 2001 From: Ko Sugawara Date: Mon, 6 Nov 2023 00:10:00 +0900 Subject: [PATCH 2/6] Use Zarr attrs only when RUN_ON_FLASK - it takes long time to update zarr attrs when working with scripts on 3d+t data --- elephant-core/elephant/datasets.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/elephant-core/elephant/datasets.py b/elephant-core/elephant/datasets.py index a2b9263..40b90fa 100644 --- a/elephant-core/elephant/datasets.py +++ b/elephant-core/elephant/datasets.py @@ -280,7 +280,7 @@ def _generate_item(self, img_input, img_label, crop_size): for i in range(self.n_dims) ] - if isinstance(self, SegmentationDatasetZarr): + if RUN_ON_FLASK and isinstance(self, SegmentationDatasetZarr): if not self.is_ae: za_label_a = zarr.open(self.zpath_seg_label, mode='a') index_pool = za_label_a.attrs.get( @@ -556,10 +556,11 @@ def _get_memmap_or_load_label(self, timepoint, img_size=None): def _get_label_at(self, ind, img_size=None): if self.use_cache: - za_label_a = zarr.open(self.zpath_seg_label, mode='a') - if za_label_a.attrs.get('updated', False): - self.cache_dict_label.clear() - za_label_a.attrs['updated'] = False + if RUN_ON_FLASK: + za_label_a = zarr.open(self.zpath_seg_label, mode='a') + if za_label_a.attrs.get('updated', False): + self.cache_dict_label.clear() + za_label_a.attrs['updated'] = False key = f'{self.za_label.store.path}-t{ind}' if img_size is not None: key += '-' + '-'.join(map(str, img_size)) From edb1a3dc50010a1ef1ae2c8daaf1354e8f5dc01c Mon Sep 17 00:00:00 2001 From: Ko Sugawara Date: Mon, 6 Nov 2023 00:35:09 +0900 Subject: [PATCH 3/6] Update generate_seg_labels script - appropriately handle datasets that contain different sizes of images --- script/generate_seg_labels.py | 41 ++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/script/generate_seg_labels.py b/script/generate_seg_labels.py index b814c42..2d879fa 100644 --- a/script/generate_seg_labels.py +++ b/script/generate_seg_labels.py @@ -154,18 +154,20 @@ def main(): p_root = p / f'0{i+1}-{suffix}-seg{"-3d" if is_3d else ""}' p_root.mkdir(exist_ok=True) + chunks = (1, 1) + zarr_shape[-2:] if is_3d else (1,) + zarr_shape[-2:] + za_img = zarr.open( str(p_root / 'imgs.zarr'), 'w', shape=zarr_shape, - chunks=(1,) + zarr_shape[1:], + chunks=chunks, dtype=zarr_dtype, ) za_seg = zarr.open( str(p_root / 'seg_labels.zarr'), 'w', shape=zarr_shape, - chunks=(1,) + zarr_shape[1:], + chunks=chunks, dtype='u1' ) visited_files = set() @@ -179,9 +181,22 @@ def main(): t = re.findall(r'(\d+)', f.name)[0] # 2d data -> 2d labels or 3d data -> 3d labels if n_dims == 2 or is_3d: - za_img[count] = skimage.io.imread( + img = skimage.io.imread( str(p / f'0{i+1}' / f't{t}.tif') ) + if is_3d: + za_img[ + count, + :img.shape[-3], + :img.shape[-2], + :img.shape[-1], + ] = img + else: + za_img[ + count, + :img.shape[-2], + :img.shape[-1], + ] = img # 3d data -> 2d labels else: z = int(re.findall(r'(\d+)', f.name)[1]) @@ -190,7 +205,11 @@ def main(): str(p / f'0{i+1}' / f't{t}.tif') ) last_t = t - za_img[count] = img_cache[z] + za_img[ + count, + :img_cache.shape[-2], + :img_cache.shape[-1], + ] = img_cache[z] label = skimage.io.imread(str(f)) if (ref_type == 'GT' and f.name in sparse_data.get(f'0{i+1}', [])): @@ -246,7 +265,19 @@ def main(): ) seg[indices_inner_p] = 2 seg[indices_inner] = 3 - za_seg[count] = seg + if is_3d: + za_seg[ + count, + :seg.shape[-3], + :seg.shape[-2], + :seg.shape[-1], + ] = seg + else: + za_seg[ + count, + :seg.shape[-2], + :seg.shape[-1], + ] = seg count += 1 visited_files.add(f.name) From 10a61613e968222afb9a6a7c1871fd6cecce980d Mon Sep 17 00:00:00 2001 From: Ko Sugawara Date: Mon, 6 Nov 2023 00:36:10 +0900 Subject: [PATCH 4/6] Add check_image_shapes script - this script is useful to check if the dataset contains different sizes of images --- script/check_image_shapes.py | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 script/check_image_shapes.py diff --git a/script/check_image_shapes.py b/script/check_image_shapes.py new file mode 100644 index 0000000..4aac790 --- /dev/null +++ b/script/check_image_shapes.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +import argparse +from pathlib import Path + +import skimage.io +from tqdm import tqdm + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('dir', help='root path of CTC dataset') + parser.add_argument( + '--3d', + dest='is_3d', + action='store_true', + help=('specify if generate 3D labels (only used for 3D+time datasets)') + ) + parser.add_argument( + '--gt', + dest='gt', + action='store_true', + help=('specify if process GT') + ) + parser.add_argument( + '--st', + dest='st', + action='store_true', + help=('specify if process ST') + ) + parser.add_argument( + '--gtst', + dest='gtst', + action='store_true', + help=('specify if process GT+ST') + ) + args = parser.parse_args() + ref_types_list = [] + if args.gt: + ref_types_list.append(('GT',)) + if args.st: + ref_types_list.append(('ST',)) + if args.gtst: + ref_types_list.append(('GT', 'ST')) + if len(ref_types_list) == 0: + ref_types_list = [('GT',), ('ST',), ('GT', 'ST')] + p = Path(args.dir) + is_3d = args.is_3d + seg_dir_name = f'SEG{"_3d" if is_3d else ""}' + for i in range(2): + p_img = p / f'0{i+1}' + shape = None + dtype = None + for f in tqdm(sorted(p_img.glob('*.tif'))): + img = skimage.io.imread(str(f)) + if shape is None: + shape = img.shape + dtype = img.dtype + else: + if shape != img.shape: + print(f'image {f.name}: shape expected {shape} but got {img.shape}.') + if dtype != img.dtype: + print(f'image {f.name}: dtype expected {dtype} but got {img.dtype}.') + for ref_types in ref_types_list: + for ref_type in ref_types: + p_seg = p / f'0{i+1}_{ref_type}' / seg_dir_name + if p_seg.exists(): + for f in tqdm(sorted(p_seg.glob('*.tif'))): + img = skimage.io.imread(str(f)) + if shape != img.shape: + print(f'image {f.name}: shape expected {shape} but got {img.shape}.') + +if __name__ == "__main__": + main() \ No newline at end of file From 7be5dd3f2e0ab3d365dccb2928a3da1da60dd6cb Mon Sep 17 00:00:00 2001 From: Ko Sugawara Date: Mon, 6 Nov 2023 00:37:08 +0900 Subject: [PATCH 5/6] Add elephant-detection script - this script runs the detection and exports the results in the CTC format --- script/elephant-detection.py | 127 +++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 script/elephant-detection.py diff --git a/script/elephant-detection.py b/script/elephant-detection.py new file mode 100644 index 0000000..98dc93d --- /dev/null +++ b/script/elephant-detection.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python +# ============================================================================== +# Copyright (c) 2023, Ko Sugawara +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================== +"""Commandline interface for prediction and export using a config file.""" + +import argparse +import collections +import io +import json +import os +from pathlib import Path + +import skimage.io +from tqdm import tqdm + +from elephant.common import detect_spots +from elephant.common import export_ctc_labels +from elephant.config import ExportConfig +from elephant.config import SegmentationEvalConfigTiff +from elephant.util import get_next_multiple + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("input", help="input directory") + parser.add_argument("config", help="config file") + parser.add_argument("--output", help="output directory") + args = parser.parse_args() + # list up input image files + files = [ + os.path.join(args.input, f) + for f in sorted(os.listdir(args.input)) + if f.endswith(".tif") + ] + with io.open(args.config, "r", encoding="utf-8") as jsonfile: + config_data = json.load(jsonfile) + if not "patch" in config_data: + is_3d = ( + config_data.get("is_3d", True) == True + and config_data.get("use_2d", False) == False + ) + config_data["patch"] = [ + int(get_next_multiple(s * 0.75, 16)) + for s in skimage.io.imread(files[0]).shape[-(2 + is_3d) :] + ] + config = SegmentationEvalConfigTiff(config_data) + print("Start detection...") + print(config) + for i, f in tqdm(enumerate(files)): + config.timepoint = i + config.tiff_input = f + spots = detect_spots( + str(config.device), + config.model_path, + config.keep_axials, + config.is_pad, + config.is_3d, + config.crop_size, + config.scales, + config.cache_maxbytes, + config.use_2d, + config.use_median, + config.patch_size, + config.crop_box, + config.c_ratio, + config.p_thresh, + config.r_min, + config.r_max, + config.output_prediction, + None, + None, + config.timepoint, + config.tiff_input, + None, + config.batch_size, + config.input_size, + ) + print("End detection.") + print("Start export...") + Path(args.output).mkdir(parents=True, exist_ok=True) + config_data["savedir"] = args.output + config_data["shape"] = skimage.io.imread(f).shape + config_data["t_start"] = i + config_data["t_end"] = i + config_export = ExportConfig(config_data) + print(config_export) + # load spots and group by t + spots_dict = collections.defaultdict(list) + for spot in spots: + current_spots = spots_dict.get(spot["t"]) + if current_spots is None: + spot["value"] = 1 + else: + spot["value"] = len(current_spots) + 1 + spots_dict[spot["t"]].append(spot) + spots_dict = collections.OrderedDict(sorted(spots_dict.items())) + # export labels + export_ctc_labels(config_export, spots_dict) + print("End export.") + print("Done") + + +if __name__ == "__main__": + main() \ No newline at end of file From 933f59d7d766eb1147e4ca1adb3d55929487b59c Mon Sep 17 00:00:00 2001 From: Ko Sugawara Date: Mon, 6 Nov 2023 00:45:56 +0900 Subject: [PATCH 6/6] Update version to 0.5.2 --- Dockerfile-test | 2 +- Makefile | 2 +- elephant-core/elephant/version.py | 2 +- elephant_server.ipynb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 9aafa18..ae0d4e1 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,4 +1,4 @@ -FROM elephant-server:0.5.2-dev +FROM elephant-server:0.5.2 LABEL maintainer="Ko Sugawara " diff --git a/Makefile b/Makefile index 0f3f22c..ea95b25 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ help: ELEPHANT_GPU?=all ELEPHANT_WORKSPACE?=${PWD}/workspace -ELEPHANT_IMAGE_NAME?=elephant-server:0.5.2-dev +ELEPHANT_IMAGE_NAME?=elephant-server:0.5.2 ELEPHANT_NVIDIA_GID?=$$(ls -n /dev/nvidia0 2>/dev/null | awk '{print $$4}') ELEPHANT_DOCKER?=docker ELEPHANT_RABBITMQ_NODENAME?=rabbit@localhost diff --git a/elephant-core/elephant/version.py b/elephant-core/elephant/version.py index 63d67f9..481a103 100644 --- a/elephant-core/elephant/version.py +++ b/elephant-core/elephant/version.py @@ -24,4 +24,4 @@ # ============================================================================== """Version definition.""" -__version__ = "0.5.2-dev" +__version__ = "0.5.2" diff --git a/elephant_server.ipynb b/elephant_server.ipynb index 8ead723..580545d 100644 --- a/elephant_server.ipynb +++ b/elephant_server.ipynb @@ -39,7 +39,7 @@ }, "outputs": [], "source": [ - "%env ELEPHANT_SERVER_VERSION = v0.5.2-dev" + "%env ELEPHANT_SERVER_VERSION = v0.5.2" ] }, {