From 362ccc543984c3a12bdedbbe5878ca55db88c45f Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Wed, 21 Feb 2024 20:36:42 +0100 Subject: [PATCH 01/32] Clip removed and docs updated #149 --- docs/source/creators/creators_description_classes.rst | 3 +++ src/deepness/processing/models/segmentor.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/creators/creators_description_classes.rst b/docs/source/creators/creators_description_classes.rst index c9e84ad..95c2ab8 100644 --- a/docs/source/creators/creators_description_classes.rst +++ b/docs/source/creators/creators_description_classes.rst @@ -34,6 +34,9 @@ For each output class, a separate vector layer can be created. Output report contains information about percentage coverage of each class. +The model should have at least two output classes, one for the background and one (or more) for the object of interest. The background class should be the first class in the output. +Model outputs should sum to 1.0 for each pixel, so the output is a probability map. To achieve this, the output should be passed through a softmax function. + =============== Detection Model diff --git a/src/deepness/processing/models/segmentor.py b/src/deepness/processing/models/segmentor.py index 9ce8252..091bba8 100644 --- a/src/deepness/processing/models/segmentor.py +++ b/src/deepness/processing/models/segmentor.py @@ -37,7 +37,8 @@ def postprocessing(self, model_output: List) -> np.ndarray: np.ndarray Batch of postprocessed masks (N,H,W,C), 0-1 """ - labels = np.clip(model_output[0], 0, 1) + # labels = np.clip(model_output[0], 0, 1) + labels = model_output[0] # no need for clipping I think - see #149 return labels From 13898f7be898b3cff2e6fe8be70cd95d73138581 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 22 Feb 2024 11:03:30 +0100 Subject: [PATCH 02/32] Handle no channel models, prepare new tests --- .../map_processor_segmentation.py | 12 +- .../dummy_segmentation_models/Untitled.ipynb | 92 ++++++++++++ .../different_output_size_512_to_484.onnx} | Bin .../dummy_model.onnx} | 0 .../one_output_sigmoid_bsx1x512x512.onnx | Bin 0 -> 415 bytes .../one_output_sigmoid_bsx512x512.onnx | Bin 0 -> 593 bytes .../one_output_softmax_bsx2x512x512.onnx | Bin 0 -> 662 bytes .../two_output_sigmoid_bsx1x512x512.onnx | Bin 0 -> 523 bytes .../two_output_sigmoid_bsx512x512.onnx | Bin 0 -> 891 bytes .../two_output_softmax_bsx2x512x512.onnx | Bin 0 -> 1013 bytes ...rocessor_segmentation_many_output_types.py | 135 ++++++++++++++++++ test/test_utils.py | 22 ++- 12 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb rename test/data/dummy_model/{dummy_segmentation_model_different_output_size.onnx => dummy_segmentation_models/different_output_size_512_to_484.onnx} (100%) rename test/data/dummy_model/{dummy_segmentation_model.onnx => dummy_segmentation_models/dummy_model.onnx} (100%) create mode 100644 test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx create mode 100644 test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx512x512.onnx create mode 100644 test/data/dummy_model/dummy_segmentation_models/one_output_softmax_bsx2x512x512.onnx create mode 100644 test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx1x512x512.onnx create mode 100644 test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx512x512.onnx create mode 100644 test/data/dummy_model/dummy_segmentation_models/two_output_softmax_bsx2x512x512.onnx create mode 100644 test/test_map_processor_segmentation_many_output_types.py diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index 97761f2..c970a36 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -1,6 +1,7 @@ """ This file implements map processing for segmentation model """ from typing import Callable + import numpy as np from qgis.core import QgsProject, QgsVectorLayer @@ -152,9 +153,14 @@ def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: result[result < self.segmentation_parameters.pixel_classification__probability_threshold] = 0.0 - if (result.shape[1] == 1): - result = (result != 0).astype(int)[:, 0] + if len(result.shape) == 3: + result = (result != 0).astype(int) + elif len(result.shape) == 4: + if (result.shape[1] == 1): + result = (result != 0).astype(int)[:, 0] + else: + result = np.argmax(result, axis=1) else: - result = np.argmax(result, axis=1) + raise ValueError(f'Unexpected result shape: {result.shape}') return result diff --git a/test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb b/test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb new file mode 100644 index 0000000..a6ff46d --- /dev/null +++ b/test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb @@ -0,0 +1,92 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "7ad5306f-18fc-4499-b764-ab6902ac301f", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "42e756a1-3ea0-4600-adcd-50ad14b938f8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 2, 512, 512]) torch.Size([1, 2, 512, 512])\n" + ] + } + ], + "source": [ + "x = torch.rand(1, 3, 512, 512)\n", + "\n", + "class Model(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " self.conv = nn.Conv2d(3, 2, 1)\n", + " \n", + " self.softmax = nn.Softmax(dim=1)\n", + " self.sigmoid = nn.Sigmoid()\n", + "\n", + " def forward(self, x):\n", + " x = self.conv(x)\n", + "\n", + " return self.softmax(x), self.softmax(x)\n", + "\n", + "m = Model()\n", + "print(m(x)[0].shape, m(x)[1].shape)\n", + "\n", + "torch.onnx.export(m,\n", + " x,\n", + " \"two_output_softmax_bsx2x512x512.onnx\",\n", + " export_params=True,\n", + " opset_version=12,\n", + " do_constant_folding=True,\n", + " input_names = ['input'],\n", + " output_names = ['output', 'output2'],\n", + " dynamic_axes={'input' : {0 : 'batch_size'},\n", + " 'output' : {0 : 'batch_size'},\n", + " 'output2' : {0 : 'batch_size'},\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79c0bfcb-47c4-44c9-9c9b-c4c75b2c3b34", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/data/dummy_model/dummy_segmentation_model_different_output_size.onnx b/test/data/dummy_model/dummy_segmentation_models/different_output_size_512_to_484.onnx similarity index 100% rename from test/data/dummy_model/dummy_segmentation_model_different_output_size.onnx rename to test/data/dummy_model/dummy_segmentation_models/different_output_size_512_to_484.onnx diff --git a/test/data/dummy_model/dummy_segmentation_model.onnx b/test/data/dummy_model/dummy_segmentation_models/dummy_model.onnx similarity index 100% rename from test/data/dummy_model/dummy_segmentation_model.onnx rename to test/data/dummy_model/dummy_segmentation_models/dummy_model.onnx diff --git a/test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d1b01fb6f72872d751702c55691d3fc6f67e7bff GIT binary patch literal 415 zcmd;J6=E-_EXglQ&X8g?(lgRCuxexGTEfW1nweKnTEfMhoS#>wSDu=go>9WZ3Faha zCKd|`>x0<(&OjnQzqACXG~Pgp3n8e)0+P}a;^Iun%tD8T51>|HOO zwA|2r64$rvTjF`%PD>Q501*UUEQXoi_eE(#LrO@AOM!z?NQR3?h$|_vBsn9#II}91 di;07oi6-{I2qfl*35V@b;7>ab z4s$&YwcHsp8k1ckvyJmqSV6*5sMT#k8Pvoqr4y*xY)m-|{J4+$XpUrE0m+SmBy6N! zRj-3`??k>cwqlQl&I+^xVo2?{Us>*~P6avU5wo4uO>hB!(I9(9%-l)9Y~yOFONQ!Z zN|Or=ToaOgC+5`WWkGEn)9@;*>mW_(IC1ng7F5}6ph~f-zx5o6V8Z`&;Exx{*0P;V zY+O2dOEkGK2uvvREjOZ}rypR1h4dg54So*ERBk`--o>wz#q4-?rq^>oIXF}NcwH=R wj*9sb4fZi2El>yyDIa-O%$^*mVgUsFEaqZ(2n8q7hE|GhY@pZ3K~+`00V3$7Z~y=R literal 0 HcmV?d00001 diff --git a/test/data/dummy_model/dummy_segmentation_models/one_output_softmax_bsx2x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/one_output_softmax_bsx2x512x512.onnx new file mode 100644 index 0000000000000000000000000000000000000000..0efb282a3fa5606b5535029da04b5baaa43a446a GIT binary patch literal 662 zcmaJNL5QZdAuv`sj5Y^DjC@F+yp1Vmb% z({+=6CaRl|xH3K4Z0pP-t~mp(f$Wkd)0QSHxTipragR35851<&FB^C($DB*vA-!DP zBQ|p#W`-W|?_wr4$Eks?>2-WnNfXC@;Q|&B;?$h*WTF~sDL+e@=oF*IK_)%Ql9Jy8 zID`L5@baX+>5Vzr+^t?%Y#l>~P+M<##I@7{5|M-iBq&1l`&4Vmm-=euV{l*IHAch! zAYM6Mc|JTCy${tC5`Azu^%^Rj^76ym>+pPU`ay68c94L#A%USmxMAsx-kIP_frR?< SI5hTQ`iOtgVu-Ga3VZ>3KE#Ls literal 0 HcmV?d00001 diff --git a/test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx1x512x512.onnx new file mode 100644 index 0000000000000000000000000000000000000000..1eb7e6b9ecd7598cb84ef7b051ef25bbb86ad269 GIT binary patch literal 523 zcmd;J7h*4{EXglQ&X8g?(lgRCu=>QzwS)(5fmoq7^Nn93O z*qkB6262#-fPQgidTxGZiheMJR$_-TxU8}3WQXW9k`jVxjW@*5EX0+Yn3)%!UX)mn zp{2^f$iWQ6i~@{K$YJHh(|_xb-S%IGb}QG;wbK#>D?o&w7t5!~yZ1$DLxWjJiA#Zl zQAmc1M~Evau_QSozBscgm5YgknTwgDfdxz?>2N_khEp{ol4@Npm}iJl?!+X(0|1J! BkktSH literal 0 HcmV?d00001 diff --git a/test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx512x512.onnx new file mode 100644 index 0000000000000000000000000000000000000000..b54d1e939ddf124c0d8451eb69b4325726236eae GIT binary patch literal 891 zcmbV~&r8EF6vxfl+3eXvjSl9NAq7DztZqlaG3IR^M8$(5lx!<&aBEAODthwI@VJBj zhMqin^FQ$5Kj2xg>)KV_R4^gG-uotbpX6bQNbVJPytY-$R?F4$*7nOR+#$#^+YLC( zwH^DSd}*?-#UU3hT8#Q+K_AZb-LNy9fQM3Js|cx46^Aa3K(51jlrzWn>!^+fNYZ4; zc0DI>3*joG(vDxB8P(xF1tgz)tYao8!5&NtgDmMj>-HVi z(T_%B8lm#trAn~|jtLp!_?+52-H;pGRGfG<4YC)s7noWJ3yRb%pqaR2zK#PVLKA)t z2d3OeW=GFi#QLEbJ&7vC6WE{5yPQB7l}TbU=}ehV$l{pKsHD_$pRE7nK7ilCf~ofs zX~bR)Gbv^^Dr0;CeadX3>rvOz)-b|C*pPx6pL4V+zdpU~+@9^#YQp2amX8E!-ZsVO z_x{7_W}Mzcg;k8m3djUnl(#LzXV)fVumA$S5u>iP4)K?e60C&k!Z&rQ+<*k&FKHvA F$RDL2`&Iw| literal 0 HcmV?d00001 diff --git a/test/data/dummy_model/dummy_segmentation_models/two_output_softmax_bsx2x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/two_output_softmax_bsx2x512x512.onnx new file mode 100644 index 0000000000000000000000000000000000000000..eeec558e05358714047d4d664702123b6d9a01cd GIT binary patch literal 1013 zcma)5%TB^T6t$(;;erW{j{u@EEUWTbnqUa-*lJ>;CUt;9CqUbDilT1(2EV`<<6nsS z18#J|LVv`KmRC#UWhR--^qih^=iFQ*V$ry?bhD^Nyy-3S|b+KHJ0VF+#M%M0q|B# zyIBmYwUZ&VNA=nPt+FGEjmNj4K1LC752e>ZF-C zG>OIxohfdOcodBJ9G=wiRm{5W>dErmdf^PF3F5FLc+hioy@G$CnAqn?eh^5LvXIBl z0FGd?2%hi78(DV5oNDFNjMCNd0j&?D{`H)b5Z*82swr(3AnT6}G z*3Z58{Nj)9Nnym>WOqvP=&jv;sy=>aj<@;)g=4UZI2?rlh61sQs?6w>0=5-+(2daD X3fs_c`b0KA#NL6ytN%mK@xj0^40kj* literal 0 HcmV?d00001 diff --git a/test/test_map_processor_segmentation_many_output_types.py b/test/test_map_processor_segmentation_many_output_types.py new file mode 100644 index 0000000..c760e07 --- /dev/null +++ b/test/test_map_processor_segmentation_many_output_types.py @@ -0,0 +1,135 @@ +from test.test_utils import (create_default_input_channels_mapping_for_rgba_bands, create_rlayer_from_file, + get_dummy_fotomap_small_path, get_dummy_segmentation_models_dict, init_qgis) +from unittest.mock import MagicMock + +import matplotlib.pyplot as plt +import numpy as np +from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle + +from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions +from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation +from deepness.processing.models.segmentor import Segmentor + +RASTER_FILE_PATH = get_dummy_fotomap_small_path() +INPUT_CHANNELS_MAPPING = create_default_input_channels_mapping_for_rgba_bands() + +MODEL_FILES_DICT = get_dummy_segmentation_models_dict() + +# 'one_output': { +# '1x1x512x512' +# '1x512x512' +# '1x2x512x512' +# }, +# 'two_outputs': { +# '1x1x512x512' +# '1x512x512' +# '1x2x512x512' +# } + + +def test_dummy_model_processing__1x1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILES_DICT['one_output']['1x1x512x512']) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, + model_output_format__single_class_number=-1, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (561, 829) + +def test_dummy_model_processing__1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILES_DICT['one_output']['1x512x512']) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, + model_output_format__single_class_number=-1, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (561, 829) + +def test_dummy_model_processing__1x2x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILES_DICT['one_output']['1x2x512x512']) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, + model_output_format__single_class_number=-1, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (561, 829) diff --git a/test/test_utils.py b/test/test_utils.py index 078ea62..0548f0e 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -15,14 +15,32 @@ def get_dummy_segmentation_model_path(): Get path of a dummy onnx model. See details in README in model directory. Model used for unit tests processing purposes """ - return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_model.onnx') + return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'dummy_model.onnx') def get_dummy_segmentation_model_different_output_size_path(): """ Get path of a dummy onnx model. See details in README in model directory. Model used for unit tests processing purposes. Its output size is different than input size. """ - return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_model_different_output_size.onnx') + return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'different_output_size_512_to_484.onnx') + +def get_dummy_segmentation_models_dict(): + """ + Get dictionary with dummy segmentation models paths. See details in README in model directory. + Models used for unit tests processing purposes + """ + return { + 'one_output': { + '1x1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'one_output_sigmoid_bsx1x512x512.onnx'), + '1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'one_output_sigmoid_bsx512x512.onnx'), + '1x2x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'one_output_softmax_bsx2x512x512.onnx'), + }, + 'two_outputs': { + '1x1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_output_sigmoid_bsx1x512x512.onnx'), + '1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_output_sigmoid_bsx512x512.onnx'), + '1x2x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_output_softmax_bsx2x512x512.onnx'), + } + } def get_dummy_recognition_model_path(): """ From 64bf2f10327ead99c3ffe0758fb8593331a95fba Mon Sep 17 00:00:00 2001 From: Bartosz Date: Fri, 23 Feb 2024 10:09:00 +0100 Subject: [PATCH 03/32] Allow for multi outputs for regressor and segmentor --- src/deepness/common/config_entry_key.py | 3 - .../map_processing_parameters.py | 15 -- src/deepness/deepness_dockwidget.py | 40 +--- src/deepness/deepness_dockwidget.ui | 63 +----- .../processing/map_processor/map_processor.py | 8 +- .../map_processor/map_processor_detection.py | 14 +- .../map_processor/map_processor_regression.py | 42 ++-- .../map_processor_segmentation.py | 125 ++++++------ .../map_processor_superresolution.py | 4 + .../map_processor/map_processor_with_model.py | 20 +- src/deepness/processing/models/detector.py | 9 +- src/deepness/processing/models/model_base.py | 6 +- .../processing/models/preprocessing_utils.py | 1 - src/deepness/processing/models/recognition.py | 3 +- src/deepness/processing/models/regressor.py | 52 ++--- src/deepness/processing/models/segmentor.py | 50 ++--- .../processing/models/superresolution.py | 4 +- src/deepness/processing/tile_params.py | 15 +- .../Untitled.ipynb | 19 +- .../dummy_regression_model.onnx | Bin .../dummy_regression_model_batched.onnx | Bin .../one_output_sigmoid_bsx1x512x512.onnx | Bin 0 -> 415 bytes .../one_output_sigmoid_bsx512x512.onnx | Bin 0 -> 586 bytes .../two_outputs_sigmoid_bsx1x512x512.onnx | Bin 0 -> 523 bytes .../two_outputs_sigmoid_bsx512x512.onnx | Bin 0 -> 802 bytes ... => two_outputs_sigmoid_bsx1x512x512.onnx} | Bin ...nx => two_outputs_sigmoid_bsx512x512.onnx} | Bin ... => two_outputs_softmax_bsx2x512x512.onnx} | Bin ...ap_processor_detection_yolo_ultralytics.py | 4 +- ...ual_test_map_processor_detection_yolov6.py | 4 +- ...map_processor_instance_yolo_ultralytics.py | 4 +- test/test_deepness_dockwidget.py | 7 +- ...st_map_processor_detection_oils_example.py | 8 +- ..._map_processor_detection_planes_example.py | 4 +- test/test_map_processor_empty_detection.py | 4 +- test/test_map_processor_recognition.py | 4 +- test/test_map_processor_regression.py | 10 +- ..._processor_regression_many_output_types.py | 163 +++++++++++++++ test/test_map_processor_segmentation.py | 187 +++++++++++++----- ...ssor_segmentation_different_output_size.py | 6 +- ...rocessor_segmentation_landcover_example.py | 26 ++- ..._segmentation_landcover_example_batched.py | 26 ++- ...rocessor_segmentation_many_output_types.py | 133 +++++++++++-- test/test_map_processor_superresolution.py | 10 +- ...test_map_processor_training_data_export.py | 4 +- test/test_remove_overlaping_detections.py | 2 - test/test_utils.py | 26 ++- 47 files changed, 664 insertions(+), 461 deletions(-) rename test/data/dummy_model/{dummy_segmentation_models => dummy_regression_models}/Untitled.ipynb (79%) rename test/data/dummy_model/{ => dummy_regression_models}/dummy_regression_model.onnx (100%) rename test/data/dummy_model/{ => dummy_regression_models}/dummy_regression_model_batched.onnx (100%) create mode 100644 test/data/dummy_model/dummy_regression_models/one_output_sigmoid_bsx1x512x512.onnx create mode 100644 test/data/dummy_model/dummy_regression_models/one_output_sigmoid_bsx512x512.onnx create mode 100644 test/data/dummy_model/dummy_regression_models/two_outputs_sigmoid_bsx1x512x512.onnx create mode 100644 test/data/dummy_model/dummy_regression_models/two_outputs_sigmoid_bsx512x512.onnx rename test/data/dummy_model/dummy_segmentation_models/{two_output_sigmoid_bsx1x512x512.onnx => two_outputs_sigmoid_bsx1x512x512.onnx} (100%) rename test/data/dummy_model/dummy_segmentation_models/{two_output_sigmoid_bsx512x512.onnx => two_outputs_sigmoid_bsx512x512.onnx} (100%) rename test/data/dummy_model/dummy_segmentation_models/{two_output_softmax_bsx2x512x512.onnx => two_outputs_softmax_bsx2x512x512.onnx} (100%) create mode 100644 test/test_map_processor_regression_many_output_types.py diff --git a/src/deepness/common/config_entry_key.py b/src/deepness/common/config_entry_key.py index 2ff74da..546aa31 100644 --- a/src/deepness/common/config_entry_key.py +++ b/src/deepness/common/config_entry_key.py @@ -40,9 +40,6 @@ class ConfigEntryKey(enum.Enum): DATA_EXPORT_SEGMENTATION_MASK_ENABLED = enum.auto(), False DATA_EXPORT_SEGMENTATION_MASK_ID = enum.auto(), '' - MODEL_OUTPUT_FORMAT = enum.auto(), '' # string of ModelOutputFormat, e.g. "ONLY_SINGLE_CLASS_AS_LAYER.value" - MODEL_OUTPUT_FORMAT_CLASS_NUMBER = enum.auto(), 0 - INPUT_CHANNELS_MAPPING__ADVANCED_MODE = enum.auto, False INPUT_CHANNELS_MAPPING__MAPPING_LIST_STR = enum.auto, [] diff --git a/src/deepness/common/processing_parameters/map_processing_parameters.py b/src/deepness/common/processing_parameters/map_processing_parameters.py index d461082..755cf74 100644 --- a/src/deepness/common/processing_parameters/map_processing_parameters.py +++ b/src/deepness/common/processing_parameters/map_processing_parameters.py @@ -15,18 +15,6 @@ class ProcessedAreaType(enum.Enum): def get_all_names(cls): return [e.value for e in cls] - -class ModelOutputFormat(enum.Enum): - ALL_CLASSES_AS_SEPARATE_LAYERS = 'All classes as separate layers' - CLASSES_AS_SEPARATE_LAYERS_WITHOUT_ZERO_CLASS = 'Classes as separate layers (without 0 class)' - ONLY_SINGLE_CLASS_AS_LAYER = 'Single class as a vector layer' - RECOGNITION_RESULT = 'Cosine distance between query image and map' - - @classmethod - def get_all_names(cls): - return [e.value for e in cls] - - @dataclass class MapProcessingParameters: """ @@ -48,9 +36,6 @@ class MapProcessingParameters: input_channels_mapping: ChannelsMapping # describes mapping of image channels to model inputs - model_output_format: ModelOutputFormat # what kind of model output do we want to achieve - model_output_format__single_class_number: int # if we want to show just one output channel - here is its number - @property def tile_size_m(self): return self.tile_size_px * self.resolution_cm_per_px / 100 diff --git a/src/deepness/deepness_dockwidget.py b/src/deepness/deepness_dockwidget.py index f27c654..cfaa582 100644 --- a/src/deepness/deepness_dockwidget.py +++ b/src/deepness/deepness_dockwidget.py @@ -16,8 +16,7 @@ from deepness.common.errors import OperationFailedException from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters, DetectorType -from deepness.common.processing_parameters.map_processing_parameters import (MapProcessingParameters, ModelOutputFormat, - ProcessedAreaType) +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters, ProcessedAreaType from deepness.common.processing_parameters.recognition_parameters import RecognitionParameters from deepness.common.processing_parameters.regression_parameters import RegressionParameters from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters @@ -79,9 +78,6 @@ def _load_ui_from_config(self): model_type_txt = ConfigEntryKey.MODEL_TYPE.get() self.comboBox_modelType.setCurrentText(model_type_txt) - model_output_format_txt = ConfigEntryKey.MODEL_OUTPUT_FORMAT.get() - self.comboBox_modelOutputFormat.setCurrentText(model_output_format_txt) - self._input_channels_mapping_widget.load_ui_from_config() self._training_data_export_widget.load_ui_from_config() @@ -92,7 +88,6 @@ def _load_ui_from_config(self): self._load_model_and_display_info(abort_if_no_file_path=True) # to prepare other ui components # needs to be loaded after the model is set up - self.comboBox_outputFormatClassNumber.setCurrentIndex(ConfigEntryKey.MODEL_OUTPUT_FORMAT_CLASS_NUMBER.get()) self.doubleSpinBox_resolution_cm_px.setValue(ConfigEntryKey.PREPROCESSING_RESOLUTION.get()) self.spinBox_batchSize.setValue(ConfigEntryKey.MODEL_BATCH_SIZE.get()) self.checkBox_local_cache.setChecked(ConfigEntryKey.PROCESS_LOCAL_CACHE.get()) @@ -125,10 +120,6 @@ def _save_ui_to_config(self): ConfigEntryKey.MODEL_TYPE.set(self.comboBox_modelType.currentText()) ConfigEntryKey.PROCESSED_AREA_TYPE.set(self.comboBox_processedAreaSelection.currentText()) - model_output_format = self.comboBox_modelOutputFormat.currentText() - ConfigEntryKey.MODEL_OUTPUT_FORMAT.set(model_output_format) - ConfigEntryKey.MODEL_OUTPUT_FORMAT_CLASS_NUMBER.set(self.comboBox_outputFormatClassNumber.currentIndex()) - ConfigEntryKey.PREPROCESSING_RESOLUTION.set(self.doubleSpinBox_resolution_cm_px.value()) ConfigEntryKey.MODEL_BATCH_SIZE.set(self.spinBox_batchSize.value()) ConfigEntryKey.PROCESS_LOCAL_CACHE.set(self.checkBox_local_cache.isChecked()) @@ -178,10 +169,6 @@ def _setup_misc_ui(self): self.comboBox_detectorType.addItem(detector_type) self._detector_type_changed() - for output_format_type in ModelOutputFormat.get_all_names(): - self.comboBox_modelOutputFormat.addItem(output_format_type) - self._model_output_format_changed() - self._rlayer_updated() # to force refresh the dependant ui elements def _set_processed_area_mask_options(self): @@ -207,7 +194,6 @@ def _create_connections(self): self.mMapLayerComboBox_inputLayer.layerChanged.connect(self._rlayer_updated) self.checkBox_pixelClassEnableThreshold.stateChanged.connect(self._set_probability_threshold_enabled) self.checkBox_removeSmallAreas.stateChanged.connect(self._set_remove_small_segment_enabled) - self.comboBox_modelOutputFormat.currentIndexChanged.connect(self._model_output_format_changed) self.radioButton_processingTileOverlapPercentage.toggled.connect(self._set_processing_overlap_enabled) self.radioButton_processingTileOverlapPixels.toggled.connect(self._set_processing_overlap_enabled) @@ -238,19 +224,10 @@ def _model_type_changed(self): self.mGroupBox_regressionParameters.setVisible(regression_enabled) self.mGroupBox_superresolutionParameters.setVisible(superresolution_enabled) self.mGroupBox_recognitionParameters.setVisible(recognition_enabled) - # Disable output format options for super-resolution or recognition models. - if recognition_enabled or superresolution_enabled: - self.mGroupBox_6.setEnabled(False) def _detector_type_changed(self): detector_type = DetectorType(self.comboBox_detectorType.currentText()) self.label_detectorTypeDescription.setText(detector_type.get_formatted_description()) - - def _model_output_format_changed(self): - txt = self.comboBox_modelOutputFormat.currentText() - model_output_format = ModelOutputFormat(txt) - class_number_selection_enabled = bool(model_output_format == ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER) - self.comboBox_outputFormatClassNumber.setEnabled(class_number_selection_enabled) def _set_processing_overlap_enabled(self): overlap_percentage_enabled = self.radioButton_processingTileOverlapPercentage.isChecked() @@ -398,7 +375,6 @@ def _load_model_and_display_info(self, abort_if_no_file_path: bool = False): scale_factor = output_0_shape[-1] / input_size_px self.doubleSpinBox_superresolutionScaleFactor.setValue(int(scale_factor)) # Disable output format options for super-resolution models - self.mGroupBox_6.setEnabled(False) except Exception as e: if IS_DEBUG: raise e @@ -418,18 +394,6 @@ def _load_model_and_display_info(self, abort_if_no_file_path: bool = False): detector_type = DetectorType(self.comboBox_detectorType.currentText()) self._model.set_model_type_param(detector_type) - self._update_model_output_format_mapping() - - def _update_model_output_format_mapping(self): - self.comboBox_outputFormatClassNumber: QComboBox - self.comboBox_outputFormatClassNumber.clear() - if not self._model: - return - - for output_number in range(self._model.get_number_of_output_channels()): - name = f'{output_number} - {self._model.get_channel_name(output_number)}' - self.comboBox_outputFormatClassNumber.addItem(name) - def get_mask_layer_id(self): if not self.get_selected_processed_area_type() == ProcessedAreaType.FROM_POLYGONS: return None @@ -565,8 +529,6 @@ def _get_map_processing_parameters(self) -> MapProcessingParameters: input_layer_id=self._get_input_layer_id(), processing_overlap=self._get_overlap_parameter(), input_channels_mapping=self._input_channels_mapping_widget.get_channels_mapping(), - model_output_format=ModelOutputFormat(self.comboBox_modelOutputFormat.currentText()), - model_output_format__single_class_number=self.comboBox_outputFormatClassNumber.currentIndex(), ) return params diff --git a/src/deepness/deepness_dockwidget.ui b/src/deepness/deepness_dockwidget.ui index c129dfd..78c6447 100644 --- a/src/deepness/deepness_dockwidget.ui +++ b/src/deepness/deepness_dockwidget.ui @@ -24,9 +24,9 @@ 0 - -423 + -268 452 - 1719 + 1564 @@ -755,65 +755,6 @@ - - - - - 0 - 0 - - - - Output format - - - - - - <html><head/><body><p>Determines how the model output should be presented.</p><p>E.g. whether we want to have the output layer only for one class, or for each class of the model.</p><p>Please refer to the plugin documentation for more details.</p></body></html> - - - - - - - Output format: - - - - - - - <html><head/><body><p>If selected, only this channel (class) of the model output will be presented as a result.</p></body></html> - - - - - - - - true - - - - NOTE: This configuration is depending on the model type. Please make sure to load the model first! - - - true - - - - - - - Single Class/channel -number: - - - - - - diff --git a/src/deepness/processing/map_processor/map_processor.py b/src/deepness/processing/map_processor/map_processor.py index 4f9814c..986ac3a 100644 --- a/src/deepness/processing/map_processor/map_processor.py +++ b/src/deepness/processing/map_processor/map_processor.py @@ -170,10 +170,12 @@ def limit_extended_extent_image_to_base_extent_with_mask(self, full_img): """ # TODO look for some inplace operation to save memory # cv2.copyTo(src=full_img, mask=area_mask_img, dst=full_img) # this doesn't work due to implementation details - full_img = cv2.copyTo(src=full_img, mask=self.area_mask_img) + + for i in range(full_img.shape[0]): + full_img[i] = cv2.copyTo(src=full_img[i], mask=self.area_mask_img) b = self.base_extent_bbox_in_full_image - result_img = full_img[b.y_min:b.y_max+1, b.x_min:b.x_max+1] + result_img = full_img[:, b.y_min:b.y_max+1, b.x_min:b.x_max+1] return result_img def _get_array_or_mmapped_array(self, final_shape_px): @@ -209,7 +211,7 @@ def tiles_generator(self) -> Tuple[np.ndarray, TileParams]: if not tile_params.is_tile_within_mask(self.area_mask_img): continue # tile outside of mask - to be skipped - + tile_img = processing_utils.get_tile_image( rlayer=self.rlayer, extent=tile_params.extent, params=self.params) diff --git a/src/deepness/processing/map_processor/map_processor_detection.py b/src/deepness/processing/map_processor/map_processor_detection.py index d0bfbfd..967a334 100644 --- a/src/deepness/processing/map_processor/map_processor_detection.py +++ b/src/deepness/processing/map_processor/map_processor_detection.py @@ -90,7 +90,9 @@ def limit_bounding_boxes_to_processed_area(self, bounding_boxes: List[Detection] return bounding_boxes_restricted def _create_result_message(self, bounding_boxes: List[Detection]) -> str: - channels = self._get_indexes_of_model_output_channels_to_create() + # hack, allways one output + model_outputs = self._get_indexes_of_model_output_channels_to_create() + channels = range(model_outputs[0]) counts_mapping = {} total_counts = 0 @@ -109,14 +111,18 @@ def _create_result_message(self, bounding_boxes: List[Detection]) -> str: else: counts_percentage = 0 - txt += f' - {self.model.get_channel_name(channel_id)}: counts = {counts} ({counts_percentage:.2f} %)\n' + txt += f' - {self.model.get_channel_name(0, channel_id)}: counts = {counts} ({counts_percentage:.2f} %)\n' return txt def _create_vlayer_for_output_bounding_boxes(self, bounding_boxes: List[Detection]): vlayers = [] - for channel_id in self._get_indexes_of_model_output_channels_to_create(): + # hack, allways one output + model_outputs = self._get_indexes_of_model_output_channels_to_create() + channels = range(model_outputs[0]) + + for channel_id in channels: filtered_bounding_boxes = [det for det in bounding_boxes if det.clss == channel_id] print(f'Detections for class {channel_id}: {len(filtered_bounding_boxes)}') @@ -166,7 +172,7 @@ def _create_vlayer_for_output_bounding_boxes(self, bounding_boxes: List[Detectio feature.setGeometry(geometry) features.append(feature) - vlayer = QgsVectorLayer("multipolygon", self.model.get_channel_name(channel_id), "memory") + vlayer = QgsVectorLayer("multipolygon", self.model.get_channel_name(0, channel_id), "memory") vlayer.setCrs(self.rlayer.crs()) prov = vlayer.dataProvider() diff --git a/src/deepness/processing/map_processor/map_processor_regression.py b/src/deepness/processing/map_processor/map_processor_regression.py index 5ad80b6..406fe8b 100644 --- a/src/deepness/processing/map_processor/map_processor_regression.py +++ b/src/deepness/processing/map_processor/map_processor_regression.py @@ -44,10 +44,9 @@ def _run(self) -> MapProcessingResult: tile_results_batched = self._process_tile(tile_img_batched) for tile_results, tile_params in zip(tile_results_batched, tile_params_batched): - for i in range(number_of_output_channels): - tile_params.set_mask_on_full_img( - tile_result=tile_results[i], - full_result_img=full_result_imgs[i]) + tile_params.set_mask_on_full_img( + tile_result=tile_results, + full_result_img=full_result_imgs) # plt.figure(); plt.imshow(full_result_img); plt.show(block=False); plt.pause(0.001) full_result_imgs = self.limit_extended_extent_images_to_base_extent_with_mask(full_imgs=full_result_imgs) @@ -67,7 +66,7 @@ def _create_result_message(self, result_imgs: List[np.ndarray]) -> str: result_img = result_imgs[i] average_value = np.mean(result_img) std = np.std(result_img) - txt += f' - {self.model.get_channel_name(channel_id)}: average_value = {average_value:.2f} (std = {std:.2f}, ' \ + txt += f' - {self.model.get_channel_name(0, channel_id)}: average_value = {average_value:.2f} (std = {std:.2f}, ' \ f'min={np.min(result_img)}, max={np.max(result_img)})\n' if len(channels) > 0: @@ -82,12 +81,7 @@ def limit_extended_extent_images_to_base_extent_with_mask(self, full_imgs: List[ :param full_imgs: :return: """ - result_imgs = [] - for i in range(len(full_imgs)): - result_img = self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_imgs[i]) - result_imgs.append(result_img) - - return result_imgs + return self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_imgs) def load_rlayer_from_file(self, file_path): """ @@ -110,7 +104,7 @@ def _create_rlayers_from_images_for_base_extent(self, result_imgs: List[np.ndarr for i, channel_id in enumerate(self._get_indexes_of_model_output_channels_to_create()): result_img = result_imgs[i] random_id = str(uuid.uuid4()).replace('-', '') - file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(channel_id)}___{random_id}.tif') + file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(0, channel_id)}___{random_id}.tif') self.save_result_img_as_tif(file_path=file_path, img=result_img) rlayer = self.load_rlayer_from_file(file_path) @@ -161,12 +155,22 @@ def save_result_img_as_tif(self, file_path: str, img: np.ndarray): print(f'***** {file_path = }') def _process_tile(self, tile_img: np.ndarray) -> np.ndarray: - result = self.model.process(tile_img) - result[np.isnan(result)] = 0 - result *= self.regression_parameters.output_scaling + many_result = self.model.process(tile_img) + many_outputs = [] + + for result in many_result: + result[np.isnan(result)] = 0 + result *= self.regression_parameters.output_scaling + + # NOTE - currently we are saving result as float32, so we are losing some accuraccy. + # result = np.clip(result, 0, 255) # old version with uint8_t - not used anymore + result = result.astype(np.float32) + + if len(result.shape) == 3: + result = np.expand_dims(result, axis=1) + + many_outputs.append(result[:, 0]) - # NOTE - currently we are saving result as float32, so we are losing some accuraccy. - # result = np.clip(result, 0, 255) # old version with uint8_t - not used anymore - result = result.astype(np.float32) + many_outputs = np.array(many_outputs).transpose((1, 0, 2, 3)) - return result + return many_outputs diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index c970a36..d609b05 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -18,10 +18,6 @@ class MapProcessorSegmentation(MapProcessorWithModel): """ MapProcessor specialized for Segmentation model (where each pixel is assigned to one class). - - Implementation note: due to opencv operations on arrays, it is easier to use value 0 for special meaning, - (that is pixel out of processing area) instead of just a class with number 0. - Therefore, internally during processing, pixels representing classes have value `class_number + 1`. """ def __init__(self, @@ -35,7 +31,7 @@ def __init__(self, self.model = params.model def _run(self) -> MapProcessingResult: - final_shape_px = (self.img_size_y_pixels, self.img_size_x_pixels) + final_shape_px = (len(self._get_indexes_of_model_output_channels_to_create()), self.img_size_y_pixels, self.img_size_x_pixels) full_result_img = self._get_array_or_mmapped_array(final_shape_px) @@ -43,8 +39,7 @@ def _run(self) -> MapProcessingResult: if self.isCanceled(): return MapProcessingResultCanceled() - # See note in the class description why are we adding/subtracting 1 here - tile_result_batched = self._process_tile(tile_img_batched) + 1 + tile_result_batched = self._process_tile(tile_img_batched) for tile_result, tile_params in zip(tile_result_batched, tile_params_batched): tile_params.set_mask_on_full_img( @@ -52,7 +47,10 @@ def _run(self) -> MapProcessingResult: full_result_img=full_result_img) blur_size = int(self.segmentation_parameters.postprocessing_dilate_erode_size // 2) * 2 + 1 # needs to be odd - full_result_img = cv2.medianBlur(full_result_img, blur_size) + + for i in range(full_result_img.shape[0]): + full_result_img[i] = cv2.medianBlur(full_result_img[i], blur_size) + full_result_img = self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_result_img) self.set_results_img(full_result_img) @@ -85,7 +83,8 @@ def _create_result_message(self, result_img: np.ndarray) -> str: area_percentage = area / total_area * 100 else: area_percentage = 0.0 - txt += f' - {self.model.get_channel_name(channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' + # TODO + txt += f' - {self.model.get_channel_name(0, channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' return txt @@ -95,48 +94,44 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: """ vlayers = [] - for channel_id in self._get_indexes_of_model_output_channels_to_create(): - # See note in the class description why are we adding/subtracting 1 here - local_mask_img = np.uint8(mask_img == (channel_id + 1)) - - # remove small areas - old implementation. Now we decided to do median blur, because the method below - # was producing pixels not belonging to any class - # local_mask_img = processing_utils.erode_dilate_image( - # img=local_mask_img, - # segmentation_parameters=self.segmentation_parameters) - - contours, hierarchy = cv2.findContours(local_mask_img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - contours = processing_utils.transform_contours_yx_pixels_to_target_crs( - contours=contours, - extent=self.base_extent, - rlayer_units_per_pixel=self.rlayer_units_per_pixel) - features = [] - - if len(contours): - processing_utils.convert_cv_contours_to_features( - features=features, - cv_contours=contours, - hierarchy=hierarchy[0], - is_hole=False, - current_holes=[], - current_contour_index=0) - else: - pass # just nothing, we already have an empty list of features - - vlayer = QgsVectorLayer("multipolygon", self.model.get_channel_name(channel_id), "memory") - vlayer.setCrs(self.rlayer.crs()) - prov = vlayer.dataProvider() - - color = vlayer.renderer().symbol().color() - OUTPUT_VLAYER_COLOR_TRANSPARENCY = 80 - color.setAlpha(OUTPUT_VLAYER_COLOR_TRANSPARENCY) - vlayer.renderer().symbol().setColor(color) - # TODO - add also outline for the layer (thicker black border) - - prov.addFeatures(features) - vlayer.updateExtents() - - vlayers.append(vlayer) + for layer_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): + for channel_id in range(layer_sizes): + # See note in the class description why are we adding/subtracting 1 here + local_mask_img = np.uint8(mask_img[layer_id] == channel_id) + + contours, hierarchy = cv2.findContours(local_mask_img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours = processing_utils.transform_contours_yx_pixels_to_target_crs( + contours=contours, + extent=self.base_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel) + features = [] + + if len(contours): + processing_utils.convert_cv_contours_to_features( + features=features, + cv_contours=contours, + hierarchy=hierarchy[0], + is_hole=False, + current_holes=[], + current_contour_index=0) + else: + pass # just nothing, we already have an empty list of features + + layer_name = self.model.get_channel_name(layer_id, channel_id) + vlayer = QgsVectorLayer("multipolygon", layer_name, "memory") + vlayer.setCrs(self.rlayer.crs()) + prov = vlayer.dataProvider() + + color = vlayer.renderer().symbol().color() + OUTPUT_VLAYER_COLOR_TRANSPARENCY = 80 + color.setAlpha(OUTPUT_VLAYER_COLOR_TRANSPARENCY) + vlayer.renderer().symbol().setColor(color) + # TODO - add also outline for the layer (thicker black border) + + prov.addFeatures(features) + vlayer.updateExtents() + + vlayers.append(vlayer) # accessing GUI from non-GUI thread is not safe, so we need to delegate it to the GUI thread def add_to_gui(): @@ -148,19 +143,25 @@ def add_to_gui(): return add_to_gui def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: - # TODO - create proper mapping for output channels - result = self.model.process(tile_img_batched) + many_result = self.model.process(tile_img_batched) + many_outputs = [] + + for result in many_result: + result[result < self.segmentation_parameters.pixel_classification__probability_threshold] = 0.0 - result[result < self.segmentation_parameters.pixel_classification__probability_threshold] = 0.0 + if len(result.shape) == 3: + result = np.expand_dims(result, axis=1) - if len(result.shape) == 3: - result = (result != 0).astype(int) - elif len(result.shape) == 4: if (result.shape[1] == 1): - result = (result != 0).astype(int)[:, 0] + result = (result != 0).astype(int) else: - result = np.argmax(result, axis=1) - else: - raise ValueError(f'Unexpected result shape: {result.shape}') + result = np.argmax(result, axis=1, keepdims=True) + + assert len(result.shape) == 4 + assert result.shape[1] == 1 + + many_outputs.append(result[:, 0]) - return result + many_outputs = np.array(many_outputs).transpose((1, 0, 2, 3)) + + return many_outputs diff --git a/src/deepness/processing/map_processor/map_processor_superresolution.py b/src/deepness/processing/map_processor/map_processor_superresolution.py index 047a4d6..b1d0d3d 100644 --- a/src/deepness/processing/map_processor/map_processor_superresolution.py +++ b/src/deepness/processing/map_processor/map_processor_superresolution.py @@ -32,6 +32,10 @@ def __init__(self, def _run(self) -> MapProcessingResult: number_of_output_channels = self.model.get_number_of_output_channels() + + # always one output + number_of_output_channels = number_of_output_channels[0] + final_shape_px = (int(self.img_size_y_pixels*self.superresolution_parameters.scale_factor), int(self.img_size_x_pixels*self.superresolution_parameters.scale_factor), number_of_output_channels) # NOTE: consider whether we can use float16/uint16 as datatype diff --git a/src/deepness/processing/map_processor/map_processor_with_model.py b/src/deepness/processing/map_processor/map_processor_with_model.py index b5a8c5e..f56a319 100644 --- a/src/deepness/processing/map_processor/map_processor_with_model.py +++ b/src/deepness/processing/map_processor/map_processor_with_model.py @@ -3,7 +3,6 @@ from typing import List -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat from deepness.processing.map_processor.map_processor import MapProcessor from deepness.processing.models.model_base import ModelBase @@ -25,21 +24,4 @@ def _get_indexes_of_model_output_channels_to_create(self) -> List[int]: Decide what model output channels/classes we want to use at presentation level (e.g. for which channels create a layer with results) """ - - output_channels = [] - if self.params.model_output_format == ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER: - channel = self.params.model_output_format__single_class_number - if channel >= self.model.get_number_of_output_channels(): - # we shouldn't get here, it should not be allowed to select it in the UI - raise Exception("Cannot get a bigger output channel than number of model outputs!") - output_channels.append(channel) - elif self.params.model_output_format == ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS: - output_channels = list(range(0, self.model.get_number_of_output_channels())) - elif self.params.model_output_format == ModelOutputFormat.CLASSES_AS_SEPARATE_LAYERS_WITHOUT_ZERO_CLASS: - output_channels = list(range(1, self.model.get_number_of_output_channels())) - elif self.params.model_output_format == ModelOutputFormat.RECOGNITION_RESULT: - output_channels = 1 - else: - raise Exception(f"Unhandled model output format {self.params.model_output_format}") - - return output_channels + return self.model.get_number_of_output_channels() diff --git a/src/deepness/processing/models/detector.py b/src/deepness/processing/models/detector.py index 5f3439c..1e7d3c1 100644 --- a/src/deepness/processing/models/detector.py +++ b/src/deepness/processing/models/detector.py @@ -1,6 +1,5 @@ """ Module including the class for the object detection task and related functions """ -import stat from dataclasses import dataclass from typing import List, Optional, Tuple @@ -140,7 +139,7 @@ def get_number_of_output_channels(self): """ class_names = self.get_class_names() if class_names is not None: - return len(class_names) # If class names are specified, we expect to have exactly this number of channels as specidied + return [len(class_names)] # If class names are specified, we expect to have exactly this number of channels as specidied model_type_params = self.model_type.get_parameters() @@ -148,10 +147,10 @@ def get_number_of_output_channels(self): if len(self.outputs_layers) == 1: if model_type_params.skipped_objectness_probability: - return self.outputs_layers[0].shape[shape_index] - 4 - return self.outputs_layers[0].shape[shape_index] - 4 - 1 # shape - 4 bboxes - 1 conf + return [self.outputs_layers[0].shape[shape_index] - 4] + return [self.outputs_layers[0].shape[shape_index] - 4 - 1] # shape - 4 bboxes - 1 conf elif len(self.outputs_layers) == 2 and self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION: - return self.outputs_layers[0].shape[shape_index] - 4 - self.outputs_layers[1].shape[1] + return [self.outputs_layers[0].shape[shape_index] - 4 - self.outputs_layers[1].shape[1]] else: raise NotImplementedError("Model with multiple output layer is not supported! Use only one output layer.") diff --git a/src/deepness/processing/models/model_base.py b/src/deepness/processing/models/model_base.py index e87f89a..3c7b97a 100644 --- a/src/deepness/processing/models/model_base.py +++ b/src/deepness/processing/models/model_base.py @@ -134,7 +134,7 @@ def get_class_names(self) -> Optional[List[str]]: return None - def get_channel_name(self, channel_id: int) -> str: + def get_channel_name(self, layer_id: int, channel_id: int) -> str: """ Get channel name by id if exists in model metadata Parameters @@ -149,7 +149,7 @@ def get_channel_name(self, channel_id: int) -> str: """ class_names = self.get_class_names() channel_id_str = str(channel_id) - default_return = f'channel_{channel_id_str}' + default_return = f'o_{layer_id}_{channel_id_str}' if class_names is not None and channel_id < len(class_names): return class_names[channel_id] @@ -398,7 +398,7 @@ def postprocessing(self, outs: List) -> np.ndarray: """ raise NotImplementedError('Base class not implemented!') - def get_number_of_output_channels(self) -> int: + def get_number_of_output_channels(self) -> List[int]: """ Abstract method for getting number of classes in the output layer Returns diff --git a/src/deepness/processing/models/preprocessing_utils.py b/src/deepness/processing/models/preprocessing_utils.py index 84cf3da..eff25fa 100644 --- a/src/deepness/processing/models/preprocessing_utils.py +++ b/src/deepness/processing/models/preprocessing_utils.py @@ -1,4 +1,3 @@ -import cv2 import numpy as np from deepness.common.processing_parameters.standardization_parameters import StandardizationParameters diff --git a/src/deepness/processing/models/recognition.py b/src/deepness/processing/models/recognition.py index f4901f2..12233e4 100644 --- a/src/deepness/processing/models/recognition.py +++ b/src/deepness/processing/models/recognition.py @@ -56,10 +56,9 @@ def get_number_of_output_channels(self): """ logging.warning(f"outputs_layers: {self.outputs_layers}") logging.info(f"outputs_layers: {self.outputs_layers}") - print(f"outputs_layers: {self.outputs_layers}") if len(self.outputs_layers) == 1: - return self.outputs_layers[0].shape[1] + return [self.outputs_layers[0].shape[1]] else: raise NotImplementedError( "Model with multiple output layers is not supported! Use only one output layer." diff --git a/src/deepness/processing/models/regressor.py b/src/deepness/processing/models/regressor.py index e991a4b..f995938 100644 --- a/src/deepness/processing/models/regressor.py +++ b/src/deepness/processing/models/regressor.py @@ -34,12 +34,11 @@ def postprocessing(self, model_output: List) -> np.ndarray: Returns ------- np.ndarray - Postprocessed batch of masks (N,H,W,C), 0-1 (one output channel) - + Output from the (Regression) model """ - return model_output[0] + return model_output - def get_number_of_output_channels(self) -> int: + def get_number_of_output_channels(self) -> List[int]: """ Returns number of channels in the output layer Returns @@ -47,10 +46,18 @@ def get_number_of_output_channels(self) -> int: int Number of channels in the output layer """ - if len(self.outputs_layers) == 1: - return self.outputs_layers[0].shape[-3] - else: - raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") + channels = [] + + for layer in self.outputs_layers: + if len(layer.shape) != 4 and len(layer.shape) != 3: + raise Exception(f'Output layer should have 3 or 4 dimensions: (Bs, H, W) or (Bs, Channels, H, W). Actually has: {layer.shape}') + + if len(layer.shape) == 3: + channels.append(1) + elif len(layer.shape) == 4: + channels.append(layer.shape[-3]) + + return channels @classmethod def get_class_display_name(cls) -> str: @@ -67,20 +74,19 @@ def check_loaded_model_outputs(self): """ Check if the model has correct output layers Correct means that: - - there is only one output layer - - output layer has 1 channel - - batch size is 1 + - there is at least one output layer + - batch size is 1 or parameter + - each output layer regresses only one channel - output resolution is square """ - if len(self.outputs_layers) == 1: - shape = self.outputs_layers[0].shape - - if len(shape) != 4: - raise Exception(f'Regression model output should have 4 dimensions: (Batch_size, Channels, H, W). \n' - f'Actually has: {shape}') - - if shape[2] != shape[3]: - raise Exception(f'Regression model can handle only square outputs masks. Has: {shape}') - - else: - raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") + for layer in self.outputs_layers: + if len(layer.shape) != 4 and len(layer.shape) != 3: + raise Exception(f'Output layer should have 3 or 4 dimensions: (Bs, H, W) or (Bs, Channels, H, W). Actually has: {layer.shape}') + + if len(layer.shape) == 4: + if layer.shape[2] != layer.shape[3]: + raise Exception(f'Regression model can handle only square outputs masks. Has: {layer.shape}') + + elif len(layer.shape) == 3: + if layer.shape[1] != layer.shape[2]: + raise Exception(f'Regression model can handle only square outputs masks. Has: {layer.shape}') diff --git a/src/deepness/processing/models/segmentor.py b/src/deepness/processing/models/segmentor.py index 091bba8..b2417a2 100644 --- a/src/deepness/processing/models/segmentor.py +++ b/src/deepness/processing/models/segmentor.py @@ -35,14 +35,11 @@ def postprocessing(self, model_output: List) -> np.ndarray: Returns ------- np.ndarray - Batch of postprocessed masks (N,H,W,C), 0-1 + Output from the (Segmentation) model """ - # labels = np.clip(model_output[0], 0, 1) - labels = model_output[0] # no need for clipping I think - see #149 - - return labels + return model_output - def get_number_of_output_channels(self): + def get_number_of_output_channels(self) -> List[int]: """ Returns model's number of class Returns @@ -50,16 +47,16 @@ def get_number_of_output_channels(self): int Number of channels in the output layer """ - if len(self.outputs_layers) == 1: - n_output_channels = self.outputs_layers[0].shape[-3] - # Support models that return a single output layer, which is converted to - # a binary mask. - if n_output_channels == 1: - return 2 - else: - return n_output_channels - else: - raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") + output_channels = [] + for layer in self.outputs_layers: + ls = layer.shape + + if len(ls) == 3: + output_channels.append(1) + elif len(ls) == 4: + output_channels.append(ls[-3]) + + return output_channels @classmethod def get_class_display_name(cls): @@ -76,20 +73,15 @@ def check_loaded_model_outputs(self): """ Checks if the model outputs are valid Valid means that: - - the model has only one output - - the output is 4D (N,C,H,W) - - the batch size is 1 + - the model has at least one output + - the output is 4D (N,C,H,W) or 3D (N,H,W) + - the batch size is 1 or dynamic - model resolution is equal to TILE_SIZE (is square) """ - if len(self.outputs_layers) == 1: - shape = self.outputs_layers[0].shape - - if len(shape) != 4: - raise Exception(f'Segmentation model output should have 4 dimensions: (B,C,H,W). Has {shape}') - - if shape[2] != shape[3]: - raise Exception(f'Segmentation model can handle only square outputs masks. Has: {shape}') + if len(self.outputs_layers) == 0: + raise Exception('Model has no output layers') - else: - raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") + for layer in self.outputs_layers: + if len(layer.shape) != 4 and len(layer.shape) != 3: + raise Exception(f'Segmentation model output should have 4 dimensions: (B,C,H,W) or 3 dimensions: (B,H,W). Has {layer.shape}') diff --git a/src/deepness/processing/models/superresolution.py b/src/deepness/processing/models/superresolution.py index 777dcdd..1fdaed8 100644 --- a/src/deepness/processing/models/superresolution.py +++ b/src/deepness/processing/models/superresolution.py @@ -39,7 +39,7 @@ def postprocessing(self, model_output: List) -> np.ndarray: """ return model_output[0] - def get_number_of_output_channels(self) -> int: + def get_number_of_output_channels(self) -> List[int]: """ Returns number of channels in the output layer Returns @@ -48,7 +48,7 @@ def get_number_of_output_channels(self) -> int: Number of channels in the output layer """ if len(self.outputs_layers) == 1: - return self.outputs_layers[0].shape[-3] + return [self.outputs_layers[0].shape[-3]] else: raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") diff --git a/src/deepness/processing/tile_params.py b/src/deepness/processing/tile_params.py index 8e06029..bea0b9e 100644 --- a/src/deepness/processing/tile_params.py +++ b/src/deepness/processing/tile_params.py @@ -83,7 +83,7 @@ def get_slice_on_full_image_for_entire_tile(self) -> Tuple[slice, slice]: y_min = self.start_pixel_y y_max = self.start_pixel_y + self.params.tile_size_px - 1 - roi_slice = np.s_[y_min:y_max + 1, x_min:x_max + 1] + roi_slice = np.s_[:, y_min:y_max + 1, x_min:x_max + 1] return roi_slice def get_slice_on_full_image_for_copying(self, tile_offset: int = 0): @@ -119,7 +119,7 @@ def get_slice_on_full_image_for_copying(self, tile_offset: int = 0): y_min += tile_offset y_max -= tile_offset - roi_slice = np.s_[y_min:y_max + 1, x_min:x_max + 1] + roi_slice = np.s_[:, y_min:y_max + 1, x_min:x_max + 1] return roi_slice def get_slice_on_tile_image_for_copying(self, roi_slice_on_full_image=None, tile_offset: int = 0): @@ -131,8 +131,9 @@ def get_slice_on_tile_image_for_copying(self, roi_slice_on_full_image=None, tile r = roi_slice_on_full_image roi_slice_on_tile = np.s_[ - r[0].start - self.start_pixel_y - tile_offset:r[0].stop - self.start_pixel_y - tile_offset, - r[1].start - self.start_pixel_x - tile_offset:r[1].stop - self.start_pixel_x - tile_offset + :, + r[1].start - self.start_pixel_y - tile_offset:r[1].stop - self.start_pixel_y - tile_offset, + r[2].start - self.start_pixel_x - tile_offset:r[2].stop - self.start_pixel_x - tile_offset ] return roi_slice_on_tile @@ -144,7 +145,7 @@ def is_tile_within_mask(self, mask_img: Optional[np.ndarray]): return True # if we don't have a mask, we are going to process all tiles roi_slice = self.get_slice_on_full_image_for_copying() - mask_roi = mask_img[roi_slice] + mask_roi = mask_img[roi_slice[0]] # check corners first if mask_roi[0, 0] and mask_roi[1, -1] and mask_roi[-1, 0] and mask_roi[-1, -1]: return True # all corners in mask, almost for sure a good tile @@ -153,8 +154,8 @@ def is_tile_within_mask(self, mask_img: Optional[np.ndarray]): return coverage_percentage > 0 # TODO - for training we can use tiles with higher coverage only def set_mask_on_full_img(self, full_result_img, tile_result): - if tile_result.shape[0] != self.params.tile_size_px or tile_result.shape[1] != self.params.tile_size_px: - tile_offset = (self.params.tile_size_px - tile_result.shape[0])//2 + if tile_result.shape[1] != self.params.tile_size_px or tile_result.shape[2] != self.params.tile_size_px: + tile_offset = (self.params.tile_size_px - tile_result.shape[1])//2 if tile_offset % 2 != 0: raise Exception("Model output shape is not even, cannot calculate offset") diff --git a/test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb b/test/data/dummy_model/dummy_regression_models/Untitled.ipynb similarity index 79% rename from test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb rename to test/data/dummy_model/dummy_regression_models/Untitled.ipynb index a6ff46d..be295a8 100644 --- a/test/data/dummy_model/dummy_segmentation_models/Untitled.ipynb +++ b/test/data/dummy_model/dummy_regression_models/Untitled.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "7ad5306f-18fc-4499-b764-ab6902ac301f", "metadata": {}, "outputs": [], @@ -13,7 +13,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 6, "id": "42e756a1-3ea0-4600-adcd-50ad14b938f8", "metadata": {}, "outputs": [ @@ -21,7 +21,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "torch.Size([1, 2, 512, 512]) torch.Size([1, 2, 512, 512])\n" + "torch.Size([1, 512, 512])\n" ] } ], @@ -32,7 +32,7 @@ " def __init__(self):\n", " super().__init__()\n", "\n", - " self.conv = nn.Conv2d(3, 2, 1)\n", + " self.conv = nn.Conv2d(3, 1, 1)\n", " \n", " self.softmax = nn.Softmax(dim=1)\n", " self.sigmoid = nn.Sigmoid()\n", @@ -40,22 +40,23 @@ " def forward(self, x):\n", " x = self.conv(x)\n", "\n", - " return self.softmax(x), self.softmax(x)\n", + " return self.sigmoid(x)[:,0] #, self.sigmoid(x)\n", "\n", "m = Model()\n", - "print(m(x)[0].shape, m(x)[1].shape)\n", + "# print(m(x)[0].shape, m(x)[1].shape)\n", + "print(m(x).shape)\n", "\n", "torch.onnx.export(m,\n", " x,\n", - " \"two_output_softmax_bsx2x512x512.onnx\",\n", + " \"one_output_sigmoid_bsx512x512.onnx\",\n", " export_params=True,\n", " opset_version=12,\n", " do_constant_folding=True,\n", " input_names = ['input'],\n", - " output_names = ['output', 'output2'],\n", + " output_names = ['output'],\n", " dynamic_axes={'input' : {0 : 'batch_size'},\n", " 'output' : {0 : 'batch_size'},\n", - " 'output2' : {0 : 'batch_size'},\n", + " # 'output2' : {0 : 'batch_size'},\n", " })" ] }, diff --git a/test/data/dummy_model/dummy_regression_model.onnx b/test/data/dummy_model/dummy_regression_models/dummy_regression_model.onnx similarity index 100% rename from test/data/dummy_model/dummy_regression_model.onnx rename to test/data/dummy_model/dummy_regression_models/dummy_regression_model.onnx diff --git a/test/data/dummy_model/dummy_regression_model_batched.onnx b/test/data/dummy_model/dummy_regression_models/dummy_regression_model_batched.onnx similarity index 100% rename from test/data/dummy_model/dummy_regression_model_batched.onnx rename to test/data/dummy_model/dummy_regression_models/dummy_regression_model_batched.onnx diff --git a/test/data/dummy_model/dummy_regression_models/one_output_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_regression_models/one_output_sigmoid_bsx1x512x512.onnx new file mode 100644 index 0000000000000000000000000000000000000000..346dd854bfa97ece0fc87f08c5616aab0be94892 GIT binary patch literal 415 zcmd;J7h*4{EXglQ&X8g?(lgRCuxexGTEfW1nweKnTEfMhoS#>wSDu=go>9WZ3Faha zCKd|`>x0<(&OjnQzqACXG~Pgp3n8e)0+P}a;^Iun%tD8T51>|HOO z(3eYWon_E>uH5>{QzQcXg3wLhmmL?6II zU!dnwAEeMnX=zNNDPnionVtE*nVp@%8qtDB>4!tNswbPs@Ydm zx-2vciMC3h&<8H!CJ~rY?m6R-2Cj925za>k8L0DjIz1i#TwW}{>mO$?UrVbT6S9f* zl>X=9yQzwS)(5fmoq7^Nn93O z*qkB6262#-fPQgidTxGZiheMJR$_-TxU8}3WQXW9k`jVxjW@*5EX0+Yn3)%!UX)mn zp{2^f$iWQ6i~@{K$YJHhGgoETzGG}t_Pt1YxKB$ItN;;yUM!O;U+#_4h6b~c5|;u8 zqmT?2j}TWUZA5t}HiDWU12(^uY zM_fzWlaQ407gA93XDhYk-G}#M_Z6LfJbk`tg#{pPueMNsnVxhG!!E=s>_`ZgLB`M} pyk{AV-kFe=GLVt(g~it0hp?7-8_GVqxP*?82M}rb586yC@)x^w=#&5e literal 0 HcmV?d00001 diff --git a/test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/two_outputs_sigmoid_bsx1x512x512.onnx similarity index 100% rename from test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx1x512x512.onnx rename to test/data/dummy_model/dummy_segmentation_models/two_outputs_sigmoid_bsx1x512x512.onnx diff --git a/test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/two_outputs_sigmoid_bsx512x512.onnx similarity index 100% rename from test/data/dummy_model/dummy_segmentation_models/two_output_sigmoid_bsx512x512.onnx rename to test/data/dummy_model/dummy_segmentation_models/two_outputs_sigmoid_bsx512x512.onnx diff --git a/test/data/dummy_model/dummy_segmentation_models/two_output_softmax_bsx2x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/two_outputs_softmax_bsx2x512x512.onnx similarity index 100% rename from test/data/dummy_model/dummy_segmentation_models/two_output_softmax_bsx2x512x512.onnx rename to test/data/dummy_model/dummy_segmentation_models/two_outputs_softmax_bsx2x512x512.onnx diff --git a/test/manual_test_map_processor_detection_yolo_ultralytics.py b/test/manual_test_map_processor_detection_yolo_ultralytics.py index 7d796cc..9cbc754 100644 --- a/test/manual_test_map_processor_detection_yolo_ultralytics.py +++ b/test/manual_test_map_processor_detection_yolo_ultralytics.py @@ -5,7 +5,7 @@ from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters, DetectorType -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection from deepness.processing.models.detector import Detector @@ -39,8 +39,6 @@ def test_map_processor_detection_yolo_ultralytics(): model=model_wrapper, confidence=0.5, iou_threshold=0.4, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, detector_type=DetectorType.YOLO_ULTRALYTICS, ) diff --git a/test/manual_test_map_processor_detection_yolov6.py b/test/manual_test_map_processor_detection_yolov6.py index f14405c..c5050fd 100644 --- a/test/manual_test_map_processor_detection_yolov6.py +++ b/test/manual_test_map_processor_detection_yolov6.py @@ -5,7 +5,7 @@ from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters, DetectorType -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection from deepness.processing.models.detector import Detector @@ -39,8 +39,6 @@ def test_map_processor_detection_yolov6(): model=model_wrapper, confidence=0.9, iou_threshold=0.4, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, detector_type=DetectorType.YOLO_v6, ) diff --git a/test/manual_test_map_processor_instance_yolo_ultralytics.py b/test/manual_test_map_processor_instance_yolo_ultralytics.py index 38be37c..75deae8 100644 --- a/test/manual_test_map_processor_instance_yolo_ultralytics.py +++ b/test/manual_test_map_processor_instance_yolo_ultralytics.py @@ -5,7 +5,7 @@ from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters, DetectorType -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection from deepness.processing.models.detector import Detector @@ -39,8 +39,6 @@ def test_map_processor_detection_yolo_ultralytics(): model=model_wrapper, confidence=0.5, iou_threshold=0.4, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, detector_type=DetectorType.YOLO_ULTRALYTICS_SEGMENTATION, ) diff --git a/test/test_deepness_dockwidget.py b/test/test_deepness_dockwidget.py index fa88f78..629fd3c 100644 --- a/test/test_deepness_dockwidget.py +++ b/test/test_deepness_dockwidget.py @@ -15,8 +15,7 @@ from deepness.common.channels_mapping import ChannelsMapping from deepness.common.config_entry_key import ConfigEntryKey -from deepness.common.processing_parameters.map_processing_parameters import (MapProcessingParameters, ModelOutputFormat, - ProcessedAreaType) +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters, ProcessedAreaType from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.common.processing_parameters.training_data_export_parameters import TrainingDataExportParameters from deepness.deepness_dockwidget import DeepnessDockWidget @@ -78,8 +77,6 @@ def test_get_inference_parameters(): ConfigEntryKey.PREPROCESSING_RESOLUTION.set(7) ConfigEntryKey.PROCESSED_AREA_TYPE.set(ProcessedAreaType.VISIBLE_PART.value) ConfigEntryKey.PREPROCESSING_TILES_OVERLAP.set(44) - ConfigEntryKey.MODEL_OUTPUT_FORMAT.set(ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER.value) - ConfigEntryKey.MODEL_OUTPUT_FORMAT_CLASS_NUMBER.set(1) dockwidget = DeepnessDockWidget(iface=MagicMock()) dockwidget._get_input_layer_id = MagicMock(return_value=1) # fake input layer id, just to test @@ -96,8 +93,6 @@ def test_get_inference_parameters(): assert params.input_channels_mapping.get_number_of_model_inputs() == 3 assert params.input_channels_mapping.get_number_of_image_channels() == 4 assert params.input_channels_mapping.get_image_channel_index_for_model_input(2) == 2 - assert params.model_output_format == ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER - assert params.model_output_format__single_class_number == 1 if __name__ == '__main__': diff --git a/test/test_map_processor_detection_oils_example.py b/test/test_map_processor_detection_oils_example.py index d426807..bf56f55 100644 --- a/test/test_map_processor_detection_oils_example.py +++ b/test/test_map_processor_detection_oils_example.py @@ -8,7 +8,7 @@ from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection from deepness.processing.models.detector import Detector @@ -40,8 +40,6 @@ def test_map_processor_detection_oil_example(): model=model_wrapper, confidence=0.5, iou_threshold=0.1, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, ) map_processor = MapProcessorDetection( @@ -72,8 +70,6 @@ def test_map_processor_detection_oil_example_using_cache(): model=model_wrapper, confidence=0.5, iou_threshold=0.1, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, ) map_processor = MapProcessorDetection( @@ -104,8 +100,6 @@ def test_map_processor_detection_oil_example_with_remove_small(): model=model_wrapper, confidence=0.5, iou_threshold=0.3, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, ) map_processor = MapProcessorDetection( diff --git a/test/test_map_processor_detection_planes_example.py b/test/test_map_processor_detection_planes_example.py index b1d7186..2f536dc 100644 --- a/test/test_map_processor_detection_planes_example.py +++ b/test/test_map_processor_detection_planes_example.py @@ -5,7 +5,7 @@ from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection from deepness.processing.models.detector import Detector @@ -37,8 +37,6 @@ def test_map_processor_detection_planes_example(): model=model_wrapper, confidence=0.5, iou_threshold=0.4, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, ) map_processor = MapProcessorDetection( diff --git a/test/test_map_processor_empty_detection.py b/test/test_map_processor_empty_detection.py index 53a43eb..e7856c3 100644 --- a/test/test_map_processor_empty_detection.py +++ b/test/test_map_processor_empty_detection.py @@ -6,7 +6,7 @@ from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.detection_parameters import DetectionParameters -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection from deepness.processing.models.detector import Detector @@ -38,8 +38,6 @@ def test_map_processor_empty_detection(): model=model_wrapper, confidence=0.99, iou_threshold=0.99, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, ) map_processor = MapProcessorDetection( diff --git a/test/test_map_processor_recognition.py b/test/test_map_processor_recognition.py index 7b0723e..78b6040 100644 --- a/test/test_map_processor_recognition.py +++ b/test/test_map_processor_recognition.py @@ -9,7 +9,7 @@ from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.recognition_parameters import RecognitionParameters from deepness.processing.map_processor.map_processor_recognition import MapProcessorRecognition from deepness.processing.models.recognition import Recognition @@ -42,8 +42,6 @@ def test_dummy_model_processing__entire_file(): input_layer_id=rlayer.id(), input_channels_mapping=INPUT_CHANNELS_MAPPING, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=0), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, query_image_path=IMAGE_FILE_PATH, ) diff --git a/test/test_map_processor_regression.py b/test/test_map_processor_regression.py index 090790e..0e24487 100644 --- a/test/test_map_processor_regression.py +++ b/test/test_map_processor_regression.py @@ -8,11 +8,9 @@ from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.regression_parameters import RegressionParameters -from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.processing.map_processor.map_processor_regression import MapProcessorRegression -from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.models.regressor import Regressor from deepness.processing.models.segmentor import Segmentor @@ -47,8 +45,6 @@ def test_dummy_model_processing__entire_file(): input_channels_mapping=INPUT_CHANNELS_MAPPING, output_scaling=1.0, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -83,8 +79,6 @@ def test_dummy_model_processing__entire_file_batched(): input_channels_mapping=INPUT_CHANNELS_MAPPING, output_scaling=1.0, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -118,8 +112,6 @@ def test_dummy_model_processing__entire_file_with_cache(): input_channels_mapping=INPUT_CHANNELS_MAPPING, output_scaling=1.0, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) diff --git a/test/test_map_processor_regression_many_output_types.py b/test/test_map_processor_regression_many_output_types.py new file mode 100644 index 0000000..4abcf44 --- /dev/null +++ b/test/test_map_processor_regression_many_output_types.py @@ -0,0 +1,163 @@ +from test.test_utils import (create_default_input_channels_mapping_for_rgba_bands, create_rlayer_from_file, + get_dummy_fotomap_small_path, get_dummy_regression_models_dict, init_qgis) +from unittest.mock import MagicMock + +import numpy as np +from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle + +from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType +from deepness.common.processing_parameters.regression_parameters import RegressionParameters +from deepness.processing.map_processor.map_processor_regression import MapProcessorRegression +from deepness.processing.models.regressor import Regressor + +RASTER_FILE_PATH = get_dummy_fotomap_small_path() +INPUT_CHANNELS_MAPPING = create_default_input_channels_mapping_for_rgba_bands() + + +MODEL_FILES_DICT = get_dummy_regression_models_dict() + +# 'one_output': { +# '1x1x512x512' +# '1x512x512' +# }, +# 'two_outputs': { +# '1x1x512x512' +# '1x512x512' +# } + + +def test_dummy_model_regression_processing__1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Regressor(MODEL_FILES_DICT['one_output']['1x512x512']) + + params = RegressionParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + output_scaling=1.0, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + model=model, + ) + + map_processor = MapProcessorRegression( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_imgs = map_processor.get_result_img() + + assert result_imgs.shape == (1, 561, 829) + +def test_dummy_model_regression_processing__1x1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Regressor(MODEL_FILES_DICT['one_output']['1x1x512x512']) + + params = RegressionParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + output_scaling=1.0, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + model=model, + ) + + map_processor = MapProcessorRegression( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_imgs = map_processor.get_result_img() + + assert result_imgs.shape == (1, 561, 829) + +def test_dummy_model_regression_processing__1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Regressor(MODEL_FILES_DICT['two_outputs']['1x512x512']) + + params = RegressionParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + output_scaling=1.0, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + model=model, + ) + + map_processor = MapProcessorRegression( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_imgs = map_processor.get_result_img() + + assert result_imgs.shape == (2, 561, 829) + +def test_dummy_model_regression_processing__1x1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Regressor(MODEL_FILES_DICT['two_outputs']['1x1x512x512']) + + params = RegressionParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + output_scaling=1.0, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + model=model, + ) + + map_processor = MapProcessorRegression( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_imgs = map_processor.get_result_img() + + assert result_imgs.shape == (2, 561, 829) + +if __name__ == '__init__': + test_dummy_model_regression_processing__1x512x512() + test_dummy_model_regression_processing__1x1x512x512() + test_dummy_model_regression_processing__1x512x512() + test_dummy_model_regression_processing__1x1x512x512() + print('All tests passed!') diff --git a/test/test_map_processor_segmentation.py b/test/test_map_processor_segmentation.py index e6b3062..77cb2f6 100644 --- a/test/test_map_processor_segmentation.py +++ b/test/test_map_processor_segmentation.py @@ -8,7 +8,7 @@ from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.models.segmentor import Segmentor @@ -27,6 +27,15 @@ 638840.370, 5802593.197, 638857.695, 5802601.792) +def model_process_mock_one_channel(x): + x = x[:, :, :, 0:1] + x = np.transpose(x, (0, 3, 1, 2)) + + return x + +def model_process_mock_two_channels(x): + return [model_process_mock_one_channel(x), model_process_mock_one_channel(x)] + def test_dummy_model_processing__entire_file(): qgs = init_qgis() @@ -46,8 +55,6 @@ def test_dummy_model_processing__entire_file(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -61,7 +68,7 @@ def test_dummy_model_processing__entire_file(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (561, 829) + assert result_img.shape == (1, 561, 829) # TODO - add detailed check for pixel values once we have output channels mapping with thresholding def test_dummy_model_processing__entire_file_overlap_in_pixels(): @@ -82,8 +89,6 @@ def test_dummy_model_processing__entire_file_overlap_in_pixels(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PIXELS, overlap_px=int(model.get_input_size_in_pixels()[0] * 0.2)), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -97,23 +102,19 @@ def test_dummy_model_processing__entire_file_overlap_in_pixels(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (561, 829) + assert result_img.shape == (1, 561, 829) -def model_process_mock(x): - x = x[:, :, :, 0:2] - return np.transpose(x, (0, 3, 1, 2)) - -def test_generic_processing_test__specified_extent_from_vlayer(): +def test_generic_processing_test__specified_extent_from_vlayer_one_channel(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) vlayer_mask = create_vlayer_from_file(VLAYER_MASK_FILE_PATH) model = MagicMock() - model.process = model_process_mock - model.get_number_of_channels = lambda: 2 - model.get_number_of_output_channels = lambda: 2 - model.get_channel_name = lambda x: str(x) + model.process = model_process_mock_one_channel + model.get_number_of_channels = lambda: 3 + model.get_number_of_output_channels = lambda: [1] + model.get_channel_name = lambda y, x: str(y)+'_'+str(x) params = SegmentationParameters( resolution_cm_per_px=3, @@ -127,8 +128,6 @@ def test_generic_processing_test__specified_extent_from_vlayer(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER, - model_output_format__single_class_number=1, model=model, ) map_processor = MapProcessorSegmentation( @@ -141,24 +140,70 @@ def test_generic_processing_test__specified_extent_from_vlayer(): # just run - we will check the results in a more detailed test map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (524, 733) + assert result_img.shape == (1, 524, 733) # just check a few pixels assert all(result_img.ravel()[[365, 41234, 59876, 234353, 111222, 134534, 223423, 65463, 156451]] == - np.asarray([0, 2, 2, 2, 2, 0, 0, 2, 0])) + np.asarray([0, 1, 1, 1, 1, 0, 0, 1, 0])) + # and counts of different values - np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([166903, 45270, 171919]), atol=3) + np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([178063, 206029]), atol=3) -def test_generic_processing_test__specified_extent_from_vlayer_using_cache(): +def test_generic_processing_test__specified_extent_from_vlayer_two_channels(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) vlayer_mask = create_vlayer_from_file(VLAYER_MASK_FILE_PATH) model = MagicMock() - model.process = model_process_mock - model.get_number_of_channels = lambda: 2 - model.get_number_of_output_channels = lambda: 2 - model.get_channel_name = lambda x: str(x) + model.process = model_process_mock_two_channels + model.get_number_of_channels = lambda: 3 + model.get_number_of_output_channels = lambda: [1, 1] + model.get_channel_name = lambda y, x: str(y)+'_'+str(x) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=512, + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.FROM_POLYGONS, + mask_layer_id=vlayer_mask.id(), + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model=model, + ) + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=vlayer_mask, + map_canvas=MagicMock(), + params=params, + ) + + # just run - we will check the results in a more detailed test + map_processor.run() + result_img = map_processor.get_result_img() + assert result_img.shape == (2, 524, 733) + + # just check a few pixels + assert all(result_img.ravel()[[365, 41234, 59876, 234353, 111222, 134534, 223423, 65463, 156451]] == + np.asarray([0, 1, 1, 1, 1, 0, 0, 1, 0])) + + # and counts of different values + np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([356126, 412058]), atol=3) + + +def test_generic_processing_test__specified_extent_from_vlayer_crs3857_one_channel(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + vlayer_mask = create_vlayer_from_file(VLAYER_MASK_CRS3857_FILE_PATH) + model = MagicMock() + model.process = model_process_mock_one_channel + model.get_number_of_channels = lambda: 3 + model.get_number_of_output_channels = lambda: [1] + model.get_channel_name = lambda y, x: str(y)+'_'+str(x) params = SegmentationParameters( resolution_cm_per_px=3, @@ -172,8 +217,6 @@ def test_generic_processing_test__specified_extent_from_vlayer_using_cache(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER, - model_output_format__single_class_number=1, model=model, ) map_processor = MapProcessorSegmentation( @@ -186,24 +229,24 @@ def test_generic_processing_test__specified_extent_from_vlayer_using_cache(): # just run - we will check the results in a more detailed test map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (524, 733) + assert result_img.shape == (1, 550, 723) # just check a few pixels assert all(result_img.ravel()[[365, 41234, 59876, 234353, 111222, 134534, 223423, 65463, 156451]] == - np.asarray([0, 2, 2, 2, 2, 0, 0, 2, 0])) + np.asarray([0, 0, 1, 1, 1, 0, 0, 1, 0])) # and counts of different values - np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([166903, 45270, 171919]), atol=3) + np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([193853, 203797]), atol=3) -def test_generic_processing_test__specified_extent_from_vlayer_crs3857(): +def test_generic_processing_test__specified_extent_from_vlayer_crs3857_two_channels(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) vlayer_mask = create_vlayer_from_file(VLAYER_MASK_CRS3857_FILE_PATH) model = MagicMock() - model.process = model_process_mock - model.get_number_of_channels = lambda: 2 - model.get_number_of_output_channels = lambda: 2 - model.get_channel_name = lambda x: str(x) + model.process = model_process_mock_two_channels + model.get_number_of_channels = lambda: 3 + model.get_number_of_output_channels = lambda: [1, 1] + model.get_channel_name = lambda y, x: str(y)+'_'+str(x) params = SegmentationParameters( resolution_cm_per_px=3, @@ -217,8 +260,6 @@ def test_generic_processing_test__specified_extent_from_vlayer_crs3857(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ONLY_SINGLE_CLASS_AS_LAYER, - model_output_format__single_class_number=1, model=model, ) map_processor = MapProcessorSegmentation( @@ -234,23 +275,24 @@ def test_generic_processing_test__specified_extent_from_vlayer_crs3857(): # for the same vlayer_mask, but with a different encoding we had quite different shaep (524, 733). # I'm not sure if it is rounding issue in Qgis Transform or some bug in plugin - assert result_img.shape == (550, 723) + assert result_img.shape == (2, 550, 723) # just check a few pixels assert all(result_img.ravel()[[365, 41234, 59876, 234353, 111222, 134534, 223423, 65463, 156451]] == - np.asarray([0, 0, 2, 2, 2, 0, 0, 2, 0])) - np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([182693, 44926, 170031]), atol=3) + np.asarray([0, 0, 1, 1, 1, 0, 0, 1, 0])) + + np.testing.assert_allclose(np.unique(result_img, return_counts=True)[1], np.array([387706, 407594]), atol=3) -def test_generic_processing_test__specified_extent_from_active_map_extent(): +def test_generic_processing_test__specified_extent_from_active_map_extent_one_channel(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) model = MagicMock() - model.process = model_process_mock - model.get_number_of_channels = lambda: 2 - model.get_number_of_output_channels = lambda: 2 - model.get_channel_name = lambda x: str(x) + model.process = model_process_mock_one_channel + model.get_number_of_channels = lambda: 3 + model.get_number_of_output_channels = lambda: [1] + model.get_channel_name = lambda y, x: str(y)+'_'+str(x) params = SegmentationParameters( resolution_cm_per_px=3, @@ -264,8 +306,6 @@ def test_generic_processing_test__specified_extent_from_active_map_extent(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.CLASSES_AS_SEPARATE_LAYERS_WITHOUT_ZERO_CLASS, - model_output_format__single_class_number=-1, model=model, ) processed_extent = PROCESSED_EXTENT_1 @@ -287,10 +327,55 @@ def test_generic_processing_test__specified_extent_from_active_map_extent(): # just run - we will check the results in a more detailed test map_processor.run() +def test_generic_processing_test__specified_extent_from_active_map_extent_two_channels(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = MagicMock() + model.process = model_process_mock_two_channels + model.get_number_of_channels = lambda: 3 + model.get_number_of_output_channels = lambda: [1, 1] + model.get_channel_name = lambda y, x: str(y)+'_'+str(x) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=512, + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.VISIBLE_PART, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model=model, + ) + processed_extent = PROCESSED_EXTENT_1 + + # we want to use a fake extent, which is the Visible Part of the map, + # so we need to mock its function calls + params.processed_area_type = ProcessedAreaType.VISIBLE_PART + map_canvas = MagicMock() + map_canvas.extent = lambda: processed_extent + map_canvas.mapSettings().destinationCrs = lambda: QgsCoordinateReferenceSystem("EPSG:32633") + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=map_canvas, + params=params, + ) + + # just run - we will check the results in a more detailed test + map_processor.run() if __name__ == '__main__': # test_dummy_model_processing__entire_file() - test_generic_processing_test__specified_extent_from_vlayer() - test_generic_processing_test__specified_extent_from_vlayer_crs3857() - test_generic_processing_test__specified_extent_from_active_map_extent() + test_generic_processing_test__specified_extent_from_vlayer_one_channel() + test_generic_processing_test__specified_extent_from_vlayer_two_channels() + test_generic_processing_test__specified_extent_from_vlayer_crs3857_one_channel() + test_generic_processing_test__specified_extent_from_vlayer_crs3857_two_channels() + test_generic_processing_test__specified_extent_from_active_map_extent_one_channel() + test_generic_processing_test__specified_extent_from_active_map_extent_two_channels() print('Done') diff --git a/test/test_map_processor_segmentation_different_output_size.py b/test/test_map_processor_segmentation_different_output_size.py index b75757a..fea8a15 100644 --- a/test/test_map_processor_segmentation_different_output_size.py +++ b/test/test_map_processor_segmentation_different_output_size.py @@ -9,7 +9,7 @@ from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.models.segmentor import Segmentor @@ -47,8 +47,6 @@ def test_dummy_model_processing_when_different_output_size(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -62,7 +60,7 @@ def test_dummy_model_processing_when_different_output_size(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (561, 829) + assert result_img.shape == (1, 561, 829) # TODO - add detailed check for pixel values once we have output channels mapping with thresholding if __name__ == "__main__": diff --git a/test/test_map_processor_segmentation_landcover_example.py b/test/test_map_processor_segmentation_landcover_example.py index 697c148..5ece49a 100644 --- a/test/test_map_processor_segmentation_landcover_example.py +++ b/test/test_map_processor_segmentation_landcover_example.py @@ -1,12 +1,12 @@ import os from pathlib import Path -import numpy as np - from test.test_utils import create_default_input_channels_mapping_for_rgb_bands, create_rlayer_from_file, init_qgis from unittest.mock import MagicMock +import numpy as np + from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.models.segmentor import Segmentor @@ -38,8 +38,6 @@ def test_map_processor_segmentation_landcover_example(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -53,22 +51,22 @@ def test_map_processor_segmentation_landcover_example(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (2351, 2068) + assert result_img.shape == (1, 2351, 2068) - assert result_img[1000, 1000] == 1 - assert result_img[2000, 2000] == 3 - assert result_img[150:300, 150:300].sum() == 41478 + assert result_img[0, 1000, 1000] == 0 + assert result_img[0, 2000, 2000] == 2 + assert np.isclose(result_img[0, 150:300, 150:300].sum(), 18978, rtol=3) unique, counts = np.unique(result_img, return_counts=True) counts = dict(zip(unique, counts)) gt_counts = { - 1: 3294546, - 2: 71169, - 3: 1054899, - 4: 365915, - 5: 75339, + 0: 3294546, + 1: 71169, + 2: 1054899, + 3: 365915, + 4: 75339, } assert set(counts.keys()) == set(gt_counts.keys()) diff --git a/test/test_map_processor_segmentation_landcover_example_batched.py b/test/test_map_processor_segmentation_landcover_example_batched.py index ac38bc0..9be6f97 100644 --- a/test/test_map_processor_segmentation_landcover_example_batched.py +++ b/test/test_map_processor_segmentation_landcover_example_batched.py @@ -1,12 +1,12 @@ import os from pathlib import Path -import numpy as np - from test.test_utils import create_default_input_channels_mapping_for_rgb_bands, create_rlayer_from_file, init_qgis from unittest.mock import MagicMock +import numpy as np + from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.models.segmentor import Segmentor @@ -38,8 +38,6 @@ def test_map_processor_segmentation_landcover_example(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -53,22 +51,22 @@ def test_map_processor_segmentation_landcover_example(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (2351, 2068) + assert result_img.shape == (1, 2351, 2068) - assert result_img[1000, 1000] == 1 - assert result_img[2000, 2000] == 3 - assert result_img[150:300, 150:300].sum() == 41478 + assert result_img[0, 1000, 1000] == 0 + assert result_img[0, 2000, 2000] == 2 + assert np.isclose(result_img[0, 150:300, 150:300].sum(), 18978, rtol=3) unique, counts = np.unique(result_img, return_counts=True) counts = dict(zip(unique, counts)) gt_counts = { - 1: 3294546, - 2: 71169, - 3: 1054899, - 4: 365915, - 5: 75339, + 0: 3294546, + 1: 71169, + 2: 1054899, + 3: 365915, + 4: 75339, } assert set(counts.keys()) == set(gt_counts.keys()) diff --git a/test/test_map_processor_segmentation_many_output_types.py b/test/test_map_processor_segmentation_many_output_types.py index c760e07..415ee23 100644 --- a/test/test_map_processor_segmentation_many_output_types.py +++ b/test/test_map_processor_segmentation_many_output_types.py @@ -7,7 +7,7 @@ from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.models.segmentor import Segmentor @@ -29,7 +29,7 @@ # } -def test_dummy_model_processing__1x1x512x512(): +def test_dummy_model_segmentation_processing__1x1x512x512(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -47,8 +47,6 @@ def test_dummy_model_processing__1x1x512x512(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -62,9 +60,9 @@ def test_dummy_model_processing__1x1x512x512(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (561, 829) + assert result_img.shape == (1, 561, 829) -def test_dummy_model_processing__1x512x512(): +def test_dummy_model_segmentation_processing__1x512x512(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -82,8 +80,6 @@ def test_dummy_model_processing__1x512x512(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -97,9 +93,9 @@ def test_dummy_model_processing__1x512x512(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (561, 829) + assert result_img.shape == (1, 561, 829) -def test_dummy_model_processing__1x2x512x512(): +def test_dummy_model_segmentation_processing__1x2x512x512(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -117,8 +113,6 @@ def test_dummy_model_processing__1x2x512x512(): postprocessing_dilate_erode_size=5, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), pixel_classification__probability_threshold=0.5, - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -132,4 +126,117 @@ def test_dummy_model_processing__1x2x512x512(): map_processor.run() result_img = map_processor.get_result_img() - assert result_img.shape == (561, 829) + assert result_img.shape == (1, 561, 829) + +# two outputs + +def test_dummy_model_segmentation_processing__two_outputs_1x1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILES_DICT['two_outputs']['1x1x512x512']) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (2, 561, 829) + +def test_dummy_model_segmentation_processing__two_outputs_1x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILES_DICT['two_outputs']['1x512x512']) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (2, 561, 829) + + + +def test_dummy_model_segmentation_processing__two_outputs_1x2x512x512(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILES_DICT['two_outputs']['1x2x512x512']) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.5, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (2, 561, 829) + +if __name__ == '__main__': + test_dummy_model_segmentation_processing__1x1x512x512() + test_dummy_model_segmentation_processing__1x512x512() + test_dummy_model_segmentation_processing__1x2x512x512() + + test_dummy_model_segmentation_processing__two_outputs_1x1x512x512() + test_dummy_model_segmentation_processing__two_outputs_1x512x512() + test_dummy_model_segmentation_processing__two_outputs_1x2x512x512() + print('All tests passed') diff --git a/test/test_map_processor_superresolution.py b/test/test_map_processor_superresolution.py index d2f5630..fb7d85b 100644 --- a/test/test_map_processor_superresolution.py +++ b/test/test_map_processor_superresolution.py @@ -7,10 +7,8 @@ from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType -from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.superresolution_parameters import SuperresolutionParameters -from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation from deepness.processing.map_processor.map_processor_superresolution import MapProcessorSuperresolution from deepness.processing.models.segmentor import Segmentor from deepness.processing.models.superresolution import Superresolution @@ -46,8 +44,6 @@ def test_dummy_model_processing__entire_file(): output_scaling=1.0, scale_factor=2.0, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=0), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -83,8 +79,6 @@ def test_dummy_model_processing__entire_file_cached(): output_scaling=1.0, scale_factor=2.0, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=0), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) @@ -119,8 +113,6 @@ def test_dummy_model_processing__entire_file_batched(): output_scaling=1.0, scale_factor=2.0, processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=0), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, model=model, ) diff --git a/test/test_map_processor_training_data_export.py b/test/test_map_processor_training_data_export.py index 2deeb85..7ce6372 100644 --- a/test/test_map_processor_training_data_export.py +++ b/test/test_map_processor_training_data_export.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions -from deepness.common.processing_parameters.map_processing_parameters import ModelOutputFormat, ProcessedAreaType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.training_data_export_parameters import TrainingDataExportParameters from deepness.processing.map_processor.map_processor_training_data_export import MapProcessorTrainingDataExport @@ -30,8 +30,6 @@ def test_export_dummy_fotomap(): input_layer_id=rlayer.id(), input_channels_mapping=create_default_input_channels_mapping_for_rgba_bands(), processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), - model_output_format=ModelOutputFormat.ALL_CLASSES_AS_SEPARATE_LAYERS, - model_output_format__single_class_number=-1, ) map_processor = MapProcessorTrainingDataExport( diff --git a/test/test_remove_overlaping_detections.py b/test/test_remove_overlaping_detections.py index 45c173d..e76fb65 100644 --- a/test/test_remove_overlaping_detections.py +++ b/test/test_remove_overlaping_detections.py @@ -8,8 +8,6 @@ def test__remove_overlaping_detections(): - init_qgis() - with open(get_predicted_detections_path(), 'rb') as f: dets = np.load(f) diff --git a/test/test_utils.py b/test/test_utils.py index 0548f0e..bfdaad7 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -36,9 +36,9 @@ def get_dummy_segmentation_models_dict(): '1x2x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'one_output_softmax_bsx2x512x512.onnx'), }, 'two_outputs': { - '1x1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_output_sigmoid_bsx1x512x512.onnx'), - '1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_output_sigmoid_bsx512x512.onnx'), - '1x2x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_output_softmax_bsx2x512x512.onnx'), + '1x1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_outputs_sigmoid_bsx1x512x512.onnx'), + '1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_outputs_sigmoid_bsx512x512.onnx'), + '1x2x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'two_outputs_softmax_bsx2x512x512.onnx'), } } @@ -66,14 +66,30 @@ def get_dummy_regression_model_path(): Get path of a dummy onnx model. See details in README in model directory. Model used for unit tests processing purposes """ - return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_model.onnx') + return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'dummy_regression_model.onnx') def get_dummy_regression_model_path_batched(): """ Get path of a dummy onnx model. See details in README in model directory. Model used for unit tests processing purposes """ - return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_model_batched.onnx') + return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'dummy_regression_model_batched.onnx') + +def get_dummy_regression_models_dict(): + """ + Get dictionary with dummy regression models paths. See details in README in model directory. + Models used for unit tests processing purposes + """ + return { + 'one_output': { + '1x1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'one_output_sigmoid_bsx1x512x512.onnx'), + '1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'one_output_sigmoid_bsx512x512.onnx'), + }, + 'two_outputs': { + '1x1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'two_outputs_sigmoid_bsx1x512x512.onnx'), + '1x512x512': os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'two_outputs_sigmoid_bsx512x512.onnx'), + } + } def get_dummy_superresolution_model_path(): """ From db3e5a474d71262719b17288978fbca9cb69c925 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 13:11:06 +0100 Subject: [PATCH 04/32] Fix pytest segmentation fault --- test/test_map_processor_detection_oils_example.py | 2 ++ test/test_utils.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/test/test_map_processor_detection_oils_example.py b/test/test_map_processor_detection_oils_example.py index d426807..670d1d3 100644 --- a/test/test_map_processor_detection_oils_example.py +++ b/test/test_map_processor_detection_oils_example.py @@ -53,6 +53,7 @@ def test_map_processor_detection_oil_example(): map_processor.run() + def test_map_processor_detection_oil_example_using_cache(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -85,6 +86,7 @@ def test_map_processor_detection_oil_example_using_cache(): map_processor.run() + def test_map_processor_detection_oil_example_with_remove_small(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) diff --git a/test/test_utils.py b/test/test_utils.py index 078ea62..ea65459 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -189,8 +189,16 @@ def get_first_arg(self): raise Exception("No argument were provided for the signal!") +_APP_INSTANCE = None + + def init_qgis(): - qgs = QgsApplication([b''], False) + global _APP_INSTANCE + if _APP_INSTANCE: + return _APP_INSTANCE + + qgs = QgsApplication([b''], GUIenabled=False) qgs.setPrefixPath('/usr/bin/qgis', True) qgs.initQgis() - return qgs + _APP_INSTANCE = qgs + return _APP_INSTANCE From 92ee5cc1265bcea01f675329dd46eec19ed854c4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:04:42 +0100 Subject: [PATCH 05/32] remove xvfb-run --- .github/workflows/python-app-ubuntu.yml | 12 ++++++------ test/test_utils.py | 13 ++++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-app-ubuntu.yml b/.github/workflows/python-app-ubuntu.yml index d47f2ec..aeb06d7 100644 --- a/.github/workflows/python-app-ubuntu.yml +++ b/.github/workflows/python-app-ubuntu.yml @@ -52,17 +52,17 @@ jobs: export PYTHONPATH="/usr/lib/python3/dist-packages":PYTHONPATH && echo "import qgis" | python3 export PYTHONPATH="$PYTHONPATH:$(pwd)/src" export PYTHONPATH="$PYTHONPATH:$(pwd)" - - # apparently there is some issue with opencv-python-headless, we need to reinstall it + + # apparently there is some issue with opencv-python-headless, we need to reinstall it pip uninstall opencv-python-headless --yes - + pip install --upgrade -r ./src/deepness/python_requirements/requirements.txt # run one this without pytest, because pytest creates obfuscated error messages # we need 'xvfb-run' to simulate UI - otherwise qgis crushes - xvfb-run python3 test/test_map_processor_segmentation.py - + python3 test/test_map_processor_segmentation.py + # run the actual tests - xvfb-run python3 -m pytest --cov=src/deepness/ --cov-report html test/ + python3 -m pytest --cov=src/deepness/ --cov-report html test/ - name: 'Upload Artifact - test coverage' uses: actions/upload-artifact@v3 with: diff --git a/test/test_utils.py b/test/test_utils.py index ea65459..603c366 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -17,6 +17,7 @@ def get_dummy_segmentation_model_path(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_model.onnx') + def get_dummy_segmentation_model_different_output_size_path(): """ Get path of a dummy onnx model. See details in README in model directory. @@ -24,6 +25,7 @@ def get_dummy_segmentation_model_different_output_size_path(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_model_different_output_size.onnx') + def get_dummy_recognition_model_path(): """ Get path of a dummy onnx model. See details in README in model directory. @@ -31,18 +33,21 @@ def get_dummy_recognition_model_path(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_recognition_model.onnx') + def get_dummy_recognition_image_path(): """ Get path of a dummy image, which can be used for testing with conjunction with dummy_mode (see get_dummy_model_path) """ return os.path.join(TEST_DATA_DIR, 'dummy_recognition_image.png') + def get_dummy_recognition_map_path(): """ Get path of a dummy map, which can be used for testing with conjunction with dummy_mode (see get_dummy_model_path) """ return os.path.join(TEST_DATA_DIR, 'dummy_recognition_map.tif') + def get_dummy_regression_model_path(): """ Get path of a dummy onnx model. See details in README in model directory. @@ -50,6 +55,7 @@ def get_dummy_regression_model_path(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_model.onnx') + def get_dummy_regression_model_path_batched(): """ Get path of a dummy onnx model. See details in README in model directory. @@ -57,6 +63,7 @@ def get_dummy_regression_model_path_batched(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_model_batched.onnx') + def get_dummy_superresolution_model_path(): """ Get path of a dummy onnx model. See details in README in model directory. @@ -64,6 +71,7 @@ def get_dummy_superresolution_model_path(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_superresolution_model.onnx') + def get_dummy_fotomap_small_path(): """ Get path of dummy fotomap tif file, which can be used @@ -189,14 +197,17 @@ def get_first_arg(self): raise Exception("No argument were provided for the signal!") -_APP_INSTANCE = None +_APP_INSTANCE: QgsApplication | None = None def init_qgis(): + print("Initializing QGIS") global _APP_INSTANCE if _APP_INSTANCE: + print("QGIS already initialized") return _APP_INSTANCE + print("QGIS not initialized yet") qgs = QgsApplication([b''], GUIenabled=False) qgs.setPrefixPath('/usr/bin/qgis', True) qgs.initQgis() From c863fbde277de97e350f41af10ba1ce61a6d4b8e Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:09:11 +0100 Subject: [PATCH 06/32] syntax fix --- test/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 603c366..e6e65e4 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,4 +1,5 @@ import os +from typing import Optional from qgis.core import (QgsApplication, QgsCoordinateReferenceSystem, QgsProject, QgsRasterLayer, QgsRectangle, QgsVectorLayer) @@ -197,7 +198,7 @@ def get_first_arg(self): raise Exception("No argument were provided for the signal!") -_APP_INSTANCE: QgsApplication | None = None +_APP_INSTANCE: Optional[QgsApplication] = None def init_qgis(): From ce5d21ffc2540a4ad8139ec7df8698002fbf174b Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:15:57 +0100 Subject: [PATCH 07/32] xvfb-run fixes --- .github/workflows/python-app-ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app-ubuntu.yml b/.github/workflows/python-app-ubuntu.yml index aeb06d7..1e3fec3 100644 --- a/.github/workflows/python-app-ubuntu.yml +++ b/.github/workflows/python-app-ubuntu.yml @@ -59,10 +59,10 @@ jobs: pip install --upgrade -r ./src/deepness/python_requirements/requirements.txt # run one this without pytest, because pytest creates obfuscated error messages # we need 'xvfb-run' to simulate UI - otherwise qgis crushes - python3 test/test_map_processor_segmentation.py + xvfb-run -e /dev/stdout python3 test/test_map_processor_segmentation.py # run the actual tests - python3 -m pytest --cov=src/deepness/ --cov-report html test/ + xvfb-run -e /dev/stdout python3 -m pytest --cov=src/deepness/ --cov-report html test/ - name: 'Upload Artifact - test coverage' uses: actions/upload-artifact@v3 with: From 7cde2f6c6c9d9790a81a4f393dcf4016bd563b5e Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:20:17 +0100 Subject: [PATCH 08/32] test fixes --- .github/workflows/python-app-ubuntu.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-app-ubuntu.yml b/.github/workflows/python-app-ubuntu.yml index 1e3fec3..5289602 100644 --- a/.github/workflows/python-app-ubuntu.yml +++ b/.github/workflows/python-app-ubuntu.yml @@ -60,9 +60,11 @@ jobs: # run one this without pytest, because pytest creates obfuscated error messages # we need 'xvfb-run' to simulate UI - otherwise qgis crushes xvfb-run -e /dev/stdout python3 test/test_map_processor_segmentation.py + rm /tmp/.X* # remove the Xvfb lock file, otherwise the next xvfb-run will fail # run the actual tests xvfb-run -e /dev/stdout python3 -m pytest --cov=src/deepness/ --cov-report html test/ + rm /tmp/.X* # remove the Xvfb lock file, otherwise the next xvfb-run will fail - name: 'Upload Artifact - test coverage' uses: actions/upload-artifact@v3 with: From 602796733dc789e3fd206d37c6f6553311c156c1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:23:06 +0100 Subject: [PATCH 09/32] tmp fix --- .github/workflows/python-app-ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app-ubuntu.yml b/.github/workflows/python-app-ubuntu.yml index 5289602..ee89ea3 100644 --- a/.github/workflows/python-app-ubuntu.yml +++ b/.github/workflows/python-app-ubuntu.yml @@ -60,11 +60,11 @@ jobs: # run one this without pytest, because pytest creates obfuscated error messages # we need 'xvfb-run' to simulate UI - otherwise qgis crushes xvfb-run -e /dev/stdout python3 test/test_map_processor_segmentation.py - rm /tmp/.X* # remove the Xvfb lock file, otherwise the next xvfb-run will fail + rm -rf /tmp/.X9* # remove the Xvfb lock file, otherwise the next xvfb-run will fail # run the actual tests xvfb-run -e /dev/stdout python3 -m pytest --cov=src/deepness/ --cov-report html test/ - rm /tmp/.X* # remove the Xvfb lock file, otherwise the next xvfb-run will fail + rm -rf /tmp/.X9* # remove the Xvfb lock file, otherwise the next xvfb-run will fail - name: 'Upload Artifact - test coverage' uses: actions/upload-artifact@v3 with: From 6e6b70290b6cd7e3118f83fa473e859bf21d4acf Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:26:40 +0100 Subject: [PATCH 10/32] test fix WIP --- .github/workflows/python-app-ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app-ubuntu.yml b/.github/workflows/python-app-ubuntu.yml index ee89ea3..9e5113e 100644 --- a/.github/workflows/python-app-ubuntu.yml +++ b/.github/workflows/python-app-ubuntu.yml @@ -59,8 +59,8 @@ jobs: pip install --upgrade -r ./src/deepness/python_requirements/requirements.txt # run one this without pytest, because pytest creates obfuscated error messages # we need 'xvfb-run' to simulate UI - otherwise qgis crushes - xvfb-run -e /dev/stdout python3 test/test_map_processor_segmentation.py - rm -rf /tmp/.X9* # remove the Xvfb lock file, otherwise the next xvfb-run will fail + # xvfb-run -e /dev/stdout python3 test/test_map_processor_segmentation.py + # rm -rf /tmp/.X9* # remove the Xvfb lock file, otherwise the next xvfb-run will fail # run the actual tests xvfb-run -e /dev/stdout python3 -m pytest --cov=src/deepness/ --cov-report html test/ From 22b8392f2d6f552d41345c5ce500f0f898aea3f5 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 23 Feb 2024 14:30:34 +0100 Subject: [PATCH 11/32] test fixes --- .github/workflows/python-app-ubuntu.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/python-app-ubuntu.yml b/.github/workflows/python-app-ubuntu.yml index 9e5113e..c22f8df 100644 --- a/.github/workflows/python-app-ubuntu.yml +++ b/.github/workflows/python-app-ubuntu.yml @@ -57,14 +57,9 @@ jobs: pip uninstall opencv-python-headless --yes pip install --upgrade -r ./src/deepness/python_requirements/requirements.txt - # run one this without pytest, because pytest creates obfuscated error messages - # we need 'xvfb-run' to simulate UI - otherwise qgis crushes - # xvfb-run -e /dev/stdout python3 test/test_map_processor_segmentation.py - # rm -rf /tmp/.X9* # remove the Xvfb lock file, otherwise the next xvfb-run will fail - # run the actual tests + # we need 'xvfb-run' to simulate UI - otherwise qgis crushes xvfb-run -e /dev/stdout python3 -m pytest --cov=src/deepness/ --cov-report html test/ - rm -rf /tmp/.X9* # remove the Xvfb lock file, otherwise the next xvfb-run will fail - name: 'Upload Artifact - test coverage' uses: actions/upload-artifact@v3 with: From 3779c5ac1035563f00e8ebf13a1be3bf344e6b50 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Fri, 23 Feb 2024 14:46:01 +0100 Subject: [PATCH 12/32] Make numpy compatible with ubuntu20 --- .../processing/map_processor/map_processor_segmentation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index d609b05..af61216 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -155,7 +155,8 @@ def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: if (result.shape[1] == 1): result = (result != 0).astype(int) else: - result = np.argmax(result, axis=1, keepdims=True) + shape = result.shape + result = np.argmax(result, axis=1).reshape(shape[0], 1, shape[2], shape[3]) assert len(result.shape) == 4 assert result.shape[1] == 1 From 9bf5aeefa89a4b9d69b1aaa02a53e79d6b341675 Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Thu, 29 Feb 2024 09:50:43 +0100 Subject: [PATCH 13/32] Why did the export test work before? --- .../map_processor/map_processor_training_data_export.py | 8 +++++--- test/test_map_processor_segmentation_landcover_example.py | 2 +- ...ap_processor_segmentation_landcover_example_batched.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/deepness/processing/map_processor/map_processor_training_data_export.py b/src/deepness/processing/map_processor/map_processor_training_data_export.py index e9d80fa..2968d21 100644 --- a/src/deepness/processing/map_processor/map_processor_training_data_export.py +++ b/src/deepness/processing/map_processor/map_processor_training_data_export.py @@ -47,7 +47,7 @@ def _run(self): vlayer_mask=vlayer_segmentation, extended_extent=self.extended_extent, rlayer_units_per_pixel=self.rlayer_units_per_pixel, - image_shape_yx=(self.img_size_y_pixels, self.img_size_x_pixels), + image_shape_yx=(1, self.img_size_y_pixels, self.img_size_x_pixels), files_handler=self.file_handler) number_of_written_tiles = 0 @@ -69,10 +69,12 @@ def _run(self): number_of_written_tiles += 1 if export_segmentation_mask: + segmentation_mask_for_tile = tile_params.get_entire_tile_from_full_img(segmentation_mask_full) + file_name = f'tile_mask_{tile_params.x_bin_number}_{tile_params.y_bin_number}.png' file_path = os.path.join(self.output_dir_path, file_name) - segmentation_mask_for_tile = tile_params.get_entire_tile_from_full_img(segmentation_mask_full) - cv2.imwrite(file_path, segmentation_mask_for_tile) + + cv2.imwrite(file_path, segmentation_mask_for_tile[0]) result_message = self._create_result_message(number_of_written_tiles) return MapProcessingResultSuccess(result_message) diff --git a/test/test_map_processor_segmentation_landcover_example.py b/test/test_map_processor_segmentation_landcover_example.py index 520a65f..3c92da3 100644 --- a/test/test_map_processor_segmentation_landcover_example.py +++ b/test/test_map_processor_segmentation_landcover_example.py @@ -71,7 +71,7 @@ def test_map_processor_segmentation_landcover_example(): assert set(counts.keys()) == set(gt_counts.keys()) for k, v in gt_counts.items(): - assert counts[k] == v + assert np.isclose(counts[k], v, atol=3) if __name__ == '__main__': diff --git a/test/test_map_processor_segmentation_landcover_example_batched.py b/test/test_map_processor_segmentation_landcover_example_batched.py index b1b3b17..34de5ab 100644 --- a/test/test_map_processor_segmentation_landcover_example_batched.py +++ b/test/test_map_processor_segmentation_landcover_example_batched.py @@ -72,7 +72,7 @@ def test_map_processor_segmentation_landcover_example(): assert set(counts.keys()) == set(gt_counts.keys()) for k, v in gt_counts.items(): - assert counts[k] == v + assert np.isclose(counts[k], v, atol=3) if __name__ == '__main__': From 5ea69a4a06f456666758cb4726d35e65a1eae004 Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Thu, 29 Feb 2024 12:24:09 +0100 Subject: [PATCH 14/32] Fix tests, changle new model names --- .../creators_add_metadata_to_model.rst | 58 +++---- .../map_processor_segmentation.py | 74 +++++---- src/deepness/processing/models/detector.py | 2 +- src/deepness/processing/models/model_base.py | 46 ++++-- .../data/dummy_model/AddMetadataToDummy.ipynb | 148 ++++++++++++++++++ .../Untitled.ipynb => GenerateDummy.ipynb} | 0 .../one_output_sigmoid_bsx1x512x512.onnx | Bin 415 -> 476 bytes .../one_output_sigmoid_bsx512x512.onnx | Bin 593 -> 652 bytes .../two_outputs_sigmoid_bsx1x512x512.onnx | Bin 523 -> 600 bytes .../two_outputs_softmax_bsx2x512x512.onnx | Bin 1013 -> 1115 bytes ...rocessor_segmentation_many_output_types.py | 42 +++++ tools/add_model_metadata.py | 8 +- 12 files changed, 305 insertions(+), 73 deletions(-) create mode 100644 test/data/dummy_model/AddMetadataToDummy.ipynb rename test/data/dummy_model/{dummy_regression_models/Untitled.ipynb => GenerateDummy.ipynb} (100%) diff --git a/docs/source/creators/creators_add_metadata_to_model.rst b/docs/source/creators/creators_add_metadata_to_model.rst index f08d929..8adf948 100644 --- a/docs/source/creators/creators_add_metadata_to_model.rst +++ b/docs/source/creators/creators_add_metadata_to_model.rst @@ -8,35 +8,37 @@ The plugin allows you to load the meta parameters of the onnx model automaticall List of parameters parsed by plugin =================================== -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| Parameter | Type | Example | Description | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| Parameter | Type | Example | Description | +======================+=======+=======================================+=============================================================+ -| model_type | str | :code:`'Segmentor'` | Types of models available: Segmentor, Regressor, Detector. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| class_names | dict | :code:`{0: 'background', 1: 'field'}` | A dictionary that maps a class id to its name. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| resolution | float | :code:`100` | Real-world resolution of images (centimeters per pixel). | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| tiles_size | int | :code:`512` | What size (in pixels) is the tile to crop. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| tiles_overlap | int | :code:`40` | How many percent of the image size overlap. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| standardization_mean | list | :code:`[0.0, 0.0, 0.0]` | Mean - if you want to standarize input after normalisation | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| standardization_std | list | :code:`[1.0, 1.0, 1.0]` | Std - if you want to standarize input after normalisation | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| seg_thresh | float | :code:`0.5` | Segmentor: class confidence threshold. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| seg_small_segment | int | :code:`7` | Segmentor: remove small occurrences of the class. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| reg_output_scaling | float | :code:`1.0` | Regressor: scaling factor for the model output. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| det_conf | float | :code:`0.6` | Detector: object confidence threshold. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| det_iou_thresh | float | :code:`0.4` | Detector: IOU threshold for NMS. | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ -| det_type | str | :code:`YOLO_v5_or_v7_default` | Detector: type of the detector model format | -+----------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| model_type | str | :code:`'Segmentor'` | Types of models available: Segmentor, Regressor, Detector. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| class_names | dict | :code:`{0: 'name1', 1: 'name2'}` | A dictionary that maps a class id to its name. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| class_names (mutli outputs) | list | :code:`[{0: 'name1'}, {0: 'name2'}]` | A dictionary that maps a class id to its name. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| resolution | float | :code:`100` | Real-world resolution of images (centimeters per pixel). | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| tiles_size | int | :code:`512` | What size (in pixels) is the tile to crop. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| tiles_overlap | int | :code:`40` | How many percent of the image size overlap. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| standardization_mean | list | :code:`[0.0, 0.0, 0.0]` | Mean - if you want to standarize input after normalisation | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| standardization_std | list | :code:`[1.0, 1.0, 1.0]` | Std - if you want to standarize input after normalisation | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| seg_thresh | float | :code:`0.5` | Segmentor: class confidence threshold. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| seg_small_segment | int | :code:`7` | Segmentor: remove small occurrences of the class. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| reg_output_scaling | float | :code:`1.0` | Regressor: scaling factor for the model output. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| det_conf | float | :code:`0.6` | Detector: object confidence threshold. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| det_iou_thresh | float | :code:`0.4` | Detector: IOU threshold for NMS. | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +| det_type | str | :code:`YOLO_v5_or_v7_default` | Detector: type of the detector model format | ++-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ ======= Example diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index af61216..a374bf6 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -64,27 +64,34 @@ def _run(self) -> MapProcessingResult: ) def _create_result_message(self, result_img: np.ndarray) -> str: - unique, counts = np.unique(result_img, return_counts=True) - counts_map = {} - for i in range(len(unique)): - counts_map[unique[i]] = counts[i] - - channels = self._get_indexes_of_model_output_channels_to_create() - txt = f'Segmentation done for {len(channels)} model output channels, with the following statistics:\n' - - # we cannot simply take image dimensions, because we may have irregular processing area from polygon - number_of_pixels_in_processing_area = np.sum([counts_map[k] for k in counts_map.keys() if k != 0]) - total_area = number_of_pixels_in_processing_area * self.params.resolution_m_per_px**2 - for channel_id in channels: - # See note in the class description why are we adding/subtracting 1 here - pixels_count = counts_map.get(channel_id + 1, 0) - area = pixels_count * self.params.resolution_m_per_px**2 - if total_area: - area_percentage = area / total_area * 100 - else: - area_percentage = 0.0 - # TODO - txt += f' - {self.model.get_channel_name(0, channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' + + txt = f'Segmentation done, with the following statistics:\n' + + for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): + + txt += f'Channels for output {output_id}:\n' + + unique, counts = np.unique(result_img[output_id], return_counts=True) + counts_map = {} + for i in range(len(unique)): + counts_map[unique[i]] = counts[i] + + # # we cannot simply take image dimensions, because we may have irregular processing area from polygon + number_of_pixels_in_processing_area = np.sum([counts_map[k] for k in counts_map.keys()]) + total_area = number_of_pixels_in_processing_area * self.params.resolution_m_per_px**2 + + for channel_id in range(layer_sizes): + # See note in the class description why are we adding/subtracting 1 here + pixels_count = counts_map.get(channel_id, 0) + area = pixels_count * self.params.resolution_m_per_px**2 + + if total_area > 0 and not np.isnan(total_area) and not np.isinf(total_area): + area_percentage = area / total_area * 100 + else: + area_percentage = 0.0 + # TODO + + txt += f'\t- {self.model.get_channel_name(output_id, channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' return txt @@ -94,10 +101,11 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: """ vlayers = [] - for layer_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): + for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): + output_vlayers = [] for channel_id in range(layer_sizes): # See note in the class description why are we adding/subtracting 1 here - local_mask_img = np.uint8(mask_img[layer_id] == channel_id) + local_mask_img = np.uint8(mask_img[output_id] == channel_id) contours, hierarchy = cv2.findContours(local_mask_img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours = processing_utils.transform_contours_yx_pixels_to_target_crs( @@ -117,7 +125,7 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: else: pass # just nothing, we already have an empty list of features - layer_name = self.model.get_channel_name(layer_id, channel_id) + layer_name = self.model.get_channel_name(output_id, channel_id) vlayer = QgsVectorLayer("multipolygon", layer_name, "memory") vlayer.setCrs(self.rlayer.crs()) prov = vlayer.dataProvider() @@ -131,14 +139,24 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: prov.addFeatures(features) vlayer.updateExtents() - vlayers.append(vlayer) + output_vlayers.append(vlayer) + + vlayers.append(output_vlayers) # accessing GUI from non-GUI thread is not safe, so we need to delegate it to the GUI thread def add_to_gui(): group = QgsProject.instance().layerTreeRoot().insertGroup(0, 'model_output') - for vlayer in vlayers: - QgsProject.instance().addMapLayer(vlayer, False) - group.addLayer(vlayer) + + if len(vlayers) == 1: + for vlayer in vlayers[0]: + QgsProject.instance().addMapLayer(vlayer, False) + group.addLayer(vlayer) + else: + for i, output_vlayers in enumerate(vlayers): + output_group = group.insertGroup(0, f'output_{i}') + for vlayer in output_vlayers: + QgsProject.instance().addMapLayer(vlayer, False) + output_group.addLayer(vlayer) return add_to_gui diff --git a/src/deepness/processing/models/detector.py b/src/deepness/processing/models/detector.py index 1e7d3c1..6750930 100644 --- a/src/deepness/processing/models/detector.py +++ b/src/deepness/processing/models/detector.py @@ -137,7 +137,7 @@ def get_number_of_output_channels(self): int Number of output channels """ - class_names = self.get_class_names() + class_names = self.get_outputs_channel_names()[0] if class_names is not None: return [len(class_names)] # If class names are specified, we expect to have exactly this number of channels as specidied diff --git a/src/deepness/processing/models/model_base.py b/src/deepness/processing/models/model_base.py index c31dfd6..147b3fc 100644 --- a/src/deepness/processing/models/model_base.py +++ b/src/deepness/processing/models/model_base.py @@ -45,6 +45,8 @@ def __init__(self, model_file_path: str): self.outputs_layers = self.sess.get_outputs() self.standardization_parameters: StandardizationParameters = self.get_metadata_standarization_parameters() + + self.output_names = self.get_outputs_channel_names() @classmethod def get_model_type_from_metadata(cls, model_file_path: str) -> Optional[str]: @@ -98,13 +100,13 @@ def get_input_size_in_pixels(self) -> int: """ return self.input_shape[-2:] - def get_class_names(self) -> Optional[List[str]]: + def get_outputs_channel_names(self) -> Optional[List[List[str]]]: """ Get class names from metadata Returns ------- - List[str] | None - List of class names or None if not found + List[List[str]] | None + List of class names for each model output or None if not found """ meta = self.sess.get_modelmeta() @@ -119,15 +121,23 @@ def get_class_names(self) -> Optional[List[str]]: except json.decoder.JSONDecodeError: class_names = ast.literal_eval(txt) # keys are integers instead of strings - use ast - sorted_by_key = sorted(class_names.items(), key=lambda kv: int(kv[0])) + if isinstance(class_names, dict): + class_names = [class_names] + + sorted_by_key = [sorted(cn.items(), key=lambda kv: int(kv[0])) for cn in class_names] - class_counter = 0 all_names = [] - for key, value in sorted_by_key: - if int(key) != class_counter: - raise Exception("Class names in the model metadata are not consecutive (missing class label)") - class_counter += 1 - all_names.append(value) + + for output_index in range(len(sorted_by_key)): + output_names = [] + class_counter = 0 + + for key, value in sorted_by_key[output_index]: + if int(key) != class_counter: + raise Exception("Class names in the model metadata are not consecutive (missing class label)") + class_counter += 1 + output_names.append(value) + all_names.append(output_names) return all_names @@ -146,14 +156,20 @@ def get_channel_name(self, layer_id: int, channel_id: int) -> str: str Channel name or empty string if not found """ - class_names = self.get_class_names() + channel_id_str = str(channel_id) - default_return = f'o_{layer_id}_{channel_id_str}' + default_return = f'channel_{channel_id_str}' - if class_names is not None and channel_id < len(class_names): - return class_names[channel_id] - else: + if self.output_names is None: return default_return + + if layer_id >= len(self.output_names): + raise Exception(f'Layer id {layer_id} is out of range of the model outputs') + + if channel_id >= len(self.output_names[layer_id]): + raise Exception(f'Channel id {channel_id} is out of range of the model outputs') + + return f'{self.output_names[layer_id][channel_id]}' def get_metadata_model_type(self) -> Optional[str]: """ Get model type from metadata diff --git a/test/data/dummy_model/AddMetadataToDummy.ipynb b/test/data/dummy_model/AddMetadataToDummy.ipynb new file mode 100644 index 0000000..5ca3bfb --- /dev/null +++ b/test/data/dummy_model/AddMetadataToDummy.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "ff1f7bbc-36d2-4b66-9df5-4c1e38a4b86b", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import json\n", + "import onnx" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "060d1bcf-b7df-4813-85cb-35fd5be9a69c", + "metadata": {}, + "outputs": [], + "source": [ + "model = onnx.load('dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx')\n", + "\n", + "class_names = [{\n", + " 0: 'Coffee',\n", + "}]\n", + "\n", + "m1 = model.metadata_props.add()\n", + "m1.key = 'model_type'\n", + "m1.value = json.dumps('Segmentor')\n", + "\n", + "m2 = model.metadata_props.add()\n", + "m2.key = 'class_names'\n", + "m2.value = json.dumps(class_names)\n", + "\n", + "onnx.save(model, 'dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4db78a0c-7d5f-4eaa-9e90-bd638c880f26", + "metadata": {}, + "outputs": [], + "source": [ + "model = onnx.load('dummy_segmentation_models/one_output_sigmoid_bsx512x512.onnx')\n", + "\n", + "class_names = {\n", + " 0: 'Coffee',\n", + "}\n", + "\n", + "m1 = model.metadata_props.add()\n", + "m1.key = 'model_type'\n", + "m1.value = json.dumps('Segmentor')\n", + "\n", + "m2 = model.metadata_props.add()\n", + "m2.key = 'class_names'\n", + "m2.value = json.dumps(class_names)\n", + "\n", + "onnx.save(model, 'dummy_segmentation_models/one_output_sigmoid_bsx512x512.onnx')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "451eec0c-06bb-40ca-b3d2-a76b1e9619c6", + "metadata": {}, + "outputs": [], + "source": [ + "model = onnx.load('dummy_segmentation_models/two_outputs_sigmoid_bsx1x512x512.onnx')\n", + "\n", + "class_names = [{\n", + " 0: 'Coffee',\n", + "},{\n", + " 0: 'Juice',\n", + "}]\n", + "\n", + "m1 = model.metadata_props.add()\n", + "m1.key = 'model_type'\n", + "m1.value = json.dumps('Segmentor')\n", + "\n", + "m2 = model.metadata_props.add()\n", + "m2.key = 'class_names'\n", + "m2.value = json.dumps(class_names)\n", + "\n", + "onnx.save(model, 'dummy_segmentation_models/two_outputs_sigmoid_bsx1x512x512.onnx')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0e933e81-e9f4-4e2c-a312-33a9e3233baa", + "metadata": {}, + "outputs": [], + "source": [ + "model = onnx.load('dummy_segmentation_models/two_outputs_softmax_bsx2x512x512.onnx')\n", + "\n", + "class_names = [{\n", + " 0: 'Coffee',\n", + " 1: 'Tea',\n", + "},{\n", + " 0: 'Juice',\n", + " 1: 'Beer',\n", + "}]\n", + "\n", + "m1 = model.metadata_props.add()\n", + "m1.key = 'model_type'\n", + "m1.value = json.dumps('Segmentor')\n", + "\n", + "m2 = model.metadata_props.add()\n", + "m2.key = 'class_names'\n", + "m2.value = json.dumps(class_names)\n", + "\n", + "onnx.save(model, 'dummy_segmentation_models/two_outputs_softmax_bsx2x512x512.onnx')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89dbca0c-fac1-421c-898a-c2f62c2905ee", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/data/dummy_model/dummy_regression_models/Untitled.ipynb b/test/data/dummy_model/GenerateDummy.ipynb similarity index 100% rename from test/data/dummy_model/dummy_regression_models/Untitled.ipynb rename to test/data/dummy_model/GenerateDummy.ipynb diff --git a/test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx1x512x512.onnx index d1b01fb6f72872d751702c55691d3fc6f67e7bff..0e183528d7fa2ecc629958934649b612a3219e20 100644 GIT binary patch delta 69 zcmbQwe200%d`8#(kR3UDq;MDZo)Vz}XBBde)F7D)<#Ny)kyu{qp YVj;okY9#|DD+MLz{Is;xRHfQj03uBmQUCw| delta 7 Ocmcb^JfC^Pd`18alLFEJ diff --git a/test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx512x512.onnx b/test/data/dummy_model/dummy_segmentation_models/one_output_sigmoid_bsx512x512.onnx index 7e2bb038a66b0efd7752dfca1022da534a7dbc22..7ef4bcf4e54423e6121c6354d2cd1c45172aa784 100644 GIT binary patch delta 67 zcmcb}(!;tTkjc78l8Y-hKP5FMzNE4sRft#(kR3UDq;MDZo)Vz}XBBdfvF7D)<#Ny)kyu{qp yVj-*OY9#|DD+MLz{Is;xR3#k+B|{K5BsEc~R!0FU=~bGUjF59mO)XNYjRgSNQz0J! delta 7 Ocmcc3@s)kUS7rbY76XU? diff --git a/test/test_map_processor_segmentation_many_output_types.py b/test/test_map_processor_segmentation_many_output_types.py index 415ee23..4abf024 100644 --- a/test/test_map_processor_segmentation_many_output_types.py +++ b/test/test_map_processor_segmentation_many_output_types.py @@ -61,6 +61,13 @@ def test_dummy_model_segmentation_processing__1x1x512x512(): result_img = map_processor.get_result_img() assert result_img.shape == (1, 561, 829) + + channels = map_processor._get_indexes_of_model_output_channels_to_create() + assert len(channels) == 1 + assert channels[0] == 1 + + name = map_processor.model.get_channel_name(0, 0) + assert name == 'Coffee' def test_dummy_model_segmentation_processing__1x512x512(): qgs = init_qgis() @@ -94,6 +101,13 @@ def test_dummy_model_segmentation_processing__1x512x512(): result_img = map_processor.get_result_img() assert result_img.shape == (1, 561, 829) + + channels = map_processor._get_indexes_of_model_output_channels_to_create() + assert len(channels) == 1 + assert channels[0] == 1 + + name = map_processor.model.get_channel_name(0, 0) + assert name == 'Coffee' def test_dummy_model_segmentation_processing__1x2x512x512(): qgs = init_qgis() @@ -162,6 +176,17 @@ def test_dummy_model_segmentation_processing__two_outputs_1x1x512x512(): result_img = map_processor.get_result_img() assert result_img.shape == (2, 561, 829) + + channels = map_processor._get_indexes_of_model_output_channels_to_create() + assert len(channels) == 2 + assert channels[0] == 1 + assert channels[1] == 1 + + name = map_processor.model.get_channel_name(0, 0) + assert name == 'Coffee' + + name = map_processor.model.get_channel_name(1, 0) + assert name == 'Juice' def test_dummy_model_segmentation_processing__two_outputs_1x512x512(): qgs = init_qgis() @@ -230,6 +255,23 @@ def test_dummy_model_segmentation_processing__two_outputs_1x2x512x512(): result_img = map_processor.get_result_img() assert result_img.shape == (2, 561, 829) + + channels = map_processor._get_indexes_of_model_output_channels_to_create() + assert len(channels) == 2 + assert channels[0] == 2 + assert channels[1] == 2 + + name = map_processor.model.get_channel_name(0, 0) + assert name == 'Coffee' + + name = map_processor.model.get_channel_name(0, 1) + assert name == 'Tea' + + name = map_processor.model.get_channel_name(1, 0) + assert name == 'Juice' + + name = map_processor.model.get_channel_name(1, 1) + assert name == 'Beer' if __name__ == '__main__': test_dummy_model_segmentation_processing__1x1x512x512() diff --git a/tools/add_model_metadata.py b/tools/add_model_metadata.py index 4756697..edbebee 100644 --- a/tools/add_model_metadata.py +++ b/tools/add_model_metadata.py @@ -4,8 +4,8 @@ """ import json -import onnx +import onnx model = onnx.load('/path/to/model') @@ -14,6 +14,12 @@ 1: 'road', } +# or as a list of dictionaries if your model has multiple outputs +# class_names = [{ +# 0: 'not_road', +# 1: 'road', +# },] + m = model.metadata_props.add() m.key = 'model_type' m.value = json.dumps('Segmentor') From 33114ea0473a9877b986fb0cc2874236281010f2 Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Thu, 29 Feb 2024 13:05:41 +0100 Subject: [PATCH 15/32] Update docs, add names to dummy regression models --- docs/source/creators/creators_tutorial.rst | 2 +- .../example/example_detection_cars_yolov7.rst | 2 +- .../map_processor/map_processor_regression.py | 22 +++++++----------- .../one_output_sigmoid_bsx1x512x512.onnx | Bin 415 -> 479 bytes .../two_outputs_sigmoid_bsx1x512x512.onnx | Bin 523 -> 600 bytes 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/source/creators/creators_tutorial.rst b/docs/source/creators/creators_tutorial.rst index b3767b5..6209ad7 100644 --- a/docs/source/creators/creators_tutorial.rst +++ b/docs/source/creators/creators_tutorial.rst @@ -6,7 +6,7 @@ Model creation tutorial (Python) Detection ========= -For one of models in our zoo - specifically for cars detection on aerial images - a complete tutorial is provided in a jupyter notebook: +For one of models in our zoo - specifically for cars detection on aerial images - a complete tutorial is `provided in a jupyter notebook '_: .. code-block:: diff --git a/docs/source/example/example_detection_cars_yolov7.rst b/docs/source/example/example_detection_cars_yolov7.rst index 5b81e83..d22651a 100644 --- a/docs/source/example/example_detection_cars_yolov7.rst +++ b/docs/source/example/example_detection_cars_yolov7.rst @@ -13,7 +13,7 @@ The example is based on the `ITCVD cars detection dataset '_: .. code-block:: diff --git a/src/deepness/processing/map_processor/map_processor_regression.py b/src/deepness/processing/map_processor/map_processor_regression.py index 406fe8b..2aa9842 100644 --- a/src/deepness/processing/map_processor/map_processor_regression.py +++ b/src/deepness/processing/map_processor/map_processor_regression.py @@ -60,18 +60,16 @@ def _run(self) -> MapProcessingResult: ) def _create_result_message(self, result_imgs: List[np.ndarray]) -> str: - channels = self._get_indexes_of_model_output_channels_to_create() - txt = f'Regression done for {len(channels)} model output channels, with the following statistics:\n' - for i, channel_id in enumerate(channels): - result_img = result_imgs[i] + txt = f'Regression done, with the following statistics:\n' + for output_id, _ in enumerate(self._get_indexes_of_model_output_channels_to_create()): + result_img = result_imgs[output_id] + average_value = np.mean(result_img) std = np.std(result_img) - txt += f' - {self.model.get_channel_name(0, channel_id)}: average_value = {average_value:.2f} (std = {std:.2f}, ' \ + + txt += f' - {self.model.get_channel_name(output_id, 0)}: average_value = {average_value:.2f} (std = {std:.2f}, ' \ f'min={np.min(result_img)}, max={np.max(result_img)})\n' - if len(channels) > 0: - total_area = result_img.shape[0] * result_img.shape[1] * self.params.resolution_m_per_px**2 - txt += f'Total are is {total_area:.2f} m^2' return txt def limit_extended_extent_images_to_base_extent_with_mask(self, full_imgs: List[np.ndarray]): @@ -101,11 +99,9 @@ def _create_rlayers_from_images_for_base_extent(self, result_imgs: List[np.ndarr # Or maybe even create vlayer directly from array, without a file? rlayers = [] - for i, channel_id in enumerate(self._get_indexes_of_model_output_channels_to_create()): - result_img = result_imgs[i] - random_id = str(uuid.uuid4()).replace('-', '') - file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(0, channel_id)}___{random_id}.tif') - self.save_result_img_as_tif(file_path=file_path, img=result_img) + for output_id, _ in enumerate(self._get_indexes_of_model_output_channels_to_create()): + file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(output_id, 0)}.tif') + self.save_result_img_as_tif(file_path=file_path, img=result_imgs[output_id]) rlayer = self.load_rlayer_from_file(file_path) OUTPUT_RLAYER_OPACITY = 0.5 diff --git a/test/data/dummy_model/dummy_regression_models/one_output_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_regression_models/one_output_sigmoid_bsx1x512x512.onnx index 346dd854bfa97ece0fc87f08c5616aab0be94892..467cd1dd2147f8b38e700a85db1fba41ac159684 100644 GIT binary patch delta 72 zcmbQwe4ly4d`5>NNiMG3{FKz3_>#(kR3UDq;MDZo)Vz}XBBdf_F7D)<#Ny)kyu{qp bVj+>}Y9#|DD+MKw#Dap%ywu`irP^2kYquA{ delta 7 Ocmcc5JfC^Pd`18ar~=gh diff --git a/test/data/dummy_model/dummy_regression_models/two_outputs_sigmoid_bsx1x512x512.onnx b/test/data/dummy_model/dummy_regression_models/two_outputs_sigmoid_bsx1x512x512.onnx index eff35f478b428ae98c93b269dedf79dae744e8c4..1a072ab0b1c7e73d626b900f83bbae8f6d6d3cea 100644 GIT binary patch delta 85 zcmeBXxxuo5o5{CGl8Y-hKP5FMzNE4sRfts*MEzcib92 delta 7 Ocmcb?(#^7gn+X64_5vvY From a99744f726a0aef863e88d45f23147d6bcb540c9 Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Thu, 29 Feb 2024 13:12:51 +0100 Subject: [PATCH 16/32] Update regression tests --- ..._processor_regression_many_output_types.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/test/test_map_processor_regression_many_output_types.py b/test/test_map_processor_regression_many_output_types.py index 4abcf44..33b3337 100644 --- a/test/test_map_processor_regression_many_output_types.py +++ b/test/test_map_processor_regression_many_output_types.py @@ -90,8 +90,15 @@ def test_dummy_model_regression_processing__1x1x512x512(): result_imgs = map_processor.get_result_img() assert result_imgs.shape == (1, 561, 829) - -def test_dummy_model_regression_processing__1x512x512(): + + channels = map_processor._get_indexes_of_model_output_channels_to_create() + assert len(channels) == 1 + assert channels[0] == 1 + + name = map_processor.model.get_channel_name(0, 0) + assert name == 'Happiness' + +def test_dummy_model_regression_processing__two_1x512x512(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -123,7 +130,7 @@ def test_dummy_model_regression_processing__1x512x512(): assert result_imgs.shape == (2, 561, 829) -def test_dummy_model_regression_processing__1x1x512x512(): +def test_dummy_model_regression_processing__two_1x1x512x512(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -154,10 +161,22 @@ def test_dummy_model_regression_processing__1x1x512x512(): result_imgs = map_processor.get_result_img() assert result_imgs.shape == (2, 561, 829) + + + channels = map_processor._get_indexes_of_model_output_channels_to_create() + assert len(channels) == 2 + assert channels[0] == 1 + assert channels[1] == 1 + + name = map_processor.model.get_channel_name(0, 0) + assert name == 'Luck' + + name = map_processor.model.get_channel_name(1, 0) + assert name == 'Failure' if __name__ == '__init__': test_dummy_model_regression_processing__1x512x512() test_dummy_model_regression_processing__1x1x512x512() - test_dummy_model_regression_processing__1x512x512() - test_dummy_model_regression_processing__1x1x512x512() + test_dummy_model_regression_processing__two_1x512x512() + test_dummy_model_regression_processing__two_1x1x512x512() print('All tests passed!') From b6e96636583ed4ad138bd2e54de0aebd9f41cbf6 Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Thu, 29 Feb 2024 14:10:27 +0100 Subject: [PATCH 17/32] Handle sigmoid segmentation models that have background in names --- .../map_processor_segmentation.py | 33 +++++++++++++++---- src/deepness/processing/models/model_base.py | 10 +++--- src/deepness/processing/models/segmentor.py | 24 ++++++++++++++ ...rocessor_segmentation_landcover_example.py | 14 ++++---- ..._segmentation_landcover_example_batched.py | 14 ++++---- 5 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index a374bf6..35ea624 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -1,5 +1,6 @@ """ This file implements map processing for segmentation model """ +from calendar import c from typing import Callable import numpy as np @@ -62,6 +63,12 @@ def _run(self) -> MapProcessingResult: message=result_message, gui_delegate=gui_delegate, ) + + def _check_output_layer_is_sigmoid_and_has_more_than_one_name(self, output_id: int) -> bool: + if self.model.outputs_names is None or self.model.outputs_are_sigmoid is None: + return False + + return len(self.model.outputs_names[output_id]) > 1 and self.model.outputs_are_sigmoid[output_id] def _create_result_message(self, result_img: np.ndarray) -> str: @@ -80,7 +87,7 @@ def _create_result_message(self, result_img: np.ndarray) -> str: number_of_pixels_in_processing_area = np.sum([counts_map[k] for k in counts_map.keys()]) total_area = number_of_pixels_in_processing_area * self.params.resolution_m_per_px**2 - for channel_id in range(layer_sizes): + for channel_id in range(1, layer_sizes + 1): # See note in the class description why are we adding/subtracting 1 here pixels_count = counts_map.get(channel_id, 0) area = pixels_count * self.params.resolution_m_per_px**2 @@ -90,8 +97,14 @@ def _create_result_message(self, result_img: np.ndarray) -> str: else: area_percentage = 0.0 # TODO - - txt += f'\t- {self.model.get_channel_name(output_id, channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' + + # hardcode if someone add "background class" for sigmoid output, we need to skip it + if self._check_output_layer_is_sigmoid_and_has_more_than_one_name(output_id): + channel_id_name = channel_id + else: + channel_id_name = channel_id - 1 + + txt += f'\t- {self.model.get_channel_name(output_id, channel_id_name)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' return txt @@ -103,7 +116,7 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): output_vlayers = [] - for channel_id in range(layer_sizes): + for channel_id in range(1, layer_sizes + 1): # See note in the class description why are we adding/subtracting 1 here local_mask_img = np.uint8(mask_img[output_id] == channel_id) @@ -124,8 +137,14 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: current_contour_index=0) else: pass # just nothing, we already have an empty list of features - - layer_name = self.model.get_channel_name(output_id, channel_id) + + # hardcode if someone add "background class" for sigmoid output, we need to skip it + if self._check_output_layer_is_sigmoid_and_has_more_than_one_name(output_id): + channel_id_name = channel_id + else: + channel_id_name = channel_id - 1 + + layer_name = self.model.get_channel_name(output_id, channel_id_name) vlayer = QgsVectorLayer("multipolygon", layer_name, "memory") vlayer.setCrs(self.rlayer.crs()) prov = vlayer.dataProvider() @@ -174,7 +193,7 @@ def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: result = (result != 0).astype(int) else: shape = result.shape - result = np.argmax(result, axis=1).reshape(shape[0], 1, shape[2], shape[3]) + result = np.argmax(result, axis=1).reshape(shape[0], 1, shape[2], shape[3]) + 1 assert len(result.shape) == 4 assert result.shape[1] == 1 diff --git a/src/deepness/processing/models/model_base.py b/src/deepness/processing/models/model_base.py index 147b3fc..2b60e40 100644 --- a/src/deepness/processing/models/model_base.py +++ b/src/deepness/processing/models/model_base.py @@ -46,7 +46,7 @@ def __init__(self, model_file_path: str): self.outputs_layers = self.sess.get_outputs() self.standardization_parameters: StandardizationParameters = self.get_metadata_standarization_parameters() - self.output_names = self.get_outputs_channel_names() + self.outputs_names = self.get_outputs_channel_names() @classmethod def get_model_type_from_metadata(cls, model_file_path: str) -> Optional[str]: @@ -160,16 +160,16 @@ def get_channel_name(self, layer_id: int, channel_id: int) -> str: channel_id_str = str(channel_id) default_return = f'channel_{channel_id_str}' - if self.output_names is None: + if self.outputs_names is None: return default_return - if layer_id >= len(self.output_names): + if layer_id >= len(self.outputs_names): raise Exception(f'Layer id {layer_id} is out of range of the model outputs') - if channel_id >= len(self.output_names[layer_id]): + if channel_id >= len(self.outputs_names[layer_id]): raise Exception(f'Channel id {channel_id} is out of range of the model outputs') - return f'{self.output_names[layer_id][channel_id]}' + return f'{self.outputs_names[layer_id][channel_id]}' def get_metadata_model_type(self) -> Optional[str]: """ Get model type from metadata diff --git a/src/deepness/processing/models/segmentor.py b/src/deepness/processing/models/segmentor.py index b2417a2..487c811 100644 --- a/src/deepness/processing/models/segmentor.py +++ b/src/deepness/processing/models/segmentor.py @@ -22,6 +22,8 @@ def __init__(self, model_file_path: str): Path to the model file """ super(Segmentor, self).__init__(model_file_path) + + self.outputs_are_sigmoid = self.check_loaded_model_outputs() def postprocessing(self, model_output: List) -> np.ndarray: """ Postprocess the model output. @@ -85,3 +87,25 @@ def check_loaded_model_outputs(self): for layer in self.outputs_layers: if len(layer.shape) != 4 and len(layer.shape) != 3: raise Exception(f'Segmentation model output should have 4 dimensions: (B,C,H,W) or 3 dimensions: (B,H,W). Has {layer.shape}') + + + def check_loaded_model_outputs(self) -> List[bool]: + """ Check if the model outputs are sigmoid (for segmentation) + + Parameters + ---------- + + Returns + ------- + List[bool] + List of booleans indicating if the model outputs are sigmoid + """ + outputs = [] + + for output in self.outputs_layers: + if len(output.shape) == 3: + outputs.append(True) + else: + outputs.append(output.shape[-3] == 1) + + return outputs diff --git a/test/test_map_processor_segmentation_landcover_example.py b/test/test_map_processor_segmentation_landcover_example.py index 3c92da3..be4ebb8 100644 --- a/test/test_map_processor_segmentation_landcover_example.py +++ b/test/test_map_processor_segmentation_landcover_example.py @@ -52,8 +52,8 @@ def test_map_processor_segmentation_landcover_example(): assert result_img.shape == (1, 2351, 2068) - assert result_img[0, 1000, 1000] == 0 - assert result_img[0, 2000, 2000] == 2 + assert result_img[0, 1000, 1000] == 1 + assert result_img[0, 2000, 2000] == 3 assert np.isclose(result_img[0, 150:300, 150:300].sum(), 18978, rtol=3) unique, counts = np.unique(result_img[0], return_counts=True) @@ -61,11 +61,11 @@ def test_map_processor_segmentation_landcover_example(): counts = dict(zip(unique, counts)) gt_counts = { - 0: 3294546, - 1: 71169, - 2: 1054899, - 3: 365915, - 4: 75339, + 1: 3294546, + 2: 71169, + 3: 1054899, + 4: 365915, + 5: 75339, } assert set(counts.keys()) == set(gt_counts.keys()) diff --git a/test/test_map_processor_segmentation_landcover_example_batched.py b/test/test_map_processor_segmentation_landcover_example_batched.py index 34de5ab..6cc0a3e 100644 --- a/test/test_map_processor_segmentation_landcover_example_batched.py +++ b/test/test_map_processor_segmentation_landcover_example_batched.py @@ -53,8 +53,8 @@ def test_map_processor_segmentation_landcover_example(): assert result_img.shape == (1, 2351, 2068) - assert result_img[0, 1000, 1000] == 0 - assert result_img[0, 2000, 2000] == 2 + assert result_img[0, 1000, 1000] == 1 + assert result_img[0, 2000, 2000] == 3 assert np.isclose(result_img[0, 150:300, 150:300].sum(), 18978, rtol=3) unique, counts = np.unique(result_img[0], return_counts=True) @@ -62,11 +62,11 @@ def test_map_processor_segmentation_landcover_example(): counts = dict(zip(unique, counts)) gt_counts = { - 0: 3294546, - 1: 71169, - 2: 1054899, - 3: 365915, - 4: 75339, + 1: 3294546, + 2: 71169, + 3: 1054899, + 4: 365915, + 5: 75339, } assert set(counts.keys()) == set(gt_counts.keys()) From c761756fcaa84dc11738c84aef2ecec7ab96ef53 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Fri, 1 Mar 2024 10:03:30 +0100 Subject: [PATCH 18/32] Add model outputs to UX, fix yolo SEGM --- src/deepness/deepness_dockwidget.py | 9 ++++++++- src/deepness/processing/models/detector.py | 2 +- src/deepness/processing/models/model_base.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/deepness/deepness_dockwidget.py b/src/deepness/deepness_dockwidget.py index 76a999a..24d1409 100644 --- a/src/deepness/deepness_dockwidget.py +++ b/src/deepness/deepness_dockwidget.py @@ -365,10 +365,17 @@ def _load_model_and_display_info(self, abort_if_no_file_path: bool = False): file_path=file_path) self._model.check_loaded_model_outputs() input_0_shape = self._model.get_input_shape() - txt += f'Input shape: {input_0_shape} = [BATCH_SIZE * CHANNELS * SIZE * SIZE]' + txt += 'Legend: [BATCH_SIZE, CHANNELS, HEIGHT, WIDTH]\n' + txt += 'Inputs:\n' + txt += f'\t- Input: {input_0_shape}\n' input_size_px = input_0_shape[-1] batch_size = self._model.get_model_batch_size() + txt += 'Outputs:\n' + + for i, output_shape in enumerate(self._model.get_output_shapes()): + txt += f'\t- Output {i}: {output_shape}\n' + # TODO idk how variable input will be handled self.spinBox_tileSize_px.setValue(input_size_px) self.spinBox_tileSize_px.setEnabled(False) diff --git a/src/deepness/processing/models/detector.py b/src/deepness/processing/models/detector.py index 6750930..0d2974c 100644 --- a/src/deepness/processing/models/detector.py +++ b/src/deepness/processing/models/detector.py @@ -296,7 +296,7 @@ def _postprocessing_YOLO_ULTRALYTICS(self, model_output): def _postprocessing_YOLO_ULTRALYTICS_SEGMENTATION(self, detections, protos): detections = np.transpose(detections, (1, 0)) - number_of_class = self.get_number_of_output_channels() + number_of_class = self.get_number_of_output_channels()[0] mask_start_index = 4 + number_of_class outputs_filtered = np.array( diff --git a/src/deepness/processing/models/model_base.py b/src/deepness/processing/models/model_base.py index 2b60e40..11dc214 100644 --- a/src/deepness/processing/models/model_base.py +++ b/src/deepness/processing/models/model_base.py @@ -75,6 +75,16 @@ def get_input_shape(self) -> tuple: """ return self.input_shape + def get_output_shapes(self) -> List[tuple]: + """ Get shapes of the outputs for the model + + Returns + ------- + List[tuple] + Shapes of the outputs (batch_size, channels, height, width) + """ + return [output.shape for output in self.outputs_layers] + def get_model_batch_size(self) -> Optional[int]: """ Get batch size of the model From eaba849c781f6412a953e1caf412639bf147abd4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 1 Mar 2024 13:45:34 +0100 Subject: [PATCH 19/32] Documentation update and code fixes --- .../creators/creators_description_classes.rst | 16 +++++++++++----- .../map_processor/map_processor_regression.py | 8 +++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/source/creators/creators_description_classes.rst b/docs/source/creators/creators_description_classes.rst index 95c2ab8..6555594 100644 --- a/docs/source/creators/creators_description_classes.rst +++ b/docs/source/creators/creators_description_classes.rst @@ -28,14 +28,20 @@ Segmentation models allow to solve problem of Image segmentation, that is assign Example application is segmenting earth surface areas into the following categories: forest, road, buildings, water, other. The segmentation model output is also an image, with same dimension as the input tile, but instead of 'CHANNELS' dimension, each output class has a separate image. -Therefore, the shape of model output is :code:`[BATCH_SIZE, NUM_CLASSES, SIZE_PX, SIZE_PX)`. +Therefore, the shape of model output is :code:`[BATCH_SIZE, NUM_CLASSES, SIZE_PX, SIZE_PX]`. -For each output class, a separate vector layer can be created. +We support the following types of models: + * single output (one head) with the following output shapes: + * :code:`[BATCH_SIZE, 1, SIZE_PX, SIZE_PX]` - one class with sigmoid activation function (binary classification) + * :code:`[BATCH_SIZE, NUM_CLASSES, SIZE_PX, SIZE_PX]` - multiple classes with softmax activation function (multi-class classification) - outputs sum to 1.0 + * multiple outputs (multiple heads) with each output head composed of the same shapes as single output. -Output report contains information about percentage coverage of each class. + Metaparameter :code:`class_names` saved in the model file should be as follows in this example: + * for single output with binary classification (sigmoid): :code:`[{0: "class_name"}]` or :code:`{0: "class_name"}` + * for single output with multi-class classification (softmax): :code:`[{0: "class0", 1: "class1", 2: "class2"}]` or :code:`{0: "class0", 1: "class1", 2: "class2"}` + * for multiple outputs (multiple heads): :code:`[{0: "class0", 1: "class1", 2: "class2"}, {0: "class0"}]` -The model should have at least two output classes, one for the background and one (or more) for the object of interest. The background class should be the first class in the output. -Model outputs should sum to 1.0 for each pixel, so the output is a probability map. To achieve this, the output should be passed through a softmax function. +Output report contains information about percentage coverage of each class. =============== diff --git a/src/deepness/processing/map_processor/map_processor_regression.py b/src/deepness/processing/map_processor/map_processor_regression.py index 2aa9842..15c3c8d 100644 --- a/src/deepness/processing/map_processor/map_processor_regression.py +++ b/src/deepness/processing/map_processor/map_processor_regression.py @@ -63,10 +63,10 @@ def _create_result_message(self, result_imgs: List[np.ndarray]) -> str: txt = f'Regression done, with the following statistics:\n' for output_id, _ in enumerate(self._get_indexes_of_model_output_channels_to_create()): result_img = result_imgs[output_id] - + average_value = np.mean(result_img) std = np.std(result_img) - + txt += f' - {self.model.get_channel_name(output_id, 0)}: average_value = {average_value:.2f} (std = {std:.2f}, ' \ f'min={np.min(result_img)}, max={np.max(result_img)})\n' @@ -100,7 +100,9 @@ def _create_rlayers_from_images_for_base_extent(self, result_imgs: List[np.ndarr rlayers = [] for output_id, _ in enumerate(self._get_indexes_of_model_output_channels_to_create()): - file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(output_id, 0)}.tif') + + random_id = str(uuid.uuid4()).replace('-', '') + file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(output_id, 0)}__{random_id}.tif') self.save_result_img_as_tif(file_path=file_path, img=result_imgs[output_id]) rlayer = self.load_rlayer_from_file(file_path) From 3b81ce9b961623cd5ad248bbb4c78d375865994f Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Fri, 1 Mar 2024 18:59:23 +0100 Subject: [PATCH 20/32] Update map_processor_segmentation.py --- .../processing/map_processor/map_processor_segmentation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index 35ea624..4c2e022 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -1,6 +1,5 @@ """ This file implements map processing for segmentation model """ -from calendar import c from typing import Callable import numpy as np From 634752253e4b1e940523c204c4e6ab3c489acb27 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Wed, 20 Mar 2024 22:18:15 +0100 Subject: [PATCH 21/32] Add YOLOv9 model handle --- .../detection_parameters.py | 6 ++ src/deepness/processing/models/detector.py | 36 ++++++++++- ...ual_test_map_processor_detection_yolov9.py | 61 +++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 test/manual_test_map_processor_detection_yolov9.py diff --git a/src/deepness/common/processing_parameters/detection_parameters.py b/src/deepness/common/processing_parameters/detection_parameters.py index f4dd56b..1f48830 100644 --- a/src/deepness/common/processing_parameters/detection_parameters.py +++ b/src/deepness/common/processing_parameters/detection_parameters.py @@ -20,6 +20,7 @@ class DetectorType(enum.Enum): YOLO_v5_v7_DEFAULT = 'YOLO_v5_or_v7_default' YOLO_v6 = 'YOLO_v6' + YOLO_v9 = 'YOLO_v9' YOLO_ULTRALYTICS = 'YOLO_Ultralytics' YOLO_ULTRALYTICS_SEGMENTATION = 'YOLO_Ultralytics_segmentation' @@ -30,6 +31,11 @@ def get_parameters(self): return DetectorTypeParameters( ignore_objectness_probability=True, ) + elif self == DetectorType.YOLO_v9: + return DetectorTypeParameters( + has_inverted_output_shape=True, + skipped_objectness_probability=True, + ) elif self == DetectorType.YOLO_ULTRALYTICS or self == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION: return DetectorTypeParameters( has_inverted_output_shape=True, diff --git a/src/deepness/processing/models/detector.py b/src/deepness/processing/models/detector.py index 0d2974c..84030d0 100644 --- a/src/deepness/processing/models/detector.py +++ b/src/deepness/processing/models/detector.py @@ -180,7 +180,12 @@ def postprocessing(self, model_output): ) batch_detection = [] - outputs_range = len(model_output[0])if self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION else len(model_output) + outputs_range = len(model_output) + + if self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION: + outputs_range = len(model_output[0]) + elif self.model_type == DetectorType.YOLO_v9: + outputs_range = len(model_output[0]) for i in range(outputs_range): masks = None @@ -190,6 +195,8 @@ def postprocessing(self, model_output): boxes, conf, classes = self._postprocessing_YOLO_v5_v7_DEFAULT(model_output[0][i]) elif self.model_type == DetectorType.YOLO_v6: boxes, conf, classes = self._postprocessing_YOLO_v6(model_output[0][i]) + elif self.model_type == DetectorType.YOLO_v9: + boxes, conf, classes = self._postprocessing_YOLO_v9(model_output[0][i]) elif self.model_type == DetectorType.YOLO_ULTRALYTICS: boxes, conf, classes = self._postprocessing_YOLO_ULTRALYTICS(model_output[0][i]) elif self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION: @@ -266,6 +273,33 @@ def _postprocessing_YOLO_v6(self, model_output): return boxes, conf, classes + def _postprocessing_YOLO_v9(self, model_output): + model_output = np.transpose(model_output, (1, 0)) + + outputs_filtered = np.array( + list(filter(lambda x: np.max(x[4:]) >= self.confidence, model_output)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [] + + probabilities = np.max(outputs_filtered[:, 4:], axis=1) + + outputs_x1y1x2y2 = self.xywh2xyxy(outputs_filtered) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2, + probs=probabilities, + iou_threshold=self.iou_threshold) + + outputs_nms = outputs_x1y1x2y2[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = np.max(outputs_nms[:, 4:], axis=1) + classes = np.argmax(outputs_nms[:, 4:], axis=1) + + return boxes, conf, classes + def _postprocessing_YOLO_ULTRALYTICS(self, model_output): model_output = np.transpose(model_output, (1, 0)) diff --git a/test/manual_test_map_processor_detection_yolov9.py b/test/manual_test_map_processor_detection_yolov9.py new file mode 100644 index 0000000..2bd849e --- /dev/null +++ b/test/manual_test_map_processor_detection_yolov9.py @@ -0,0 +1,61 @@ +import os +from pathlib import Path +from test.test_utils import create_default_input_channels_mapping_for_rgb_bands, create_rlayer_from_file, init_qgis +from unittest.mock import MagicMock + +from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions +from deepness.common.processing_parameters.detection_parameters import DetectionParameters, DetectorType +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType +from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection +from deepness.processing.models.detector import Detector + +# Files and model from github issue: https://github.com/PUTvision/qgis-plugin-deepness/discussions/101 + +HOME_DIR = Path(__file__).resolve().parents[1] +EXAMPLE_DATA_DIR = os.path.join(HOME_DIR, 'examples') + +MODEL_FILE_PATH = os.path.join(EXAMPLE_DATA_DIR, 'manually_downloaded/yolov9_trees.onnx') +RASTER_FILE_PATH = os.path.join(EXAMPLE_DATA_DIR, 'deeplabv3_segmentation_landcover/N-33-60-D-c-4-2.tif') + +print(RASTER_FILE_PATH) + +INPUT_CHANNELS_MAPPING = create_default_input_channels_mapping_for_rgb_bands() + + +def test_map_processor_detection_yolov6(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model_wrapper = Detector(MODEL_FILE_PATH) + + params = DetectionParameters( + resolution_cm_per_px=20, + tile_size_px=model_wrapper.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=0), + model=model_wrapper, + confidence=0.1, + iou_threshold=0.4, + detector_type=DetectorType.YOLO_v9, + ) + + map_processor = MapProcessorDetection( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + + assert len(map_processor.get_all_detections()) == 36 + + +if __name__ == '__main__': + test_map_processor_detection_yolov6() + print('Done') From 9ff4811f194f2af2978be3721e1004d7ec3f29c0 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Wed, 20 Mar 2024 22:46:45 +0100 Subject: [PATCH 22/32] Restore two channels for sigmoid segmentation --- .../creators/creators_description_classes.rst | 6 +++--- .../map_processor_segmentation.py | 12 +++-------- src/deepness/processing/models/segmentor.py | 15 ++++++++++++-- ...rocessor_segmentation_many_output_types.py | 20 +++++++++++++++---- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/docs/source/creators/creators_description_classes.rst b/docs/source/creators/creators_description_classes.rst index 6555594..8528675 100644 --- a/docs/source/creators/creators_description_classes.rst +++ b/docs/source/creators/creators_description_classes.rst @@ -37,9 +37,9 @@ We support the following types of models: * multiple outputs (multiple heads) with each output head composed of the same shapes as single output. Metaparameter :code:`class_names` saved in the model file should be as follows in this example: - * for single output with binary classification (sigmoid): :code:`[{0: "class_name"}]` or :code:`{0: "class_name"}` + * for single output with binary classification (sigmoid): :code:`[{0: "background", 1: "class_name"}]` * for single output with multi-class classification (softmax): :code:`[{0: "class0", 1: "class1", 2: "class2"}]` or :code:`{0: "class0", 1: "class1", 2: "class2"}` - * for multiple outputs (multiple heads): :code:`[{0: "class0", 1: "class1", 2: "class2"}, {0: "class0"}]` + * for multiple outputs (multiple heads): :code:`[{0: "class0", 1: "class1", 2: "class2"}, {0: "background", 1: "class_name"}]` Output report contains information about percentage coverage of each class. @@ -66,7 +66,7 @@ Regression models allow to solve problem of Regression Analysis, that is assigni Example application is determining the moisture content in soil, as percentage from 0.0 to 100.0 %, with an individual value assigned to each pixel. The segmentation model output is also an image, with same dimension as the input tile, with one or many output maps. Each output map contains the values for pixels. -Therefore, the shape of model output is :code:`[BATCH_SIZE, NUMBER_OF_OUTPUT_MAPS, SIZE_PX, SIZE_PX)`. +Therefore, the shape of model output is :code:`[BATCH_SIZE, NUMBER_OF_OUTPUT_MAPS, SIZE_PX, SIZE_PX]`. One output layer will be created for each output map (channel). For each output, a raster layer will be created, where each pixel has individual value assigned. diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index 4c2e022..9ee0bf6 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -86,7 +86,7 @@ def _create_result_message(self, result_img: np.ndarray) -> str: number_of_pixels_in_processing_area = np.sum([counts_map[k] for k in counts_map.keys()]) total_area = number_of_pixels_in_processing_area * self.params.resolution_m_per_px**2 - for channel_id in range(1, layer_sizes + 1): + for channel_id in range(layer_sizes): # See note in the class description why are we adding/subtracting 1 here pixels_count = counts_map.get(channel_id, 0) area = pixels_count * self.params.resolution_m_per_px**2 @@ -115,7 +115,7 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): output_vlayers = [] - for channel_id in range(1, layer_sizes + 1): + for channel_id in range(layer_sizes): # See note in the class description why are we adding/subtracting 1 here local_mask_img = np.uint8(mask_img[output_id] == channel_id) @@ -137,13 +137,7 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: else: pass # just nothing, we already have an empty list of features - # hardcode if someone add "background class" for sigmoid output, we need to skip it - if self._check_output_layer_is_sigmoid_and_has_more_than_one_name(output_id): - channel_id_name = channel_id - else: - channel_id_name = channel_id - 1 - - layer_name = self.model.get_channel_name(output_id, channel_id_name) + layer_name = self.model.get_channel_name(output_id, channel_id) vlayer = QgsVectorLayer("multipolygon", layer_name, "memory") vlayer.setCrs(self.rlayer.crs()) prov = vlayer.dataProvider() diff --git a/src/deepness/processing/models/segmentor.py b/src/deepness/processing/models/segmentor.py index 487c811..91a0a5c 100644 --- a/src/deepness/processing/models/segmentor.py +++ b/src/deepness/processing/models/segmentor.py @@ -24,6 +24,13 @@ def __init__(self, model_file_path: str): super(Segmentor, self).__init__(model_file_path) self.outputs_are_sigmoid = self.check_loaded_model_outputs() + + for idx in range(len(self.outputs_layers)): + if self.outputs_names is None: + continue + + if len(self.outputs_names[idx]) == 1 and self.outputs_are_sigmoid[idx]: + self.outputs_names[idx] = ['background', self.outputs_names[idx][0]] def postprocessing(self, model_output: List) -> np.ndarray: """ Postprocess the model output. @@ -54,9 +61,13 @@ def get_number_of_output_channels(self) -> List[int]: ls = layer.shape if len(ls) == 3: - output_channels.append(1) + output_channels.append(2) elif len(ls) == 4: - output_channels.append(ls[-3]) + chn = ls[-3] + if chn == 1: + output_channels.append(2) + else: + output_channels.append(chn) return output_channels diff --git a/test/test_map_processor_segmentation_many_output_types.py b/test/test_map_processor_segmentation_many_output_types.py index 4abf024..0dab7ea 100644 --- a/test/test_map_processor_segmentation_many_output_types.py +++ b/test/test_map_processor_segmentation_many_output_types.py @@ -64,9 +64,12 @@ def test_dummy_model_segmentation_processing__1x1x512x512(): channels = map_processor._get_indexes_of_model_output_channels_to_create() assert len(channels) == 1 - assert channels[0] == 1 + assert channels[0] == 2 name = map_processor.model.get_channel_name(0, 0) + assert name == 'background' + + name = map_processor.model.get_channel_name(0, 1) assert name == 'Coffee' def test_dummy_model_segmentation_processing__1x512x512(): @@ -104,9 +107,12 @@ def test_dummy_model_segmentation_processing__1x512x512(): channels = map_processor._get_indexes_of_model_output_channels_to_create() assert len(channels) == 1 - assert channels[0] == 1 + assert channels[0] == 2 name = map_processor.model.get_channel_name(0, 0) + assert name == 'background' + + name = map_processor.model.get_channel_name(0, 1) assert name == 'Coffee' def test_dummy_model_segmentation_processing__1x2x512x512(): @@ -179,13 +185,19 @@ def test_dummy_model_segmentation_processing__two_outputs_1x1x512x512(): channels = map_processor._get_indexes_of_model_output_channels_to_create() assert len(channels) == 2 - assert channels[0] == 1 - assert channels[1] == 1 + assert channels[0] == 2 + assert channels[1] == 2 name = map_processor.model.get_channel_name(0, 0) + assert name == 'background' + + name = map_processor.model.get_channel_name(0, 1) assert name == 'Coffee' name = map_processor.model.get_channel_name(1, 0) + assert name == 'background' + + name = map_processor.model.get_channel_name(1, 1) assert name == 'Juice' def test_dummy_model_segmentation_processing__two_outputs_1x512x512(): From 3d5794d860d792d83b320a3a7200e9143d5c2f23 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 21 Mar 2024 11:31:29 +0100 Subject: [PATCH 23/32] Export fixed --- .../map_processor_training_data_export.py | 5 +- ...test_map_processor_training_data_export.py | 83 +++++++------------ 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/src/deepness/processing/map_processor/map_processor_training_data_export.py b/src/deepness/processing/map_processor/map_processor_training_data_export.py index 2968d21..604c214 100644 --- a/src/deepness/processing/map_processor/map_processor_training_data_export.py +++ b/src/deepness/processing/map_processor/map_processor_training_data_export.py @@ -3,6 +3,7 @@ import datetime import os +import numpy as np from qgis.core import QgsProject from deepness.common.lazy_package_loader import LazyPackageLoader @@ -47,8 +48,10 @@ def _run(self): vlayer_mask=vlayer_segmentation, extended_extent=self.extended_extent, rlayer_units_per_pixel=self.rlayer_units_per_pixel, - image_shape_yx=(1, self.img_size_y_pixels, self.img_size_x_pixels), + image_shape_yx=(self.img_size_y_pixels, self.img_size_x_pixels), files_handler=self.file_handler) + + segmentation_mask_full = segmentation_mask_full[np.newaxis, ...] number_of_written_tiles = 0 for tile_img, tile_params in self.tiles_generator(): diff --git a/test/test_map_processor_training_data_export.py b/test/test_map_processor_training_data_export.py index 7ce6372..37919bb 100644 --- a/test/test_map_processor_training_data_export.py +++ b/test/test_map_processor_training_data_export.py @@ -1,8 +1,13 @@ +import os +from glob import glob from test.test_utils import (create_default_input_channels_mapping_for_rgba_bands, create_rlayer_from_file, create_vlayer_from_file, get_dummy_fotomap_area_path, get_dummy_fotomap_small_path, init_qgis) from unittest.mock import MagicMock +import cv2 +import numpy as np + from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType from deepness.common.processing_parameters.training_data_export_parameters import TrainingDataExportParameters @@ -40,60 +45,32 @@ def test_export_dummy_fotomap(): ) map_processor.run() - # TODO - validate processing result (we expect to have xxx tiles in directory yyy) -# -# def test_export_google_earth(): -# """ -# Just a test to debug part of processing with Google Earth Satellite images. -# idk how to create this layer in Python, so I loaded a project which contains this layer. -# But then, this layer works only partially, therefore this test is commented -# :return: -# """ -# qgs = init_qgis() -# -# project = QgsProject.instance() -# project.read('/home/przemek/Desktop/corn/borecko/qq.qgz') -# for layer_id, layer in project.mapLayers().items(): -# if 'Google Satellite' in layer.name(): -# rlayer = layer -# break -# -# if not rlayer.dataProvider(): -# # it looks like the google satellite layer is not working outside of GUI, -# # even if loading from project where it is -# print('Cannot perform "export_google_earth_test" - cannot use google satellite layer') -# return -# -# params = TrainingDataExportParameters( -# export_image_tiles=True, -# resolution_cm_per_px=3, -# segmentation_mask_layer_id=None, -# output_directory_path='/tmp/qgis_test', -# tile_size_px=512, # same x and y dimensions, so take x -# processed_area_type=ProcessedAreaType.VISIBLE_PART, -# mask_layer_id=None, -# input_layer_id=rlayer.id(), -# input_channels_mapping=create_default_input_channels_mapping_for_google_satellite_bands(), -# processing_overlap=20, -# ) -# -# processed_extent = QgsRectangle( -# 1881649.80, 6867603.86, -# 1881763.08, 6867662.50) -# -# map_canvas = MagicMock() -# map_canvas.extent = lambda: processed_extent -# map_canvas.mapSettings().destinationCrs = lambda: QgsCoordinateReferenceSystem("EPSG:3857") -# -# map_processor = MapProcessorTrainingDataExport( -# rlayer=rlayer, -# vlayer_mask=None, -# map_canvas=map_canvas, -# params=params, -# ) -# -# map_processor.run() + images_results = glob(os.path.join(map_processor.output_dir_path, '*_img_*.png')) + masks_results = glob(os.path.join(map_processor.output_dir_path, '*_mask_*.png')) + + assert len(images_results) == 4 + assert len(masks_results) == 4 + + mask_values = [ + (237225, 24919), + (236341, 25803), + (140591, 121553), + (133202, 128942) + ] + + + for i, mask_file in enumerate(masks_results): + mask = cv2.imread(mask_file, cv2.IMREAD_UNCHANGED) + + assert len(mask.shape) == 2 + assert mask.shape[0] == 512 + assert mask.shape[1] == 512 + + assert np.unique(mask).tolist() == [0, 255] + + assert np.isclose(np.sum(mask < 128), mask_values[i][0], atol=10) + assert np.isclose(np.sum(mask >= 128), mask_values[i][1], atol=10) if __name__ == '__main__': From ab99982a4ad871e3b9500ec91d2422fc480712bb Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 21 Mar 2024 11:37:24 +0100 Subject: [PATCH 24/32] Sort files in export test --- test/test_map_processor_training_data_export.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_map_processor_training_data_export.py b/test/test_map_processor_training_data_export.py index 37919bb..558aba4 100644 --- a/test/test_map_processor_training_data_export.py +++ b/test/test_map_processor_training_data_export.py @@ -46,20 +46,19 @@ def test_export_dummy_fotomap(): map_processor.run() - images_results = glob(os.path.join(map_processor.output_dir_path, '*_img_*.png')) - masks_results = glob(os.path.join(map_processor.output_dir_path, '*_mask_*.png')) + images_results = sorted(glob(os.path.join(map_processor.output_dir_path, '*_img_*.png'))) + masks_results = sorted(glob(os.path.join(map_processor.output_dir_path, '*_mask_*.png'))) assert len(images_results) == 4 assert len(masks_results) == 4 - + mask_values = [ - (237225, 24919), (236341, 25803), - (140591, 121553), - (133202, 128942) + (133202, 128942), + (237225, 24919), + (140591, 121553) ] - for i, mask_file in enumerate(masks_results): mask = cv2.imread(mask_file, cv2.IMREAD_UNCHANGED) @@ -71,6 +70,7 @@ def test_export_dummy_fotomap(): assert np.isclose(np.sum(mask < 128), mask_values[i][0], atol=10) assert np.isclose(np.sum(mask >= 128), mask_values[i][1], atol=10) + # print(np.sum(mask < 128), np.sum(mask >= 128)) if __name__ == '__main__': From 7e7d0098be9c0247f2231fd5a0663327be30c37e Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 21 Mar 2024 11:46:10 +0100 Subject: [PATCH 25/32] Add YOLOv9 model --- docs/source/main/main_ui_explanation.rst | 2 -- docs/source/main/model_zoo/MODEL_ZOO.md | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/main/main_ui_explanation.rst b/docs/source/main/main_ui_explanation.rst index d733aa9..64d4443 100644 --- a/docs/source/main/main_ui_explanation.rst +++ b/docs/source/main/main_ui_explanation.rst @@ -82,8 +82,6 @@ These options may be a fixed value for some models. **IoU threshold** - Parameter used in Non-Maximum Suppression in post-processing. Defines the threshold of overlap between neighboring detections, to consider them as the same object. -**Remove overlapping detections** - If checked then the overlapping detections (which may be an artifact of overlapped processing) will be removed. - ------------ Tiles export ------------ diff --git a/docs/source/main/model_zoo/MODEL_ZOO.md b/docs/source/main/model_zoo/MODEL_ZOO.md index 78305bf..eb13c07 100644 --- a/docs/source/main/model_zoo/MODEL_ZOO.md +++ b/docs/source/main/model_zoo/MODEL_ZOO.md @@ -39,6 +39,7 @@ The [Model ZOO](https://chmura.put.poznan.pl/s/2pJk4izRurzQwu3) is a collection | [Airbus Oil Storage Detection](https://chmura.put.poznan.pl/s/gMundpKsYUC7sNb) | 512 | 150 | YOLOv5-m model for object detection on satellite images. Based on the [Airbus Oil Storage Detection dataset](https://www.kaggle.com/datasets/airbusgeo/airbus-oil-storage-detection-dataset). | [Image](https://chmura.put.poznan.pl/s/T3pwaKlbFDBB2C3) | | [Aerial Cars Detection](https://chmura.put.poznan.pl/s/vgOeUN4H4tGsrGm) | 640 | 10 | YOLOv7-m model for cars detection on aerial images. Based on the [ITCVD](https://arxiv.org/pdf/1801.07339.pdf). | [Image](https://chmura.put.poznan.pl/s/cPzw1mkXlprSUIJ) | | [UAVVaste Instance Segmentation](https://chmura.put.poznan.pl/s/v99rDlSPbyNpOCH) | 640 | 0.5 | YOLOv8-L Instance Segmentation model for litter detection on high-quality UAV images. Based on the [UAVVaste dataset](https://github.com/PUTvision/UAVVaste). | [Image](https://chmura.put.poznan.pl/s/KFQTlS2qtVnaG0q) | +| [Tree-Tops Detection](https://chmura.put.poznan.pl/s/A9zdp4mKAATEAGu) | 640 | 10 | YOLOv9 model for treetops detection on aerial images. Model is trained on the mix of publicly available datasets. | [Image](https://chmura.put.poznan.pl/s/F0lkIX9xhcwD4PG) | ## Super Resolution Models | Model | Input size | CM/PX | Scale Factor | Description | Example image | From 186463aae8df38565d0dd298c621fd1ed85ba3ff Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 21 Mar 2024 11:50:27 +0100 Subject: [PATCH 26/32] Docs improvements --- .../creators/creators_add_metadata_to_model.rst | 16 +++++++++++++++- .../creators/creators_description_classes.rst | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/source/creators/creators_add_metadata_to_model.rst b/docs/source/creators/creators_add_metadata_to_model.rst index 8adf948..9454c12 100644 --- a/docs/source/creators/creators_add_metadata_to_model.rst +++ b/docs/source/creators/creators_add_metadata_to_model.rst @@ -11,7 +11,7 @@ List of parameters parsed by plugin +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ | Parameter | Type | Example | Description | +======================+=======+=======================================+=============================================================+ -| model_type | str | :code:`'Segmentor'` | Types of models available: Segmentor, Regressor, Detector. | +| model_type | str | :code:`'Segmentor'` | Types of models. | +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ | class_names | dict | :code:`{0: 'name1', 1: 'name2'}` | A dictionary that maps a class id to its name. | +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ @@ -40,6 +40,20 @@ List of parameters parsed by plugin | det_type | str | :code:`YOLO_v5_or_v7_default` | Detector: type of the detector model format | +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ +Available model types: +- :code:`Segmentor` +- :code:`Detector` +- :code:`Regressor` +- :code:`Recognition` +- :code:`Superresolution` + +Availeble detector types: +- :code:`YOLO_v5_or_v7_default` +- :code:`YOLO_v6` +- :code:`YOLO_v9` +- :code:`YOLO_Ultralytics` +- :code:`YOLO_Ultralytics_segmentation` + ======= Example ======= diff --git a/docs/source/creators/creators_description_classes.rst b/docs/source/creators/creators_description_classes.rst index 6555594..752cbcd 100644 --- a/docs/source/creators/creators_description_classes.rst +++ b/docs/source/creators/creators_description_classes.rst @@ -52,7 +52,7 @@ Detection models allow to solve problem of objects detection, that is finding an Example application is detection of oil and water tanks on satellite images. The detection model output is list of bounding boxes, with assigned class and confidence value. This information is not really standardized between different model architectures. -Currently plugin supports :code:`YOLOv5`, :code:`YOLOv7` and :code:`ULTRALYTICS` output types. Detection model also supports the instance segmentation output type from :code:`ULTRALYTICS`. +Currently plugin supports :code:`YOLOv5`, :code:`YOLOv6`, :code:`YOLOv7`, :code:`YOLOv9` and :code:`ULTRALYTICS` output types. Detection model also supports the instance segmentation output type from :code:`ULTRALYTICS`. For each object class, a separate vector layer can be created, with information saved as rectangle polygons (so the output can be potentially easily exported to a text). From c492e4e7489143f560ad01a94bf16eba71c68c1a Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 21 Mar 2024 11:51:43 +0100 Subject: [PATCH 27/32] Docs improvements --- docs/source/creators/creators_add_metadata_to_model.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/creators/creators_add_metadata_to_model.rst b/docs/source/creators/creators_add_metadata_to_model.rst index 9454c12..2b8fbc0 100644 --- a/docs/source/creators/creators_add_metadata_to_model.rst +++ b/docs/source/creators/creators_add_metadata_to_model.rst @@ -10,8 +10,8 @@ List of parameters parsed by plugin +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ | Parameter | Type | Example | Description | -+======================+=======+=======================================+=============================================================+ -| model_type | str | :code:`'Segmentor'` | Types of models. | ++======================+=======+=======================================+====================================================================+ +| model_type | str | :code:`'Segmentor'` | Types of models. | +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ | class_names | dict | :code:`{0: 'name1', 1: 'name2'}` | A dictionary that maps a class id to its name. | +-----------------------------+-------+---------------------------------------+-------------------------------------------------------------+ From 0d05b73c21a0916d409c0e4ff18ed257477381ae Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 21 Mar 2024 11:53:28 +0100 Subject: [PATCH 28/32] Update test name --- test/manual_test_map_processor_detection_yolov9.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/manual_test_map_processor_detection_yolov9.py b/test/manual_test_map_processor_detection_yolov9.py index 2bd849e..3532966 100644 --- a/test/manual_test_map_processor_detection_yolov9.py +++ b/test/manual_test_map_processor_detection_yolov9.py @@ -22,7 +22,7 @@ INPUT_CHANNELS_MAPPING = create_default_input_channels_mapping_for_rgb_bands() -def test_map_processor_detection_yolov6(): +def test_map_processor_detection_yolov9(): qgs = init_qgis() rlayer = create_rlayer_from_file(RASTER_FILE_PATH) @@ -57,5 +57,5 @@ def test_map_processor_detection_yolov6(): if __name__ == '__main__': - test_map_processor_detection_yolov6() + test_map_processor_detection_yolov9() print('Done') From 90646f6ad040ff4d3a08f9fcff7561f40a243e63 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Thu, 21 Mar 2024 13:32:33 +0100 Subject: [PATCH 29/32] Fixes after testing with single sigmoid --- .../map_processor_segmentation.py | 40 +++++------ src/deepness/processing/models/segmentor.py | 32 ++------- test/data/dummy_model/GenerateDummy.ipynb | 62 ++++++++++++++++-- .../one_output_sigmoid_red_detector.onnx | Bin 0 -> 748 bytes 4 files changed, 81 insertions(+), 53 deletions(-) create mode 100644 test/data/dummy_model/one_output_sigmoid_red_detector.onnx diff --git a/src/deepness/processing/map_processor/map_processor_segmentation.py b/src/deepness/processing/map_processor/map_processor_segmentation.py index 9ee0bf6..ac9b825 100644 --- a/src/deepness/processing/map_processor/map_processor_segmentation.py +++ b/src/deepness/processing/map_processor/map_processor_segmentation.py @@ -47,10 +47,10 @@ def _run(self) -> MapProcessingResult: full_result_img=full_result_img) blur_size = int(self.segmentation_parameters.postprocessing_dilate_erode_size // 2) * 2 + 1 # needs to be odd - + for i in range(full_result_img.shape[0]): full_result_img[i] = cv2.medianBlur(full_result_img[i], blur_size) - + full_result_img = self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_result_img) self.set_results_img(full_result_img) @@ -62,48 +62,42 @@ def _run(self) -> MapProcessingResult: message=result_message, gui_delegate=gui_delegate, ) - + def _check_output_layer_is_sigmoid_and_has_more_than_one_name(self, output_id: int) -> bool: if self.model.outputs_names is None or self.model.outputs_are_sigmoid is None: return False - + return len(self.model.outputs_names[output_id]) > 1 and self.model.outputs_are_sigmoid[output_id] def _create_result_message(self, result_img: np.ndarray) -> str: - + txt = f'Segmentation done, with the following statistics:\n' - + for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): - + txt += f'Channels for output {output_id}:\n' - + unique, counts = np.unique(result_img[output_id], return_counts=True) counts_map = {} for i in range(len(unique)): counts_map[unique[i]] = counts[i] - + # # we cannot simply take image dimensions, because we may have irregular processing area from polygon number_of_pixels_in_processing_area = np.sum([counts_map[k] for k in counts_map.keys()]) total_area = number_of_pixels_in_processing_area * self.params.resolution_m_per_px**2 - + for channel_id in range(layer_sizes): # See note in the class description why are we adding/subtracting 1 here pixels_count = counts_map.get(channel_id, 0) area = pixels_count * self.params.resolution_m_per_px**2 - + if total_area > 0 and not np.isnan(total_area) and not np.isinf(total_area): area_percentage = area / total_area * 100 else: area_percentage = 0.0 # TODO - - # hardcode if someone add "background class" for sigmoid output, we need to skip it - if self._check_output_layer_is_sigmoid_and_has_more_than_one_name(output_id): - channel_id_name = channel_id - else: - channel_id_name = channel_id - 1 - - txt += f'\t- {self.model.get_channel_name(output_id, channel_id_name)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' + + txt += f'\t- {self.model.get_channel_name(output_id, channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' return txt @@ -136,7 +130,7 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: current_contour_index=0) else: pass # just nothing, we already have an empty list of features - + layer_name = self.model.get_channel_name(output_id, channel_id) vlayer = QgsVectorLayer("multipolygon", layer_name, "memory") vlayer.setCrs(self.rlayer.crs()) @@ -152,13 +146,13 @@ def _create_vlayer_from_mask_for_base_extent(self, mask_img) -> Callable: vlayer.updateExtents() output_vlayers.append(vlayer) - + vlayers.append(output_vlayers) # accessing GUI from non-GUI thread is not safe, so we need to delegate it to the GUI thread def add_to_gui(): group = QgsProject.instance().layerTreeRoot().insertGroup(0, 'model_output') - + if len(vlayers) == 1: for vlayer in vlayers[0]: QgsProject.instance().addMapLayer(vlayer, False) @@ -194,5 +188,5 @@ def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: many_outputs.append(result[:, 0]) many_outputs = np.array(many_outputs).transpose((1, 0, 2, 3)) - + return many_outputs diff --git a/src/deepness/processing/models/segmentor.py b/src/deepness/processing/models/segmentor.py index 91a0a5c..6dabf26 100644 --- a/src/deepness/processing/models/segmentor.py +++ b/src/deepness/processing/models/segmentor.py @@ -22,13 +22,13 @@ def __init__(self, model_file_path: str): Path to the model file """ super(Segmentor, self).__init__(model_file_path) - + self.outputs_are_sigmoid = self.check_loaded_model_outputs() - + for idx in range(len(self.outputs_layers)): if self.outputs_names is None: continue - + if len(self.outputs_names[idx]) == 1 and self.outputs_are_sigmoid[idx]: self.outputs_names[idx] = ['background', self.outputs_names[idx][0]] @@ -82,41 +82,23 @@ def get_class_display_name(cls): """ return cls.__name__ - def check_loaded_model_outputs(self): - """ Checks if the model outputs are valid - - Valid means that: - - the model has at least one output - - the output is 4D (N,C,H,W) or 3D (N,H,W) - - the batch size is 1 or dynamic - - model resolution is equal to TILE_SIZE (is square) - - """ - if len(self.outputs_layers) == 0: - raise Exception('Model has no output layers') - - for layer in self.outputs_layers: - if len(layer.shape) != 4 and len(layer.shape) != 3: - raise Exception(f'Segmentation model output should have 4 dimensions: (B,C,H,W) or 3 dimensions: (B,H,W). Has {layer.shape}') - - def check_loaded_model_outputs(self) -> List[bool]: """ Check if the model outputs are sigmoid (for segmentation) - + Parameters ---------- - + Returns ------- List[bool] List of booleans indicating if the model outputs are sigmoid """ outputs = [] - + for output in self.outputs_layers: if len(output.shape) == 3: outputs.append(True) else: outputs.append(output.shape[-3] == 1) - + return outputs diff --git a/test/data/dummy_model/GenerateDummy.ipynb b/test/data/dummy_model/GenerateDummy.ipynb index be295a8..538bfb5 100644 --- a/test/data/dummy_model/GenerateDummy.ipynb +++ b/test/data/dummy_model/GenerateDummy.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "id": "7ad5306f-18fc-4499-b764-ab6902ac301f", "metadata": {}, "outputs": [], @@ -33,7 +33,7 @@ " super().__init__()\n", "\n", " self.conv = nn.Conv2d(3, 1, 1)\n", - " \n", + "\n", " self.softmax = nn.Softmax(dim=1)\n", " self.sigmoid = nn.Sigmoid()\n", "\n", @@ -62,11 +62,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "79c0bfcb-47c4-44c9-9c9b-c4c75b2c3b34", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 3, 512, 512])\n", + "torch.Size([1, 1, 512, 512])\n", + "torch.Size([1, 3, 512, 512])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/przemek/Projects/qgis-plugin-deepness/.venv/lib/python3.10/site-packages/torch/onnx/utils.py:2095: UserWarning: Provided key red for dynamic axes is not a valid input/output name\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "x = torch.rand(1, 3, 512, 512)\n", + "\n", + "class Model(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " self.sigmoid = nn.Sigmoid()\n", + "\n", + " def forward(self, x):\n", + " print(x.shape) # torch.Size([1, 3, 512, 512])\n", + " x = (x[:,0:1]-0.5)\n", + "\n", + " # unsqueeze to add channel dimension\n", + " # x = x.unsqueeze(1)\n", + "\n", + " return self.sigmoid(x) #, self.sigmoid(x)\n", + "\n", + "m = Model()\n", + "# print(m(x)[0].shape, m(x)[1].shape)\n", + "print(m(x).shape)\n", + "\n", + "torch.onnx.export(m,\n", + " x,\n", + " \"one_output_sigmoid_red_detector.onnx\",\n", + " export_params=True,\n", + " opset_version=12,\n", + " do_constant_folding=True,\n", + " input_names = ['input'],\n", + " output_names = ['output'],\n", + " dynamic_axes={'input' : {0 : 'batch_size'},\n", + " 'output' : {0 : 'batch_size'},\n", + " # 'output2' : {0 : 'batch_size'},\n", + " })" + ] } ], "metadata": { diff --git a/test/data/dummy_model/one_output_sigmoid_red_detector.onnx b/test/data/dummy_model/one_output_sigmoid_red_detector.onnx new file mode 100644 index 0000000000000000000000000000000000000000..2d3815b72cab5c40be29e6457c55b6500bd30652 GIT binary patch literal 748 zcma))%}#?r6opHHGG42aYuykRhQx(!Qd(L!Zj3fA?3SGy0+f_V%1_$CL=zJq#dq?B zq=j}y#YPsxaOU1`&K!nlsAVrKT?Ub%+I_n}y!lk&0nwVI$%=UibJLt*D zcC~AO`s^+8uzO4-U7HfAQ9d7%0*~0)wYWnLIuz7BsEXA1g|cf=_8!!qu12+}(LYgd zpwgrW2WrnLs@yWvL@QQuM|fg63mSynVyhHPb)&;X%y-PognY-sNjlqKMRaMZ6zSfs zlf8qV+lX#B6^|?@_xUEu`qPw)0c`lbhOP3>7`nBh^Ejn5%PHTcR!tCMk0$PX>1EMV zuZj^(xF8aqLmR>OSP;1@`Wix$$RLwjmS4H(DoZ)Fjt*+{f#_phKlX%;G HQ*VC(6V1~u literal 0 HcmV?d00001 From 5720a745f3b517b811657dcf26b567e833f2cde0 Mon Sep 17 00:00:00 2001 From: Bartosz Ptak Date: Fri, 22 Mar 2024 12:48:38 +0100 Subject: [PATCH 30/32] Update version to 0.6.2 --- src/deepness/metadata.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deepness/metadata.txt b/src/deepness/metadata.txt index 4dc9c33..803762f 100644 --- a/src/deepness/metadata.txt +++ b/src/deepness/metadata.txt @@ -6,7 +6,7 @@ name=Deepness: Deep Neural Remote Sensing qgisMinimumVersion=3.22 description=Inference of deep neural network models (ONNX) for segmentation, detection and regression -version=0.6.1 +version=0.6.2 author=PUT Vision email=przemyslaw.aszkowski@gmail.com From a0505a7d4c2038fa1fcef992ed410e0d623f5af0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 22 Mar 2024 12:59:56 +0100 Subject: [PATCH 31/32] Sigmoid model test added --- ...test_map_processor_segmentation_sigmoid.py | 72 +++++++++++++++++++ test/test_utils.py | 10 +++ 2 files changed, 82 insertions(+) create mode 100644 test/test_map_processor_segmentation_sigmoid.py diff --git a/test/test_map_processor_segmentation_sigmoid.py b/test/test_map_processor_segmentation_sigmoid.py new file mode 100644 index 0000000..3ba3f12 --- /dev/null +++ b/test/test_map_processor_segmentation_sigmoid.py @@ -0,0 +1,72 @@ +from test.test_utils import (create_default_input_channels_mapping_for_rgba_bands, create_rlayer_from_file, + create_vlayer_from_file, get_dummy_fotomap_area_crs3857_path, get_dummy_fotomap_area_path, + get_dummy_fotomap_small_path, get_dummy_segmentation_model_path, get_dummy_sigmoid_model_path, init_qgis) +from unittest.mock import MagicMock + +import matplotlib.pyplot as plt +import numpy as np +from qgis.core import QgsCoordinateReferenceSystem, QgsRectangle + +from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions +from deepness.common.processing_parameters.map_processing_parameters import ProcessedAreaType +from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation +from deepness.processing.models.segmentor import Segmentor + +RASTER_FILE_PATH = get_dummy_fotomap_small_path() + +VLAYER_MASK_FILE_PATH = get_dummy_fotomap_area_path() + +VLAYER_MASK_CRS3857_FILE_PATH = get_dummy_fotomap_area_crs3857_path() + +MODEL_FILE_PATH = get_dummy_sigmoid_model_path() + +INPUT_CHANNELS_MAPPING = create_default_input_channels_mapping_for_rgba_bands() + + +def test_sigmoid_model_processing__entire_file(): + qgs = init_qgis() + + rlayer = create_rlayer_from_file(RASTER_FILE_PATH) + model = Segmentor(MODEL_FILE_PATH) + + params = SegmentationParameters( + resolution_cm_per_px=3, + tile_size_px=model.get_input_size_in_pixels()[0], # same x and y dimensions, so take x + batch_size=1, + local_cache=False, + processed_area_type=ProcessedAreaType.ENTIRE_LAYER, + mask_layer_id=None, + input_layer_id=rlayer.id(), + input_channels_mapping=INPUT_CHANNELS_MAPPING, + postprocessing_dilate_erode_size=5, + processing_overlap=ProcessingOverlap(ProcessingOverlapOptions.OVERLAP_IN_PERCENT, percentage=20), + pixel_classification__probability_threshold=0.6, + model=model, + ) + + map_processor = MapProcessorSegmentation( + rlayer=rlayer, + vlayer_mask=None, + map_canvas=MagicMock(), + params=params, + ) + + map_processor.run() + result_img = map_processor.get_result_img() + + assert result_img.shape == (1, 561, 829) + non_zero_pixels = np.count_nonzero(result_img) + assert non_zero_pixels == 25002 # number of RED pixels in the image + + # you should see only the part of RASTER_FILE_PATH that is pure red pixels. Use snippet below for debugging + # from matplotlib import pyplot as plt + # plt.imshow(result_img[0]) + # plt.show() + + # TODO - add detailed check for pixel values once we have output channels mapping with thresholding + + +if __name__ == '__main__': + test_sigmoid_model_processing__entire_file() + print('Done') diff --git a/test/test_utils.py b/test/test_utils.py index 5181718..255aadd 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -19,6 +19,14 @@ def get_dummy_segmentation_model_path(): return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'dummy_model.onnx') +def get_dummy_sigmoid_model_path(): + """ + Get path of a dummy onnx model. See details in README in model directory. + Model used for unit tests processing purposes + """ + return os.path.join(TEST_DATA_DIR, 'dummy_model', 'one_output_sigmoid_red_detector.onnx') + + def get_dummy_segmentation_model_different_output_size_path(): """ Get path of a dummy onnx model. See details in README in model directory. @@ -26,6 +34,7 @@ def get_dummy_segmentation_model_different_output_size_path(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_segmentation_models', 'different_output_size_512_to_484.onnx') + def get_dummy_segmentation_models_dict(): """ Get dictionary with dummy segmentation models paths. See details in README in model directory. @@ -82,6 +91,7 @@ def get_dummy_regression_model_path_batched(): """ return os.path.join(TEST_DATA_DIR, 'dummy_model', 'dummy_regression_models', 'dummy_regression_model_batched.onnx') + def get_dummy_regression_models_dict(): """ Get dictionary with dummy regression models paths. See details in README in model directory. From 6e7bf99780ebae051b4591c8b73485511e19c764 Mon Sep 17 00:00:00 2001 From: Przemyslaw Aszkowski Date: Fri, 22 Mar 2024 13:06:12 +0100 Subject: [PATCH 32/32] Asserting result is close, so it can run on GPU --- test/test_map_processor_segmentation_sigmoid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_map_processor_segmentation_sigmoid.py b/test/test_map_processor_segmentation_sigmoid.py index 3ba3f12..46d74f3 100644 --- a/test/test_map_processor_segmentation_sigmoid.py +++ b/test/test_map_processor_segmentation_sigmoid.py @@ -57,7 +57,7 @@ def test_sigmoid_model_processing__entire_file(): assert result_img.shape == (1, 561, 829) non_zero_pixels = np.count_nonzero(result_img) - assert non_zero_pixels == 25002 # number of RED pixels in the image + assert abs(non_zero_pixels - 25002) < 50 # number of RED pixels in the image # you should see only the part of RASTER_FILE_PATH that is pure red pixels. Use snippet below for debugging # from matplotlib import pyplot as plt