Az/custom annotation

cvat/apps/custom_annotation/README.md

@@ -0,0 +1,94 @@
+## Custom annotation
+
+### Description
+
+This application will be enabled automatically if OpenVINO component was installed and allows to use custom detection models for preannotation.
+Supported frameworks:
+* DLDT form OpenVINO toolkit
+
+Application uses OpenCV dnn module with DLDT backed for inference.
+
+### Usage
+To annotate task with custom model you need prepare 4 files:
+1. **Model config** - a text file contains network configuration. It could be a file with the following extensions:
+   * *.xml (DLDT)
+1. **Model weights** - a binary file contains trained weights. The following file extensions are expected for models from different frameworks:
+   * *.bin (DLDT)
+1. **Preprocessing configureation and label map** - simple json file that describes image dimentions and preprocessing options. For more details please view [OpenCV](https://docs.opencv.org/3.4/d6/d0f/group__dnn.html#ga0b7b7c3c530b747ef738178835e1e70f) documentation.  
+Label values in label_map should be exactly equal to labels wich task was created, otherwise will be ignored.
+  Example:
+    ```json
+    {
+      "blob_params": {
+        "width": 300,
+        "height":300,
+        "mean": "127.5, 127.5, 127.5",
+        "scalefactor": 0.0078431372549,
+        "swapRB": false,
+        "crop": false
+      },
+      "label_map": {
+        "0": "background",
+        "1": "aeroplane",
+        "2": "bicycle",
+        "3": "bird",
+        "4": "boat",
+        "5": "bottle",
+        "6": "bus",
+        "7": "car",
+        "8": "cat",
+        "9": "chair",
+        "10": "cow",
+        "11": "diningtable",
+        "12": "dog",
+        "13": "horse",
+        "14": "motorbike",
+        "15": "person",
+        "16": "pottedplant",
+        "17": "sheep",
+        "18": "sofa",
+        "19": "train",
+        "20": "tvmonitor"
+      }
+    }
+    ```
+1. **Interpretation script** - python scripts that converts output results from net to CVAT format. File must contain function with following signature: `process_detections(detections):`. There detection is list object of dictionaries that represent detections for each frame of task with folloing keys:
+   * frame_id - frame number
+   * frame_height - frame height
+   * frame_width - frame width
+   * detections - output blob (See [cv::dnn::Net::forward](https://docs.opencv.org/3.4/db/d30/classcv_1_1dnn_1_1Net.html#a98ed94cb6ef7063d3697259566da310b) for details).  
+    Example for SSD based network
+    ```python
+    def process_detections(detections):
+      def clip(value):
+        return max(min(1.0, value), 0.0)
+
+      boxes = []
+      for frame_results in detections:
+        frame_height = frame_results['frame_height']
+        frame_width = frame_results['frame_width']
+        frame_number = frame_results['frame_id']
+
+        for i in range(frame_results['detections'].shape[2]):
+          confidence = frame_results['detections'][0, 0, i, 2]
+          if confidence < 0.4: continue
+
+          class_id = str(int(frame_results['detections'][0, 0, i, 1]))
+          xtl = '{:.2f}'.format(clip(frame_results['detections'][0, 0, i, 3]) * frame_width)
+          ytl = '{:.2f}'.format(clip(frame_results['detections'][0, 0, i, 4]) * frame_height)
+          xbr = '{:.2f}'.format(clip(frame_results['detections'][0, 0, i, 5]) * frame_width)
+          ybr = '{:.2f}'.format(clip(frame_results['detections'][0, 0, i, 6]) * frame_height)
+
+          boxes.append({
+            'label': class_id,
+            'frame': frame_number,
+            'xtl': xtl,
+            'ytl': ytl,
+            'xbr': xbr,
+            'ybr': ybr,
+            'attributes': {
+                'confidence': '{:.2f}'.format(confidence),
+            }
+          })
+      return {'boxes': boxes }
+    ```

cvat/apps/custom_annotation/__init__.py

@@ -0,0 +1,8 @@
+
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from cvat.settings.base import JS_3RDPARTY
+
+JS_3RDPARTY['dashboard'] = JS_3RDPARTY.get('dashboard', []) + ['custom_annotation/js/custom_annotation.js']

cvat/apps/custom_annotation/admin.py

@@ -0,0 +1,9 @@
+
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.contrib import admin
+
+# Register your models here.
+

cvat/apps/custom_annotation/apps.py

@@ -0,0 +1,11 @@
+
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.apps import AppConfig
+
+
+class CustomAnnotationConfig(AppConfig):
+    name = 'custom_annotation'
+

cvat/apps/custom_annotation/image_loader.py

@@ -0,0 +1,23 @@
+import cv2
+
+class ImageLoader():
+    def __init__(self, image_list):
+        self.image_list = image_list
+
+    def __getitem__(self, i):
+        return self.image_list[i]
+
+    def __iter__(self):
+        for imagename in self.image_list:
+            yield imagename, self.load_image(imagename)
+
+    def __len__(self):
+        return len(self.image_list)
+
+    @staticmethod
+    def load_image(path_to_image):
+        return cv2.imread(path_to_image)
+
+    @staticmethod
+    def _resize_image(image, size):
+        return cv2.resize(image, size)

cvat/apps/custom_annotation/migrations/__init__.py

@@ -0,0 +1,5 @@
+
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+

cvat/apps/custom_annotation/model_loader.py

@@ -0,0 +1,51 @@
+import cv2
+import json
+
+class BlobParamaters():
+    def __init__(self, scalefactor, input_size, mean, swapRB, crop):
+        self.scalefactor = scalefactor
+        self.input_size = input_size
+        self.mean = mean
+        self.swapRB = swapRB
+        self.crop = crop
+
+class ModelLoader():
+    def __init__(self, path_to_model, blob_params):
+        self.path_to_model = path_to_model
+        self.blob_params = blob_params
+
+    def load(self):
+        self.net = cv2.dnn.readNet(*self.path_to_model)
+        self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)
+        self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_DEFAULT)
+
+    def setInput(self, images):
+        blob = cv2.dnn.blobFromImages(images,
+            self.blob_params.scalefactor,
+            self.blob_params.input_size,
+            self.blob_params.mean,
+            self.blob_params.swapRB,
+            self.blob_params.crop)
+
+        self.net.setInput(blob)
+
+    def forward(self):
+        return self.net.forward()
+
+def read_model_config(config_path):
+        with open(config_path, 'r') as f:
+            return json.load(f)
+
+def get_blob_props(config):
+    blob_params = config['blob_params']
+    return BlobParamaters(
+        scalefactor=blob_params['scalefactor'] if 'scalefactor' in blob_params else 1.0,
+        input_size=(blob_params['height'], blob_params['width']) if 'height' in blob_params and 'width' in blob_params else (),
+        mean=tuple(float(v) for v in blob_params['mean'].split(',')) if 'mean' in blob_params else tuple(),
+        swapRB=blob_params['swapRB'] if 'swapRB' in blob_params else False,
+        crop=blob_params['crop'] if 'crop' in blob_params else False,
+    )
+
+def get_model_label_map(config):
+    return config['label_map']
+

cvat/apps/custom_annotation/models.py

@@ -0,0 +1,9 @@
+
+# Copyright (C) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.db import models
+
+# Create your models here.
+

cvat/apps/custom_annotation/static/custom_annotation/js/custom_annotation.js

@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2018 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+ "use strict";
+
+ window.cvat = window.cvat || {};
+window.cvat.dashboard = window.cvat.dashboard || {};
+window.cvat.dashboard.uiCallbacks = window.cvat.dashboard.uiCallbacks || [];
+window.cvat.dashboard.uiCallbacks.push(function(newElements) {
+    let tids = [];
+    for (let el of newElements) {
+        tids.push(el.id.split('_')[1]);
+    }
+
+    $.ajax({
+        type: 'POST',
+        url: '/custom_annotation/meta/get',
+        data: JSON.stringify(tids),
+        contentType: "application/json; charset=utf-8",
+        success: (data) => {
+            newElements.each(function(idx) {
+                let elem = $(newElements[idx]);
+                let tid = +elem.attr('id').split('_')[1];
+
+                const customAnnoButton = $('<button> Run custom annotation </button>').addClass('semiBold dashboardButtonUI dashboardCustomAnno');
+                customAnnoButton.appendTo(elem.find('div.dashboardButtonsUI')[0]);
+
+                if ((tid in data) && (data[tid].active)) {
+                    customAnnoButton.text('Cancel custom annotation');
+                    customAnnoButton.addClass('customAnnotationProcess');
+                    window.cvat.custom_annotation.checkCustomAnnotationRequest(tid, customAnnoButton);
+                }
+
+                customAnnoButton.on('click', () => {
+                    if (customAnnoButton.hasClass('customAnnotationProcess')) {
+                        $.post(`/custom_annotation/cancel/task/${tid}`).fail( (data) => {
+                            let message = `Error during cansel custom annotation request. Code: ${data.status}. Message: ${data.responseText || data.statusText}`;
+                            showMessage(message);
+                            throw Error(message);
+                        });
+                    }
+                    else {
+                        let dialogWindow = $(`#${window.cvat.custom_annotation.modalWindowId}`);
+                        dialogWindow.attr('current_tid', tid);
+                        dialogWindow.removeClass('hidden');
+                    }
+                });
+            });
+        },
+        error: (data) => {
+            let message = `Can not get custom annotation meta info. Code: ${data.status}. Message: ${data.responseText || data.statusText}`;
+            showMessage(message);
+            throw Error(message);
+        }
+    });
+});
+
+window.cvat.custom_annotation = {
+    modalWindowId: 'customAnnotationWindow',
+    customAnnoFromId: 'customAnnotationForm',
+    customAnnoModelFieldId: 'customAnnotationModelField',
+    customAnnoWeightsFieldId: 'customAnnotationWeightsField',
+    customAnnoConfigFieldId: 'customAnnotationConfigField',
+    customAnnoConvertFieldId: 'customAnnotationConvertField',
+    customAnnoCloseButtonId: 'customAnnoCloseButton',
+    customAnnoSubmitButtonId: 'customAnnoSubmitButton',
+
+    checkCustomAnnotationRequest: (tid, customAnnoButton) => {
+        setTimeout(timeoutCallback, 1000);
+        function timeoutCallback() {
+            $.get(`/custom_annotation/check/task/${tid}`).done((data) => {
+                if (data.status == "started" || data.status == "queued") {
+                    let progress = Math.round(data.progress) || 0;
+                    customAnnoButton.text(`Cancel custom annotation (${progress}%)`);
+                    setTimeout(timeoutCallback, 1000);
+                }
+                else {
+                    customAnnoButton.text("Run custom annotation");
+                    customAnnoButton.removeClass("customAnnotationProcess");
+                }
+            }).fail((data) => {
+                let message = `Error was occured during check annotation status. ` +
+                `Code: ${data.status}, text: ${data.responseText || data.statusText}`;
+                badResponse(message);
+            });
+        }
+    },
+};
+
+document.addEventListener("DOMContentLoaded", () => {
+    $(`<div id="${window.cvat.custom_annotation.modalWindowId}" class="modal hidden">
+        <form id="${window.cvat.custom_annotation.customAnnoFromId}" class="modal-content" autocomplete="on" onsubmit="return false" style="width: 700px;">
+            <center>
+                <label class="semiBold h1"> Custom annotation setup </label>
+            </center>
+
+            <table style="width: 100%; text-align: left;">
+                <tr>
+                    <td style="width: 25%"> <label class="regular h2"> Model </label> </td>
+                    <td> <input id="${window.cvat.custom_annotation.customAnnoModelFieldId}" type="file" name="model" /> </td>
+                </tr>
+                <tr>
+                    <td style="width: 25%"> <label class="regular h2"> Weights </label> </td>
+                    <td> <input id="${window.cvat.custom_annotation.customAnnoWeightsFieldId}" type="file" name="weights" /> </td>
+                </tr>
+                <tr>
+                    <td style="width: 25%"> <label class="regular h2"> Config </label> </td>
+                    <td> <input id="${window.cvat.custom_annotation.customAnnoConfigFieldId}" type="file" name="config" accept=".json" /> </td>
+                </tr>
+                <tr>
+                    <td style="width: 25%"> <label class="regular h2"> Convertation script </label> </td>
+                    <td> <input id="${window.cvat.custom_annotation.customAnnoConvertFieldId}" type="file" name="convert" /> </td>
+                </tr>
+            </table>
+            <div>
+                <button id="${window.cvat.custom_annotation.customAnnoCloseButtonId}" class="regular h2"> Close </button>
+                <button id="${window.cvat.custom_annotation.customAnnoSubmitButtonId}" class="regular h2"> Submit </button>
+            </div>
+        </form>
+
+    </div>`).appendTo('body');
+
+   let annoWindow = $(`#${window.cvat.custom_annotation.modalWindowId}`);
+   let closeWindowButton = $(`#${window.cvat.custom_annotation.customAnnoCloseButtonId}`);
+   let submitButton = $(`#${window.cvat.custom_annotation.customAnnoSubmitButtonId}`);
+
+   closeWindowButton.on('click', () => {
+        annoWindow.addClass('hidden');
+    });
+
+    submitButton.on('click', function() {
+        const tid = annoWindow.attr('current_tid');
+        const modelInput = $(`#${window.cvat.custom_annotation.customAnnoModelFieldId}`);
+        const weightsInput = $(`#${window.cvat.custom_annotation.customAnnoWeightsFieldId}`);
+        const configInput = $(`#${window.cvat.custom_annotation.customAnnoConfigFieldId}`);
+        const convFileInput = $(`#${window.cvat.custom_annotation.customAnnoConvertFieldId}`);
+
+        const modelFile = modelInput.prop('files')[0];
+        const weightsFile = weightsInput.prop('files')[0];
+        const configFile = configInput.prop('files')[0];
+        const convFile = convFileInput.prop('files')[0];
+
+        if (!modelFile || !weightsFile || !configFile || !convFile) {
+            showMessage("All files must be selected");
+            return;
+        }
+
+        let taskData = new FormData();
+        taskData.append('model', modelFile);
+        taskData.append('weights', weightsFile);
+        taskData.append('config', configFile);
+        taskData.append('conv_script', convFile);
+
+        $.ajax({
+            url: `/custom_annotation/create/task/${tid}`,
+            type: 'POST',
+            data: taskData,
+            contentType: false,
+            processData: false,
+        }).done(() => {
+                annoWindow.addClass('hidden');
+                const customAnnoButton = $(`#dashboardTask_${tid} div.dashboardButtonsUI button.dashboardCustomAnno`);
+                customAnnoButton.addClass('customAnnotationProcess');
+                window.cvat.custom_annotation.checkCustomAnnotationRequest(tid, customAnnoButton);
+        }).fail((data) => {
+            let message = `Error was occured during run annotation request. ` +
+                `Code: ${data.status}, text: ${data.responseText || data.statusText}`;
+            badResponse(message);
+        });
+
+        function badResponse(message) {
+            showMessage(message);
+            throw Error(message);
+        }
+    });
+});