Merge branch 'develop' into logstash_pq

cvat-ai · Feb 15, 2021 · 86b1a4e · 86b1a4e
2 parents 6d9d995 + e382313
commit 86b1a4e
Show file tree

Hide file tree

Showing 113 changed files with 1,387 additions and 970 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -23,16 +23,16 @@ jobs:
           DJANGO_SU_EMAIL: '[email protected]'
           DJANGO_SU_PASSWORD: '12qwaszx'
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.ci.yml build
-          docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
-          docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
           docker-compose up -d
           docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
       - name: Code instrumentation
         run: |
           npm ci
           npm run coverage
-          docker-compose up -d --build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
       - name: End-to-end testing
         run: |
           cd ./tests

diff --git a/.github/workflows/publish_docker_images.yml b/.github/workflows/publish_docker_images.yml
@@ -0,0 +1,54 @@
+name: Publish Docker images
+on:
+  release:
+    types: [published]
+
+jobs:
+  build_and_push_to_registry:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+            node-version: 12
+
+      - name: Build images
+        run: |
+          CLAM_AV=yes INSTALL_SOURCES=yes docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
+
+      - name: Run unit tests
+        env:
+          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
+          CONTAINER_COVERAGE_DATA_DIR: '/coverage_data'
+          DJANGO_SU_NAME: 'admin'
+          DJANGO_SU_EMAIL: '[email protected]'
+          DJANGO_SU_PASSWORD: '12qwaszx'
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test'
+          docker-compose up -d
+          docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+
+      - name: Run end-to-end tests
+        run: |
+          cd ./tests
+          npm ci
+          npx cypress run --headless --browser chrome
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push to Docker Hub
+        env:
+          DOCKERHUB_WORKSPACE: 'openvino'
+          SERVER_IMAGE_REPO: 'cvat_server'
+          UI_IMAGE_REPO: 'cvat_ui'
+        run: |
+          docker tag "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest"
+
+          docker tag "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest"
diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml
@@ -18,7 +18,7 @@ jobs:
           DJANGO_SU_PASSWORD: "12qwaszx"
           API_ABOUT_PAGE: "localhost:8080/api/v1/server/about"
         run: |
-          docker-compose -f docker-compose.yml -f ./tests/docker-compose.email.yml up -d --build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f ./tests/docker-compose.email.yml up -d --build
           /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
           docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
       - name: End-to-end testing

diff --git a/.remarkrc.js b/.remarkrc.js
@@ -3,7 +3,7 @@ exports.settings = { bullet: '*', paddedTable: false };
 exports.plugins = [
     'remark-preset-lint-recommended',
     'remark-preset-lint-consistent',
-    ['remark-preset-lint-markdown-style-guide', 'mixed'],
+    ['remark-lint-list-item-indent', 'space'],
     ['remark-lint-no-dead-urls', { skipOffline: true }],
     ['remark-lint-maximum-line-length', 120],
     ['remark-lint-maximum-heading-length', 120],

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -42,8 +42,11 @@
             "request": "launch",
             "stopOnEntry": false,
             "justMyCode": false,
-            "pythonPath": "${command:python.interpreterPath}",
+            "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
+            "env": {
+                "CVAT_SERVERLESS":  "1",
+            },
             "args": [
                 "runserver",
                 "--noreload",
@@ -73,7 +76,7 @@
             "request": "launch",
             "stopOnEntry": false,
             "justMyCode": false,
-            "pythonPath": "${command:python.interpreterPath}",
+            "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
                 "rqworker",
@@ -92,7 +95,7 @@
             "request": "launch",
             "stopOnEntry": false,
             "justMyCode": false,
-            "pythonPath": "${command:python.interpreterPath}",
+            "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
                 "rqscheduler",
@@ -108,7 +111,7 @@
             "request": "launch",
             "justMyCode": false,
             "stopOnEntry": false,
-            "pythonPath":"${command:python.interpreterPath}",
+            "python":"${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
                 "rqworker",
@@ -127,7 +130,7 @@
             "request": "launch",
             "justMyCode": false,
             "stopOnEntry": false,
-            "pythonPath": "${command:python.interpreterPath}",
+            "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
                 "update_git_states"
@@ -143,7 +146,7 @@
             "request": "launch",
             "justMyCode": false,
             "stopOnEntry": false,
-            "pythonPath": "${command:python.interpreterPath}",
+            "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
                 "migrate"
@@ -159,7 +162,7 @@
             "request": "launch",
             "justMyCode": false,
             "stopOnEntry": false,
-            "pythonPath": "${command:python.interpreterPath}",
+            "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "args": [
                 "test",

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,11 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - CVAT-3D: Load all frames corresponding to the job instance
   (<https://github.com/openvinotoolkit/cvat/pull/2645>)
 - Intelligent scissors with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/2689>)
+- [Inside Outside Guidence](https://github.com/shiyinzhang/Inside-Outside-Guidance) serverless
+  function for interative segmentation
+- Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and
+  [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub (<https://github.com/openvinotoolkit/cvat/pull/2766>)
 
 ### Changed
 
 - Updated HTTPS install README section (cleanup and described more robust deploy)
 - Logstash is improved for using with configurable elasticsearch outputs (<https://github.com/openvinotoolkit/cvat/pull/2531>)
+- Bumped nuclio version to 1.5.16
+- All methods for interative segmentation accept negative points as well
 - Persistent queue added to logstash (<https://github.com/openvinotoolkit/cvat/pull/2744>)
 
 ### Deprecated
@@ -40,6 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A couple of css-related issues (top bar disappear, wrong arrow position on collapse elements) (<https://github.com/openvinotoolkit/cvat/pull/2736>)
 - Issue with point region doesn't work in Firefox (<https://github.com/openvinotoolkit/cvat/pull/2727>)
 - Fixed cuboid perpective change (<https://github.com/openvinotoolkit/cvat/pull/2733>)
+- Annotation page popups (ai tools, drawing) reset state after detecting, tracking, drawing (<https://github.com/openvinotoolkit/cvat/pull/2780>)
 
 ### Security
 

diff --git a/Dockerfile b/Dockerfile
@@ -34,10 +34,12 @@ RUN curl -sL https://github.com/cisco/openh264/archive/v${OPENH264_VERSION}.tar.
     make -j5 && make install PREFIX=${PREFIX} && make clean
 
 WORKDIR /tmp/ffmpeg
-RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
-    tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+RUN curl -sL https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 --output - | \
+    tar -jx --strip-components=1 && \
     ./configure --disable-nonfree --disable-gpl --enable-libopenh264 --enable-shared --disable-static --prefix="${PREFIX}" && \
-    make -j5 && make install && make distclean
+    # make clean keeps the configuration files that let to know how the original sources were used to create the binary
+    make -j5 && make install && make clean && \
+    tar -zcf "/tmp/ffmpeg-$FFMPEG_VERSION.tar.gz" . && mv "/tmp/ffmpeg-$FFMPEG_VERSION.tar.gz" .
 
 # Install requirements
 RUN python3 -m venv /opt/venv
@@ -93,8 +95,17 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     echo 'application/wasm wasm' >> /etc/mime.types
 
-ARG CLAM_AV
-ENV CLAM_AV=${CLAM_AV}
+# Add a non-root user
+ENV USER=${USER}
+ENV HOME /home/${USER}
+RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \
+    if [ -z ${socks_proxy} ]; then \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
+    else \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
+    fi
+
+ARG CLAM_AV="no"
 RUN if [ "$CLAM_AV" = "yes" ]; then \
         apt-get update && \
         apt-get --no-install-recommends install -yq \
@@ -106,16 +117,6 @@ RUN if [ "$CLAM_AV" = "yes" ]; then \
         rm -rf /var/lib/apt/lists/*; \
     fi
 
-# Add a non-root user
-ENV USER=${USER}
-ENV HOME /home/${USER}
-RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \
-    if [ -z ${socks_proxy} ]; then \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
-    else \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
-    fi
-
 ARG INSTALL_SOURCES='no'
 WORKDIR ${HOME}/sources
 RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \
@@ -131,7 +132,7 @@ RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \
             done &&                                           \
         rm -rf /var/lib/apt/lists/*;                          \
     fi
-COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.bz2 ${HOME}/sources/
+COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.gz ${HOME}/sources/
 
 # Copy python virtual enviroment and FFmpeg binaries from build-image
 COPY --from=build-image /opt/venv /opt/venv

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -1,4 +1,4 @@
-FROM cvat/server
+FROM openvino/cvat_server
 
 ENV DJANGO_CONFIGURATION=testing
 USER root

diff --git a/LICENSE b/LICENSE
@@ -20,3 +20,12 @@ OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
 OR OTHER DEALINGS IN THE SOFTWARE.
 
+This software uses LGPL licensed libraries from the [FFmpeg](https://www.ffmpeg.org) project.
+The exact steps on how FFmpeg was configured and compiled can be found in the [Dockerfile](Dockerfile).
+
+FFmpeg is an open source framework licensed under LGPL and GPL.
+See https://www.ffmpeg.org/legal.html. You are solely responsible
+for determining if your use of FFmpeg requires any
+additional licenses. Intel is not responsible for obtaining any
+such licenses, nor liable for any licensing fees due in
+connection with your use of FFmpeg.
diff --git a/README.md b/README.md
@@ -63,18 +63,26 @@ For more information about supported formats look at the
 | [ImageNet](http://www.image-net.org)                                          | X      | X      |
 | [CamVid](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/)          | X      | X      |
 
-## Deep learning models for automatic labeling
+## Deep learning serverless functions for automatic labeling
+
+<!--lint disable maximum-line-length-->
 
 | Name                                                                                                    | Type       | Framework  | CPU | GPU |
 | ------------------------------------------------------------------------------------------------------- | ---------- | ---------- | --- | --- |
-| [Deep Extreme Cut](/serverless/openvino/dextr/nuclio)                                                   | interactor | OpenVINO   | X   |
+| [Deep Extreme Cut](/serverless/openvino/dextr/nuclio)                                                   | interactor | OpenVINO   | X   |     |
+| [Faster RCNN](/serverless/openvino/omz/public/faster_rcnn_inception_v2_coco/nuclio)                     | detector   | OpenVINO   | X   |     |
+| [Mask RCNN](/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio)           | detector   | OpenVINO   | X   |     |
+| [YOLO v3](/serverless/openvino/omz/public/yolo-v3-tf/nuclio)                                            | detector   | OpenVINO   | X   |     |
+| [Object reidentification](/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio)     | reid       | OpenVINO   | X   |     |
+| [Semantic segmentation for ADAS](/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio) | detector   | OpenVINO   | X   |     |
+| [Text detection v4](/serverless/openvino/omz/intel/text-detection-0004/nuclio)                          | detector   | OpenVINO   | X   |     |
+| [SiamMask](/serverless/pytorch/foolwood/siammask/nuclio)                                                | tracker    | PyTorch    | X   |     |
+| [f-BRS](/serverless/pytorch/saic-vul/fbrs/nuclio)                                                       | interactor | PyTorch    | X   |     |
+| [Inside-Outside Guidance](/serverless/pytorch/shiyinzhang/iog/nuclio)                                   | interactor | PyTorch    | X   |     |
 | [Faster RCNN](/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio)                              | detector   | TensorFlow | X   | X   |
-| [Mask RCNN](/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio)           | detector   | OpenVINO   | X   |
-| [YOLO v3](/serverless/openvino/omz/public/yolo-v3-tf/nuclio)                                            | detector   | OpenVINO   | X   |
-| [Text detection v4](/serverless/openvino/omz/intel/text-detection-0004/nuclio)                          | detector   | OpenVINO   | X   |
-| [Semantic segmentation for ADAS](/serverless/openvino/omz/intel/semantic-segmentation-adas-0001/nuclio) | detector   | OpenVINO   | X   |
-| [Mask RCNN](/serverless/tensorflow/matterport/mask_rcnn/nuclio)                                         | detector   | TensorFlow | X   |
-| [Object reidentification](/serverless/openvino/omz/intel/person-reidentification-retail-300/nuclio)     | reid       | OpenVINO   | X   |
+| [Mask RCNN](/serverless/tensorflow/matterport/mask_rcnn/nuclio)                                         | detector   | TensorFlow | X   |     |
+
+<!--lint enable maximum-line-length-->
 
 ## Online demo: [cvat.org](https://cvat.org)
 
@@ -91,18 +99,36 @@ Limitations:
 - No more than 10 tasks per user
 - Uploaded data is limited to 500Mb
 
+## Prebuilt Docker images
+
+Prebuilt docker images for CVAT releases are available on Docker Hub:
+
+- [cvat_server](https://hub.docker.com/r/openvino/cvat_server)
+- [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui)
+
 ## REST API
 
-Automatically generated Swagger documentation for Django REST API is
-available on `<cvat_origin>/api/swagger`
-(default: `localhost:8080/api/swagger`).
+Automatically generated Swagger documentation for Django REST API is available
+on `<cvat_origin>/api/swagger`(default: `localhost:8080/api/swagger`).
 
-Swagger documentation is visiable on allowed hostes, Update environement variable in docker-compose.yml file with cvat hosted machine IP or domain name. Example - `ALLOWED_HOSTS: 'localhost, 127.0.0.1'`)
+Swagger documentation is visiable on allowed hostes, Update environement
+variable in docker-compose.yml file with cvat hosted machine IP or domain
+name. Example - `ALLOWED_HOSTS: 'localhost, 127.0.0.1'`.
 
 ## LICENSE
 
 Code released under the [MIT License](https://opensource.org/licenses/MIT).
 
+This software uses LGPL licensed libraries from the [FFmpeg](https://www.ffmpeg.org) project.
+The exact steps on how FFmpeg was configured and compiled can be found in the [Dockerfile](Dockerfile).
+
+FFmpeg is an open source framework licensed under LGPL and GPL.
+See [https://www.ffmpeg.org/legal.html](https://www.ffmpeg.org/legal.html). You are solely responsible
+for determining if your use of FFmpeg requires any
+additional licenses. Intel is not responsible for obtaining any
+such licenses, nor liable for any licensing fees due in
+connection with your use of FFmpeg.
+
 ## Questions
 
 CVAT usage related questions or unclear concepts can be posted in our
@@ -129,4 +155,6 @@ Other ways to ask questions and get our support:
 
 ## Projects using CVAT
 
-- [Onepanel](https://github.com/onepanelio/core) - Onepanel is an open source vision AI platform that fully integrates CVAT with scalable data processing and parallelized training pipelines.
+- [Onepanel](https://github.com/onepanelio/core) - Onepanel is an open source
+  vision AI platform that fully integrates CVAT with scalable data processing
+  and parallelized training pipelines.
diff --git a/components/serverless/docker-compose.serverless.yml b/components/serverless/docker-compose.serverless.yml
@@ -2,7 +2,7 @@ version: '3.3'
 services:
   serverless:
     container_name: nuclio
-    image: quay.io/nuclio/dashboard:1.5.8-amd64
+    image: quay.io/nuclio/dashboard:1.5.16-amd64
     restart: always
     networks:
       default:
@@ -16,6 +16,7 @@ services:
       https_proxy:
       no_proxy: 172.28.0.1,${no_proxy}
       NUCLIO_CHECK_FUNCTION_CONTAINERS_HEALTHINESS: 'true'
+      NUCLIO_DASHBOARD_DEFAULT_FUNCTION_MOUNT_MODE: 'volume'
     ports:
       - '8070:8070'
 

diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT