Merge pull request openvinotoolkit#8 from openvinotoolkit/master

Update forked branch

.ci/azure/analyze_gtest_log.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Intel Corporation
+# Copyright (C) 2018-2021 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 """

.ci/azure/linux_conditional_compilation.yml

@@ -0,0 +1,87 @@
+jobs:
+- job: LinCC
+  # About 150% of total time
+  timeoutInMinutes: 90
+
+  pool:
+    name: LIN_VMSS_VENV_F16S_WU2
+
+  variables:
+    system.debug: true
+    VSTS_HTTP_RETRY: 5
+    VSTS_HTTP_TIMEOUT: 200
+    WORKERS_NUMBER: 16
+    BUILD_TYPE: Release
+    REPO_DIR: $(Build.Repository.LocalPath)
+    OPENVINO_CONTRIB_REPO_DIR: $(REPO_DIR)/../openvino_contrib
+    MODELS_PATH: $(REPO_DIR)/../testdata
+    WORK_DIR: $(Pipeline.Workspace)/_w
+    BUILD_DIR: $(WORK_DIR)/build
+    BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
+    INSTALL_DIR: $(WORK_DIR)/install_pkg
+    SETUPVARS: $(INSTALL_DIR)/bin/setupvars.sh
+
+  steps:
+  - script: |
+      curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
+      whoami
+      uname -a
+      echo Python3 info ; which python3 ; python3 --version
+      echo Python info ; which python ; python --version
+      echo Java info ; which java ; java -version
+      echo gcc info ; which gcc ; gcc --version
+      lsb_release
+      env
+      cat /proc/cpuinfo
+      cat /proc/meminfo
+      cat /etc/fstab
+      vmstat -s
+      df
+      lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
+      free -h
+    displayName: 'System info'
+
+  - script: |
+      rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
+      rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
+    displayName: 'Make dir'
+
+  - checkout: self
+    clean: true
+    lfs: false
+    submodules: recursive
+    path: openvino
+
+  - script: |
+      sudo apt --assume-yes install libusb-1.0-0-dev
+      python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/requirements.txt
+      # Speed up build
+      wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
+      unzip ninja-linux.zip
+      sudo cp -v ninja /usr/local/bin/
+    workingDirectory: $(WORK_DIR)
+    displayName: 'Install dependencies'
+
+  - task: CMake@1
+    inputs:
+      cmakeArgs: >
+        -GNinja
+        -DVERBOSE_BUILD=ON
+        -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
+        -DENABLE_FASTER_BUILD=ON
+        -DENABLE_PROFILING_ITT=ON
+        -DSELECTIVE_BUILD=COLLECT
+        $(REPO_DIR)
+      workingDirectory: $(BUILD_DIR)
+
+  - script: ninja
+    workingDirectory: $(BUILD_DIR)
+    displayName: 'Build'
+
+  - script: ls -alR $(REPO_DIR)/bin/
+    displayName: 'List files'
+
+  - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
+    workingDirectory: $(BUILD_DIR)
+    displayName: 'Install'
+

.ci/azure/linux_ngraph_onnx.yml

@@ -43,7 +43,7 @@ jobs:
       sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
       sudo mkdir -p $(MODELS_DIR)
       sudo apt --assume-yes install nfs-common
-      sudo mount -v -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
+      sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
     displayName: 'Make dirs'
 
   - checkout: self

.ci/azure/windows_conditional_compilation.yml

@@ -0,0 +1,89 @@
+jobs:
+- job: WinCC
+  # About 150% of total time
+  timeoutInMinutes: 120
+
+  pool:
+    name: WIN_VMSS_VENV_F8S_WU2
+
+  variables:
+    system.debug: true
+    VSTS_HTTP_RETRY: 5
+    VSTS_HTTP_TIMEOUT: 200
+    WORKERS_NUMBER: 8
+    BUILD_TYPE: Release
+    REPO_DIR: $(Build.Repository.LocalPath)
+    OPENVINO_CONTRIB_REPO_DIR: $(REPO_DIR)\..\openvino_contrib
+    MODELS_PATH: $(REPO_DIR)\..\testdata
+    WORK_DIR: $(Pipeline.Workspace)\_w
+    BUILD_DIR: D:\build
+    BIN_DIR: $(REPO_DIR)\bin\intel64
+    MSVS_VARS_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
+    MSVC_COMPILER_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC\14.24.28314\bin\Hostx64\x64\cl.exe
+    INSTALL_DIR: $(WORK_DIR)\install_pkg
+    SETUPVARS: $(INSTALL_DIR)\bin\setupvars.bat
+    IB_DIR: C:\Program Files (x86)\IncrediBuild
+    IB_TESTCONSOLE: $(IB_DIR)\IBTestConsole.exe
+    TEST_ENV_PATH: $(REPO_DIR)\inference-engine\temp\tbb\bin;$(REPO_DIR)\inference-engine\temp\opencv_4.5.1\opencv\bin;$(IB_DIR);%PATH%
+
+  steps:
+  - script: |
+      powershell -command "Invoke-RestMethod -Headers @{\"Metadata\"=\"true\"} -Method GET -Uri http://169.254.169.254/metadata/instance/compute?api-version=2019-06-01 | format-custom"
+      where python3
+      where python
+      python --version
+      where java
+      java -version
+      wmic computersystem get TotalPhysicalMemory
+      wmic cpu list
+      wmic logicaldisk get description,name
+      wmic VOLUME list
+      set
+    displayName: 'System info'
+
+  - script: |
+      rd /Q /S $(WORK_DIR) & mkdir $(WORK_DIR)
+      rd /Q /S $(BUILD_DIR) & mkdir $(BUILD_DIR)
+    displayName: 'Make dir'
+
+  - script: |
+      certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat
+      call install_ib_console.bat
+    workingDirectory: $(WORK_DIR)
+    displayName: 'Install IncrediBuild'
+
+  - checkout: self
+    clean: true
+    lfs: false
+    submodules: recursive
+    path: openvino
+
+  - script: |
+      certutil -urlcache -split -f https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip ninja-win.zip
+      powershell -command "Expand-Archive -Force ninja-win.zip"
+    workingDirectory: $(WORK_DIR)
+    displayName: 'Install dependencies'
+
+  - script: |
+      set PATH=$(WORK_DIR)\ninja-win;%PATH%
+      call "$(MSVS_VARS_PATH)" && cmake -GNinja -DENABLE_FASTER_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_PROFILING_ITT=ON -DSELECTIVE_BUILD=COLLECT -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
+    workingDirectory: $(BUILD_DIR)
+    displayName: 'CMake'
+
+  - script: |
+      set PATH=$(WORK_DIR)\ninja-win;%PATH%
+      call "$(MSVS_VARS_PATH)" && "C:\Program Files (x86)\IncrediBuild\BuildConsole.exe" /COMMAND="ninja"
+    workingDirectory: $(BUILD_DIR)
+    displayName: 'Build Win'
+
+  - script: dir $(REPO_DIR)\bin\ /s
+    displayName: 'List files'
+
+  - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
+    workingDirectory: $(BUILD_DIR)
+    displayName: 'Install'
+
+  - script: echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent
+    displayName: Stop IncrediBuild
+    continueOnError: true
+    enabled: false

.ci/openvino-onnx/Dockerfile

@@ -10,6 +10,7 @@ ENV https_proxy ${https_proxy}
 ENV CI=true
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED 1
+ARG PROTOBUF_LITE=OFF
 
 # Install base dependencies
 RUN apt-get update && apt-get install -y locales && apt-get clean autoclean && apt-get autoremove -y
@@ -68,14 +69,13 @@ RUN cmake .. \
     -DNGRAPH_INTERPRETER_ENABLE=ON \
     -DNGRAPH_DEBUG_ENABLE=OFF \
     -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON \
-    -DCMAKE_INSTALL_PREFIX=/openvino/dist
+    -DCMAKE_INSTALL_PREFIX=/openvino/dist \
+    -DNGRAPH_USE_PROTOBUF_LITE=${PROTOBUF_LITE}
 RUN make -j $(nproc) install
 
 # Run tests via tox
 WORKDIR /openvino/ngraph/python
 ENV NGRAPH_CPP_BUILD_PATH=/openvino/dist/deployment_tools/ngraph
 ENV LD_LIBRARY_PATH=/openvino/dist/deployment_tools/ngraph/lib
-ENV NGRAPH_ONNX_IMPORT_ENABLE=TRUE
 ENV PYTHONPATH=/openvino/bin/intel64/Release/lib/python_api/python3.8:${PYTHONPATH}
-RUN git clone --recursive https://github.com/pybind/pybind11.git -b v2.5.0 --depth 1
 CMD tox

.ci/openvino-onnx/Jenkinsfile

@@ -4,11 +4,16 @@
 DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container"
 DOCKER_IMAGE_TAG = "openvino-onnx-ci-image"
 
+BACKEND_CONFIGURATIONS = [
+    [ protobuf_lite : "ON" ],
+    [ protobuf_lite : "OFF" ]
+]
+
 // workaround for aborting previous builds on PR update
 @NonCPS
  def stopPreviousRunningBuilds() {
      def jobname = env.JOB_NAME
-     if (jobname.startsWith("onnx/openvino_ci/PR")){
+     if (jobname.startsWith("onnx-ci/openvino onnx ci/openvino/PR")){
          def buildnum = env.BUILD_NUMBER.toInteger()
          def job = Jenkins.instance.getItemByFullName(jobname)
          def job_newest = job.builds.first()
@@ -77,16 +82,24 @@ def gitSubmoduleUpdate(String repository_name) {
     }
 }
 
+def prepare_repository() {
+    dir("${WORKDIR}") {
+        checkout scm
+        gitSubmoduleUpdate(PROJECT_NAME)
+    }
+}
+
 def updateModels() {
     sh """
         ./ngraph/python/tests/test_onnx/model_zoo_preprocess.sh -d ${HOME}/ONNX_CI/data -o
     """
 }
 
-def buildDockerImage() {
+def buildDockerImage(String protobuf_lite="OFF") {
     updateModels()
     sh """
-        docker build --tag=${DOCKER_IMAGE_TAG}  --file=.ci/openvino-onnx/Dockerfile \
+        docker build --tag=${DOCKER_IMAGE_TAG} --build-arg PROTOBUF_LITE=${protobuf_lite} \
+        --file=.ci/openvino-onnx/Dockerfile \
         --build-arg http_proxy=http://proxy-chain.intel.com:911/ \
         --build-arg https_proxy=http://proxy-chain.intel.com:912/ .
     """
@@ -100,54 +113,57 @@ def runTests() {
     """
 }
 
-
-pipeline {
-    agent {
-        label "OpenVino"
-    }
-    environment {
-        PROJECT_NAME = "openvino"
-        WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
-    }
-    options {
-        skipDefaultCheckout true
-        timeout(activity: true, time: 60, unit: 'MINUTES')
+def getConfigurationsMap() {
+    def configurationsMap = [:]
+    for (backend in BACKEND_CONFIGURATIONS) {
+        def configuration = backend.clone()
+        configuration.name = "protobuf-lite ${configuration.protobuf_lite}"
+        configurationsMap[configuration.name] = {
+            stage(configuration.name) { CONFIGURATION_WORKFLOW(configuration) }
+        }
     }
-    stages {
-        stage("Clone repository") {
-            steps{
+    return configurationsMap
+}
+
+CONFIGURATION_WORKFLOW = { configuration ->
+    node("OpenVino") {
+        try {
+            PROJECT_NAME = "openvino"
+            WORKDIR = "${HOME}/workspace/${BUILD_NUMBER}"
+
+            stage("Clone repository") {
                 stopPreviousRunningBuilds()
-                dir("${WORKDIR}") {
-                    checkout scm
+                try {
+                    prepare_repository()
+                }
+                catch(e){
+                    sleep(time: 30, unit: "SECONDS")
+                    prepare_repository()
                 }
-                gitSubmoduleUpdate(PROJECT_NAME)
             }
-        }
-        stage("Prepare Docker environment") {
-            steps{
+            stage("Prepare Docker environment") {
               dir("${WORKDIR}") {
-                    buildDockerImage()
+                    buildDockerImage(configuration.protobuf_lite)
                 }
             }
-        }
-        stage("Run tests") {
-            options {
-                  timeout(time: 60, unit: 'MINUTES')
-              }
-            steps{
-                runTests()
+            stage("Run tests") {
+                timeout(time: 20, unit: 'MINUTES') {
+                    runTests()
+                }
             }
         }
-    }
-    post {
-        failure {
-            script {
+        catch(e) {
+                // Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
+                if ("$e".contains("143")) {
+                    currentBuild.result = "ABORTED"
+                } else {
+                    currentBuild.result = "FAILURE"
+                }
                 gitPrInfo = getGitPrInfo(PROJECT_NAME)
                 notifyByEmail(gitPrInfo)
-            }
         }
-        cleanup {
-            dir("${WORKDIR}") {
+        finally {
+            stage("Cleanup") {
                 deleteDir()
                 sh """
                     docker image prune -f
@@ -157,3 +173,22 @@ pipeline {
         }
     }
 }
+
+pipeline {
+    agent none
+    options {
+        skipDefaultCheckout true
+        timeout(activity: true, time: 60, unit: 'MINUTES')
+    }
+    stages {
+        stage('Parallel CI') {
+            steps {
+                script {
+                    parallelStagesMap = getConfigurationsMap()
+                    parallel parallelStagesMap
+                }
+            }
+
+        }
+    }
+}

.ci/openvino-onnx/watchdog/src/git_wrapper.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-# Copyright (C) 2018-2020 Intel Corporation
+# Copyright (C) 2018-2021 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import logging

.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-# Copyright (C) 2018-2020 Intel Corporation
+# Copyright (C) 2018-2021 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import requests