build ubuntu: use install_openvino_deps script 2021.3R

.github/workflows/codestyle.yml

@@ -24,15 +24,15 @@ jobs:
 
       - name: Running Bandit
         if: ${{ matrix.os == 'ubuntu-18.04' }}
-        run: python -m bandit -r ./ -f screen |& tee bandit.log
+        run: python -m bandit -r ./ -f screen
 
       - name: Running PEP checks
         if: ${{ matrix.os == 'ubuntu-18.04' }}
-        run: python -m flake8 ./ --config=setup.cfg --show-source |& tee flake8.log
+        run: python -m flake8 ./ --config=setup.cfg --show-source
 
       - name: Running MyPy checks
         if: ${{ matrix.os == 'ubuntu-18.04' }}
-        run: python -m mypy ./ --config-file ./setup.cfg --show-error-context --show-column-numbers --pretty |& tee mypy.log
+        run: python -m mypy ./ --config-file ./setup.cfg --show-error-context --show-column-numbers --pretty
 
       - name: Running pytest (not Docker image tests)
         if: ${{ always() }}
@@ -44,7 +44,4 @@ jobs:
         with:
           name: codestyle_checks
           path: |
-            ./bandit.log
-            ./flake8.log
-            ./mypy.log
             ./utils_unittests.html

dockerfiles/ubuntu18/build_custom/Dockerfile

@@ -4,20 +4,18 @@ FROM ubuntu:18.04 AS setup_openvino
 
 # hadolint ignore=DL3008
 RUN apt-get update; \
+    DEBIAN_FRONTEND=noninteractive \
     apt-get install -y --no-install-recommends \
+        tzdata \
         git \
         git-lfs \
         sudo \
-        curl \
         python3-dev \
         python3-pip \
         python3-venv \
         python3-setuptools; \
     rm -rf /var/lib/apt/lists/*
 
-# Set location
-RUN ln -snf "/usr/share/zoneinfo/$(curl https://ipapi.co/timezone -k)" /etc/localtime
-
 ARG openvino_branch="master"
 
 WORKDIR /tmp
@@ -75,8 +73,9 @@ FROM copy_openvino AS openvino
 
 # hadolint ignore=DL3008
 RUN apt-get update; \
+    DEBIAN_FRONTEND=noninteractive \
     apt-get install -y --no-install-recommends \
-        curl \
+        tzdata \
         wget \
         udev \
         sudo \
@@ -86,9 +85,6 @@ RUN apt-get update; \
         python3-setuptools; \
     rm -rf /var/lib/apt/lists/*
 
-# Set location
-RUN ln -snf "/usr/share/zoneinfo/$(curl https://ipapi.co/timezone -k)" /etc/localtime
-
 WORKDIR ${INTEL_OPENVINO_DIR}
 RUN chmod +x install_dependencies/install_openvino_dependencies.sh; \
     ./install_dependencies/install_openvino_dependencies.sh

dockerfiles/ubuntu18/build_custom/README.md

@@ -31,7 +31,7 @@ You can manually setup cmake parameters to build a custom package from source co
 * [openvino_cmake.txt](openvino_cmake.txt)
 * [opencv_cmake.txt](opencv_cmake.txt)
 
-**Notes**:  
+>**Note**:  
 By default, these files already contain some parameters.  
 Do not override PATH/PREFIX options. This can break a build of package.
 
@@ -65,13 +65,12 @@ docker build --target opencv -t ie:opencv .
 ## How to test
 You can use our default pipeline to test your image:
 ```
-python3 docker_openvino.py test -t [image:tag] -dist dev -p 2021.2 
+python3 docker_openvino.py test -t [image:tag] -dist custom 
 ```
 
-If Open Model Zoo is not contained in your image:
-```
-python3 docker_openvino.py test -t [image:tag] -dist runtime -u [package_url] 
-```
+>**Note**:  
+Docker_ci automatically runs the corresponding tests.  
+By default, the product version is equal to the latest release version. Use `-p` to override this.
 
 ## How to run
 Please follow [Run built image](../get-started.md#run-built-image) section in DockerHub CI getting started guide.

dockerfiles/ubuntu18/openvino_cgvh_data_dev_2021.3.dockerfile

@@ -0,0 +1,291 @@
+# Copyright (C) 2019-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+FROM ubuntu:18.04 AS base
+
+# hadolint ignore=DL3002
+USER root
+WORKDIR /
+
+SHELL ["/bin/bash", "-xo", "pipefail", "-c"]
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl tzdata ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# download source for pypi-kenlm LGPL package
+WORKDIR /tmp
+RUN curl -L https://files.pythonhosted.org/packages/7f/e6/1639d2de28c27632e3136015ecfd67774cca6f55146507baeaef06b113ba/pypi-kenlm-0.1.20190403.tar.gz --output pypi-kenlm.tar.gz
+
+
+# get product from URL
+ARG package_url
+ARG TEMP_DIR=/tmp/openvino_installer
+
+WORKDIR ${TEMP_DIR}
+# hadolint ignore=DL3020
+ADD ${package_url} ${TEMP_DIR}
+
+# install product by copying archive content
+ARG TEMP_DIR=/tmp/openvino_installer
+ENV INTEL_OPENVINO_DIR /opt/intel/openvino
+
+RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
+    OV_BUILD="$(find . -maxdepth 1 -type d -name "*openvino*" | grep -oP '(?<=_)\d+.\d+.\d+')" && \
+    OV_YEAR="$(find . -maxdepth 1 -type d -name "*openvino*" | grep -oP '(?<=_)\d+')" && \
+    OV_FOLDER="$(find . -maxdepth 1 -type d -name "*openvino*")" && \
+    mkdir -p /opt/intel/openvino_"$OV_BUILD"/ && \
+    cp -rf "$OV_FOLDER"/*  /opt/intel/openvino_"$OV_BUILD"/ && \
+    rm -rf "${TEMP_DIR:?}"/"$OV_FOLDER" && \
+    ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino && \
+    ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino_"$OV_YEAR" && \
+    rm -rf ${INTEL_OPENVINO_DIR}/deployment_tools/tools/workbench && rm -rf ${TEMP_DIR}
+
+
+
+# for GPU
+ARG GMMLIB
+ARG IGC_CORE
+ARG IGC_OPENCL
+ARG INTEL_OPENCL
+ARG INTEL_OCLOC
+ARG TEMP_DIR=/tmp/opencl
+
+WORKDIR ${TEMP_DIR}
+RUN curl -L "https://github.com/intel/compute-runtime/releases/download/${INTEL_OPENCL}/intel-gmmlib_${GMMLIB}_amd64.deb" --output "intel-gmmlib_${GMMLIB}_amd64.deb" && \
+    curl -L "https://github.com/intel/compute-runtime/releases/download/${INTEL_OPENCL}/intel-igc-core_${IGC_CORE}_amd64.deb" --output "intel-igc-core_${IGC_CORE}_amd64.deb" && \
+    curl -L "https://github.com/intel/compute-runtime/releases/download/${INTEL_OPENCL}/intel-igc-opencl_${IGC_OPENCL}_amd64.deb" --output "intel-igc-opencl_${IGC_OPENCL}_amd64.deb" && \
+    curl -L "https://github.com/intel/compute-runtime/releases/download/${INTEL_OPENCL}/intel-opencl_${INTEL_OPENCL}_amd64.deb" --output "intel-opencl_${INTEL_OPENCL}_amd64.deb" && \
+    curl -L "https://github.com/intel/compute-runtime/releases/download/${INTEL_OPENCL}/intel-ocloc_${INTEL_OCLOC}_amd64.deb" --output "intel-ocloc_${INTEL_OCLOC}_amd64.deb"
+
+
+# for VPU
+ARG BUILD_DEPENDENCIES="autoconf \
+                        automake \
+                        build-essential \
+                        libtool \
+                        unzip"
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${BUILD_DEPENDENCIES} && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+RUN curl -L https://github.com/libusb/libusb/archive/v1.0.22.zip --output v1.0.22.zip && \
+    unzip v1.0.22.zip && rm -rf v1.0.22.zip
+
+WORKDIR /opt/libusb-1.0.22
+RUN ./bootstrap.sh && \
+    ./configure --disable-udev --enable-shared && \
+    make -j4
+
+# -----------------
+FROM ubuntu:18.04 AS ov_base
+
+LABEL Description="This is the data_dev image for Intel(R) Distribution of OpenVINO(TM) toolkit on Ubuntu 18.04 LTS"
+LABEL Vendor="Intel Corporation"
+
+USER root
+WORKDIR /
+
+SHELL ["/bin/bash", "-xo", "pipefail", "-c"]
+
+# Creating user openvino and adding it to groups "video" and "users" to use GPU and VPU
+RUN useradd -ms /bin/bash -G video,users openvino && \
+    chown openvino -R /home/openvino
+
+RUN mkdir /opt/intel
+
+ENV INTEL_OPENVINO_DIR /opt/intel/openvino
+
+COPY --from=base /opt/intel /opt/intel
+
+WORKDIR /thirdparty
+
+ARG INSTALL_SOURCES="yes"
+
+
+ARG DEPS="dpkg-dev \
+          tzdata \
+          curl \
+          libopenexr22 \
+          flex"
+ARG LGPL_DEPS="g++ \
+               gcc \
+               libc6-dev \
+               libgtk-3-0 \
+               libgstreamer1.0-0 \
+               gstreamer1.0-plugins-base \
+               gstreamer1.0-plugins-good \
+               gstreamer1.0-plugins-bad \
+               gstreamer1.0-vaapi \
+               ffmpeg \
+               libgl-dev \
+               libtag-extras1 \
+               libfaac0 \
+               python3-gi \
+               libfluidsynth1 \
+               libnettle6 \
+               gstreamer1.0-plugins-ugly \
+               gstreamer1.0-alsa \
+               libglib2.0"
+ARG INSTALL_PACKAGES="-c=opencv_req -c=python -c=opencv_opt -c=dlstreamer"
+
+
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ${DEPS} && \
+    dpkg --get-selections | grep -v deinstall | awk '{print $1}' > base_packages.txt  && \
+    rm -rf /var/lib/apt/lists/*
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${LGPL_DEPS} && \
+    ${INTEL_OPENVINO_DIR}/install_dependencies/install_openvino_dependencies.sh -y ${INSTALL_PACKAGES} && \
+    if [ "$INSTALL_SOURCES" = "yes" ]; then \
+      sed -Ei 's/# deb-src /deb-src /' /etc/apt/sources.list && \
+      apt-get update && \
+	  dpkg --get-selections | grep -v deinstall | awk '{print $1}' > all_packages.txt && \
+	  grep -v -f base_packages.txt all_packages.txt | while read line; do \
+	  package=`echo $line`; \
+	  name=(${package//:/ }); \
+      grep -l GPL /usr/share/doc/${name[0]}/copyright; \
+      exit_status=$?; \
+	  if [ $exit_status -eq 0 ]; then \
+	    apt-get source -q --download-only $package;  \
+	  fi \
+      done && \
+      echo "Download source for `ls | wc -l` third-party packages: `du -sh`"; fi && \
+    rm -rf /var/lib/apt/lists/* && rm -rf *.txt
+
+
+
+# setup Python
+ENV PYTHON_VER python3.6
+
+
+
+RUN ${PYTHON_VER} -m pip install --upgrade pip
+
+# data dev package
+WORKDIR /tmp
+
+RUN ${PYTHON_VER} -m pip install --no-cache-dir cmake && \
+    ${PYTHON_VER} -m pip install --no-cache-dir -r ${INTEL_OPENVINO_DIR}/python/${PYTHON_VER}/requirements.txt && \
+    find "${INTEL_OPENVINO_DIR}/" -type f \( -name "*requirements.*" -o  -name "*requirements_ubuntu18.*" -o \( -name "*requirements*.in" -and -not -name "*requirements-tensorflow.in" \) \) -not -path "*/accuracy_checker/*" -not -path "*/post_training_optimization_toolkit/*" -not -path "*/python3*/*" -not -path "*/python2*/*" -print -exec ${PYTHON_VER} -m pip install --no-cache-dir -r "{}" \;
+
+
+ENV VENV_TF2 /opt/intel/venv_tf2
+
+RUN ${PYTHON_VER} -m venv ${VENV_TF2} && \
+    source ${VENV_TF2}/bin/activate && \
+    pip install --no-cache-dir -U pip==19.3.1 && \
+    find "${INTEL_OPENVINO_DIR}/deployment_tools/model_optimizer/" -type f \( -name "*requirements*.txt" -and -not -name "*requirements_tf.txt" \) -print -exec ${PYTHON_VER} -m pip install --no-cache-dir -r "{}" \; && \
+    find "${INTEL_OPENVINO_DIR}/deployment_tools/open_model_zoo/tools/downloader/" -type f -name "*requirements*.in" -print -exec ${PYTHON_VER} -m pip install --no-cache-dir -r "{}" \; && \
+    deactivate
+
+
+WORKDIR ${INTEL_OPENVINO_DIR}/deployment_tools/open_model_zoo/tools/accuracy_checker
+RUN source ${INTEL_OPENVINO_DIR}/bin/setupvars.sh && \
+    ${PYTHON_VER} -m pip install --no-cache-dir -r ${INTEL_OPENVINO_DIR}/deployment_tools/open_model_zoo/tools/accuracy_checker/requirements.in && \
+    ${PYTHON_VER} ${INTEL_OPENVINO_DIR}/deployment_tools/open_model_zoo/tools/accuracy_checker/setup.py install && \
+    rm -rf ${INTEL_OPENVINO_DIR}/deployment_tools/open_model_zoo/tools/accuracy_checker/build
+
+# download source for pypi-kenlm LGPL package
+COPY --from=base /tmp/pypi-kenlm.tar.gz /thirdparty/pypi-kenlm.tar.gz
+
+WORKDIR ${INTEL_OPENVINO_DIR}/deployment_tools/tools/post_training_optimization_toolkit
+RUN ${PYTHON_VER} -m pip install --no-cache-dir -r ${INTEL_OPENVINO_DIR}/deployment_tools/tools/post_training_optimization_toolkit/requirements.txt && \
+    ${PYTHON_VER} ${INTEL_OPENVINO_DIR}/deployment_tools/tools/post_training_optimization_toolkit/setup.py install && \
+    rm -rf ${INTEL_OPENVINO_DIR}/deployment_tools/tools/post_training_optimization_toolkit/build
+
+# for CPU
+
+# for GPU
+ARG TEMP_DIR=/tmp/opencl
+
+COPY --from=base ${TEMP_DIR} ${TEMP_DIR}
+
+WORKDIR ${TEMP_DIR}
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ocl-icd-libopencl1 && \
+    rm -rf /var/lib/apt/lists/* && \
+    dpkg -i ${TEMP_DIR}/*.deb && \
+    ldconfig && \
+    rm -rf ${TEMP_DIR}
+
+# for VPU
+ARG LGPL_DEPS=udev
+
+WORKDIR /thirdparty
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    dpkg --get-selections | grep -v deinstall | awk '{print $1}' > no_vpu_packages.txt && \
+    apt-get install -y --no-install-recommends ${LGPL_DEPS} && \
+    if [ "$INSTALL_SOURCES" = "yes" ]; then \
+      sed -Ei 's/# deb-src /deb-src /' /etc/apt/sources.list && \
+      apt-get update && \
+	  dpkg --get-selections | grep -v deinstall | awk '{print $1}' > vpu_packages.txt && \
+	  grep -v -f no_vpu_packages.txt vpu_packages.txt | while read line; do \
+	  package=`echo $line`; \
+	  name=(${package//:/ }); \
+      grep -l GPL /usr/share/doc/${name[0]}/copyright; \
+      exit_status=$?; \
+	  if [ $exit_status -eq 0 ]; then \
+	    apt-get source -q --download-only $package;  \
+	  fi \
+      done && \
+      echo "Download source for `ls | wc -l` third-party packages: `du -sh`"; fi && \
+    rm -rf /var/lib/apt/lists/* && rm -rf *.txt
+
+COPY --from=base /opt/libusb-1.0.22 /opt/libusb-1.0.22
+
+WORKDIR /opt/libusb-1.0.22/libusb
+RUN /bin/mkdir -p '/usr/local/lib' && \
+    /bin/bash ../libtool   --mode=install /usr/bin/install -c   libusb-1.0.la '/usr/local/lib' && \
+    /bin/mkdir -p '/usr/local/include/libusb-1.0' && \
+    /usr/bin/install -c -m 644 libusb.h '/usr/local/include/libusb-1.0' && \
+    /bin/mkdir -p '/usr/local/lib/pkgconfig'
+
+WORKDIR /opt/libusb-1.0.22/
+RUN /usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \
+    cp ${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/97-myriad-usbboot.rules /etc/udev/rules.d/ && \
+    ldconfig
+
+# for HDDL
+WORKDIR /tmp
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        libboost-filesystem1.65-dev \
+        libboost-thread1.65-dev \
+        libjson-c3 libxxf86vm-dev && \
+    rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
+
+
+# Post-installation cleanup and setting up OpenVINO environment variables
+RUN if [ -f "${INTEL_OPENVINO_DIR}"/bin/setupvars.sh ]; then \
+        printf "\nexport TBB_DIR=\${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/cmake\n" >> ${INTEL_OPENVINO_DIR}/bin/setupvars.sh; \
+        printf "\nsource \${INTEL_OPENVINO_DIR}/bin/setupvars.sh\n" >> /home/openvino/.bashrc; \
+        printf "\nsource \${INTEL_OPENVINO_DIR}/bin/setupvars.sh\n" >> /root/.bashrc; \
+    fi; \
+    if [ -d "${INTEL_OPENVINO_DIR}"/opt/intel/mediasdk ]; then \
+        ln --symbolic "${INTEL_OPENVINO_DIR}"/opt/intel/mediasdk/ /opt/intel/mediasdk || true; \
+        printf "\nexport LIBVA_DRIVER_NAME=iHD \nexport LIBVA_DRIVERS_PATH=\${INTEL_OPENVINO_DIR}/opt/intel/mediasdk/lib64/ \nexport GST_VAAPI_ALL_DRIVERS=1 \nexport LIBRARY_PATH=\${INTEL_OPENVINO_DIR}/opt/intel/mediasdk/lib64/:\$LIBRARY_PATH \nexport LD_LIBRARY_PATH=\${INTEL_OPENVINO_DIR}/opt/intel/mediasdk/lib64/:\$LD_LIBRARY_PATH \n" >> /home/openvino/.bashrc; \
+        printf "\nexport LIBVA_DRIVER_NAME=iHD \nexport LIBVA_DRIVERS_PATH=\${INTEL_OPENVINO_DIR}/opt/intel/mediasdk/lib64/ \nexport GST_VAAPI_ALL_DRIVERS=1 \nexport LIBRARY_PATH=\${INTEL_OPENVINO_DIR}/opt/intel/mediasdk/lib64/:\$LIBRARY_PATH \nexport LD_LIBRARY_PATH=\${INTEL_OPENVINO_DIR}/opt/intel/mediasdk/lib64/:\$LD_LIBRARY_PATH \n" >> /root/.bashrc; \
+    fi;
+
+RUN apt-get update && \
+    apt-get autoremove -y dpkg-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+USER openvino
+WORKDIR ${INTEL_OPENVINO_DIR}
+
+CMD ["/bin/bash"]
+
+# Setup custom layers below