diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 036e4a25526ac0..352049ffbd9211 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -88,6 +88,7 @@ /src/frontends/tensorflow_common/ @openvinotoolkit/openvino-tf-frontend-maintainers /src/frontends/tensorflow_lite/ @openvinotoolkit/openvino-tf-frontend-maintainers /src/frontends/pytorch/ @openvinotoolkit/openvino-pytorch-frontend-maintainers +/src/frontends/jax/ @openvinotoolkit/openvino-jax-frontend-maintainers # OpenVINO ONNX Frontend: /src/frontends/onnx/ @openvinotoolkit/openvino-onnx-frontend-maintainers @@ -99,7 +100,7 @@ /tests/layer_tests/ @openvinotoolkit/openvino-tests-maintainers @openvinotoolkit/openvino-mo-maintainers /tests/layer_tests/pytorch_tests/ @openvinotoolkit/openvino-pytorch-frontend-maintainers /tests/layer_tests/tensorflow_tests @openvinotoolkit/openvino-tf-frontend-maintainers -/tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers +/tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers @openvinotoolkit/openvino-jax-frontend-maintainers /tests/model_hub_tests @openvinotoolkit/openvino-tf-frontend-maintainers /tests/model_hub_tests/pytorch @openvinotoolkit/openvino-pytorch-frontend-maintainers diff --git a/.github/actions/smart-ci/smart_ci.py b/.github/actions/smart-ci/smart_ci.py index 80d285da0e8d0e..1c8558f4779108 100644 --- a/.github/actions/smart-ci/smart_ci.py +++ b/.github/actions/smart-ci/smart_ci.py @@ -225,7 +225,7 @@ def main(): skip_workflow = False if is_merge_queue or (args.pr and not run_full_scope): - if args.skip_when_only_listed_labels_set: + if args.skip_when_only_listed_labels_set and changed_component_names: excepted_labels = set(args.skip_when_only_listed_labels_set.split(',')) excepted_labels_only = changed_component_names - excepted_labels == set() skip_workflow = excepted_labels_only diff --git a/.github/components.yml b/.github/components.yml index 8ec4b759db39e6..9c5d7c0089c9ca 100644 --- a/.github/components.yml +++ b/.github/components.yml @@ -13,6 +13,9 @@ LP_transformations: - TFL_FE - ONNX_FE - PDPD_FE + - JAX_FE + build: + - TOKENIZERS # TF_FE tests depends on tokenizers build preprocessing: revalidate: @@ -32,12 +35,14 @@ CPU: - PyTorch_FE - TF_FE - ONNX_FE + - JAX_FE build: - AUTO - HETERO - AUTO_BATCH - TEMPLATE - IR_FE + - TOKENIZERS # TF_FE tests depends on tokenizers build GPU: build: @@ -124,7 +129,7 @@ TF_FE: build: - CPU - Python_API - - TOKENIZERS + - TOKENIZERS # TF_FE tests depends on tokenizers build TFL_FE: revalidate: @@ -139,7 +144,13 @@ PyTorch_FE: build: - CPU - Python_API - - TOKENIZERS + +JAX_FE: + revalidate: + - MO + build: + - CPU + - Python_API C_API: build: @@ -155,6 +166,7 @@ Python_API: - samples - MO - tools + - TF_FE build: - CPU - HETERO @@ -164,9 +176,10 @@ Python_API: - IR_FE - ONNX_FE - PDPD_FE - - TF_FE - TFL_FE - PyTorch_FE + - JAX_FE + - TOKENIZERS # TF_FE tests depends on tokenizers build JS_API: build: @@ -200,6 +213,7 @@ MO: - TF_FE build: - Python_API + - TOKENIZERS # TF_FE tests depends on tokenizers build tools: build: @@ -223,5 +237,6 @@ ONNX_RT: build: [] TOKENIZERS: - revalidate: [] + revalidate: + - TF_FE build: [] diff --git a/.github/dependency_review.yml b/.github/dependency_review.yml index 7a1b3a4386a9da..11639f4d2d4b22 100644 --- a/.github/dependency_review.yml +++ b/.github/dependency_review.yml @@ -3,8 +3,10 @@ allow-licenses: - 'BSD-2-Clause' - 'BSD-3-Clause' - 'BSD-2-Clause AND BSD-3-Clause' + - 'BSD-3-Clause AND BSD-3-Clause-Clear' - 'MIT' - 'Apache-2.0' + - 'Apache-2.0 AND BSD-3-Clause' - 'ISC' - 'Apache-2.0 AND MIT' - 'BlueOak-1.0.0' diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 67a7eec64a342f..ae48310adafe6f 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-24878 +pr-25303 diff --git a/.github/dockerfiles/ov_build/fedora_33/Dockerfile b/.github/dockerfiles/ov_build/fedora_33/Dockerfile new file mode 100644 index 00000000000000..fc94c37d67a321 --- /dev/null +++ b/.github/dockerfiles/ov_build/fedora_33/Dockerfile @@ -0,0 +1,23 @@ +FROM openvinogithubactions.azurecr.io/dockerio/library/fedora:33 + +USER root + +RUN yum update -y && yum install -y git + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +ENV PATH="$SCCACHE_HOME:$PATH" diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile new file mode 100644 index 00000000000000..f3d8caa5220f28 --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile @@ -0,0 +1,55 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt update && \ + apt install software-properties-common git ca-certificates && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + apt update && \ + apt install \ + scons \ + wget \ + ninja-build \ + build-essential \ + python3-pip && \ + # vcpkg requires cmake 3.19 or later + python3 -m pip install -U pip cmake~=3.28.0 && \ + # vcpkg's tool dependencies + apt install curl zip unzip tar && \ + # vcpkg 'python3' port dependencies + apt install autoconf libtool autoconf-archive && \ + # vcpkg tree of dependencies require extra packages + apt install pkgconf linux-libc-dev && \ + apt --no-install-recommends install default-jdk && \ + rm -rf /var/lib/apt/lists/* + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +ENV PATH="$SCCACHE_HOME:$PATH" + +# Install Android SDK, NDK and Tools +ENV ANDROID_TOOLS /deps/android_tools +ENV ANDROID_NDK_HOME /deps/android_tools/ndk-bundle +RUN mkdir -p ${ANDROID_NDK_HOME} +ENV ANDROID_SDK_VERSION 29 + +RUN wget https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip && \ + unzip commandlinetools-linux-7583922_latest.zip +RUN echo "yes" | ./cmdline-tools/bin/sdkmanager --sdk_root=${ANDROID_TOOLS} --install "ndk-bundle" "platform-tools" "platforms;android-${ANDROID_SDK_VERSION}" diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile new file mode 100644 index 00000000000000..5eda853da56545 --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile @@ -0,0 +1,68 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + # install compilers to build OpenVINO for RISC-V 64 + apt-get install gcc-riscv64-linux-gnu g++-riscv64-linux-gnu && \ + apt-get install \ + curl \ + git \ + cmake \ + ccache \ + ninja-build \ + fdupes \ + patchelf \ + ca-certificates \ + gpg-agent \ + tzdata \ + # Python \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-distutils \ + # Compilers + gcc \ + g++ \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install RISC-V native debian packages +RUN echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted > riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy universe >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates universe >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy multiverse >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse >> riscv64-sources.list && \ + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security main restricted >> riscv64-sources.list && \ + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security universe >> riscv64-sources.list && \ + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security multiverse >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy main >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy universe >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main >> riscv64-sources.list && \ + mv riscv64-sources.list /etc/apt/sources.list.d/ + +RUN dpkg --add-architecture riscv64 && \ + apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/riscv64-sources.list && \ + apt-get install -y --no-install-recommends libpython3-dev:riscv64 + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py diff --git a/.github/dockerfiles/ov_build/webassembly/Dockerfile b/.github/dockerfiles/ov_build/webassembly/Dockerfile new file mode 100644 index 00000000000000..66765ed9341efe --- /dev/null +++ b/.github/dockerfiles/ov_build/webassembly/Dockerfile @@ -0,0 +1,33 @@ +FROM openvinogithubactions.azurecr.io/dockerio/emscripten/emsdk:3.1.61 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + apt-get update && \ + apt-get install \ + git \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +ENV PATH="$SCCACHE_HOME:$PATH" diff --git a/.github/labeler.yml b/.github/labeler.yml index aeb8e8f2655819..64a8661cf1e2e8 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -141,6 +141,7 @@ 'category: TF FE': - 'src/frontends/tensorflow/**/*' - 'src/frontends/tensorflow_common/**/*' +- 'src/bindings/python/src/openvino/frontend/tensorflow/**/*' - 'tests/layer_tests/tensorflow_tests/**/*' - 'tests/layer_tests/tensorflow2_keras_tests/**/*' - 'tests/layer_tests/jax_tests/**/*' @@ -161,6 +162,11 @@ - any: ['tests/model_hub_tests/**', '!tests/model_hub_tests/tensorflow/**/*'] +'category: JAX FE': +- 'src/frontends/jax/**/*' +- 'src/bindings/python/src/openvino/frontend/jax/**/*' +- 'tests/layer_tests/jax_tests/**/*' + 'category: tools': - any: ['tools/**', '!tools/mo/**/*', diff --git a/.github/scripts/workflow_rerun/rerunner.py b/.github/scripts/workflow_rerun/rerunner.py index f92130734c3fe6..b37f51e282a1cd 100644 --- a/.github/scripts/workflow_rerun/rerunner.py +++ b/.github/scripts/workflow_rerun/rerunner.py @@ -6,12 +6,14 @@ import tempfile from pathlib import Path +import requests from github import Github, Auth from workflow_rerun.argument_parser import get_arguments from workflow_rerun.constants import GITHUB_TOKEN, LOGGER from workflow_rerun.log_analyzer import LogAnalyzer from workflow_rerun.log_collector import collect_logs_for_run + if __name__ == '__main__': args = get_arguments() @@ -53,7 +55,12 @@ LOGGER.info(f'RUNNING IN DRY RUN MODE, NOT RETRIGGERING, EXITING') sys.exit(0) - status = run.rerun() + # PyGitHub does not expose the "/repos/{owner}/{repo}/actions/runs/RUN_ID/rerun-failed-jobs" endpoint + # so we have to use requests + response = requests.post(url=f'https://api.github.com/repos/{repository_name}/actions/runs/{run_id}/rerun-failed-jobs', + headers={'Authorization': f'Bearer {GITHUB_TOKEN}'}) + status = response.status_code == 201 + if status: LOGGER.info(f'RUN RETRIGGERED SUCCESSFULLY: {run.html_url}') else: diff --git a/.github/workflows/android_arm64.yml b/.github/workflows/android_arm64.yml index ad42737805b584..de5b6c0011e34d 100644 --- a/.github/workflows/android_arm64.yml +++ b/.github/workflows/android_arm64.yml @@ -20,6 +20,7 @@ jobs: runs-on: ubuntu-latest outputs: affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action @@ -40,15 +41,37 @@ jobs: skip_when_only_listed_labels_set: 'docs' skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' - Build: + Docker: needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_22_04_android_arm64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Smart_CI, Docker] timeout-minutes: 150 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_android_arm64 }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -64,8 +87,8 @@ jobs: OPENVINO_REPO: '/__w/openvino/openvino/openvino' VCPKG_ROOT: '/__w/openvino/openvino/vcpkg' BUILD_DIR: '/__w/openvino/openvino/build' - ANDROID_TOOLS: '/__w/openvino/openvino/android_tools' - ANDROID_NDK_HOME: '/__w/openvino/openvino/android_tools/ndk-bundle' + ANDROID_TOOLS: '/deps/android_tools' + ANDROID_NDK_HOME: '/deps/android_tools/ndk-bundle' ANDROID_SDK_VERSION: 29 ANDROID_ABI_CONFIG: arm64-v8a VCPKG_DEFAULT_BINARY_CACHE: '/mount/caches/ccache/android_arm64/vcpkg_cache' @@ -73,9 +96,6 @@ jobs: SCCACHE_AZURE_KEY_PREFIX: android_arm64 if: "!needs.smart_ci.outputs.skip_workflow" steps: - - name: Install git - run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: @@ -107,35 +127,6 @@ jobs: - name: System info uses: ./openvino/.github/actions/system_info - # - # Dependencies - # - - - name: Install dependencies - run: | - # generic dependencies - apt --assume-yes install ccache scons ninja-build build-essential python3-pip - - # vcpkg requires cmake 3.19 or later - python3 -m pip install -U pip cmake~=3.28.0 - # vcpkg's tool dependencies - apt --assume-yes install curl zip unzip tar - # vcpkg 'python3' port dependencies - apt --assume-yes install autoconf libtool autoconf-archive - # vcpkg tree of dependencies require extra packages - apt --assume-yes install pkgconf linux-libc-dev - - # Install Android SDK, NDK and Tools - apt -y --no-install-recommends install unzip wget default-jdk - wget https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip - unzip commandlinetools-linux-7583922_latest.zip - echo "yes" | ./cmdline-tools/bin/sdkmanager --sdk_root=${ANDROID_TOOLS} --install "ndk-bundle" "platform-tools" "platforms;android-${{ env.ANDROID_SDK_VERSION }}" - - - name: Install sccache - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - with: - version: "v0.7.5" - # # Build # diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index ceff14e1e09774..223a7418156e43 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -31,7 +31,7 @@ jobs: - name: suggester / clang-format if: startsWith(github.event_name, 'pull_request') - uses: reviewdog/action-suggester@9e1cd88b79ba3c0023c94e44accd72344f032093 # v1.13.0 + uses: reviewdog/action-suggester@a1d57ff096639094e0ba35ef3039e79316364796 # v1.15.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} level: warning @@ -60,7 +60,7 @@ jobs: # always provide suggestions even for skipped scripts in ov_shellcheck tagret - name: ShellCheck action if: always() - uses: reviewdog/action-shellcheck@6e3a862f231c6895fbd335b70adef8f9243d5762 # v1.21.0 + uses: reviewdog/action-shellcheck@52f34f737a16c65b8caa8c51ae1b23036afe5685 # v1.23.0 with: level: style reporter: github-pr-review diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a145276590d3e6..436f335276d169 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -137,6 +137,6 @@ jobs: lcov --capture --directory ${{ github.workspace }}/. --output-file coverage.info genhtml coverage.info --output-directory coverage-report - name: Collect coverage - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4.4.1 + uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0 with: verbose: true diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index 66dd87a4466ca6..02cd0abf018319 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -20,6 +20,7 @@ jobs: runs-on: ubuntu-latest outputs: affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action @@ -40,15 +41,42 @@ jobs: skip_when_only_listed_labels_set: 'docs' skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' - Build: + - name: Show affected components + run: | + echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}" + shell: bash + + Docker: needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/fedora_33 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Docker, Smart_CI] timeout-minutes: 150 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: fedora:33 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.fedora_33 }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -69,9 +97,6 @@ jobs: SCCACHE_AZURE_KEY_PREFIX: fedora33_x86_64_Release if: "!needs.smart_ci.outputs.skip_workflow" steps: - - name: Install git - run: yum update -y && yum install -y git - - name: Clone OpenVINO uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: @@ -89,14 +114,6 @@ jobs: # Dependencies # - - name: Install build dependencies - run: bash ${OPENVINO_REPO}/install_build_dependencies.sh - - - name: Install sccache - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - with: - version: "v0.7.5" - - name: Install python dependencies run: | python3 -m pip install -U pip @@ -204,14 +221,14 @@ jobs: if-no-files-found: 'error' RPM_Packages: - needs: Build + needs: [Docker, Build] timeout-minutes: 10 defaults: run: shell: bash - runs-on: ubuntu-20.04 + runs-on: aks-linux-4-cores-16gb container: - image: fedora:33 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.fedora_33 }} env: RPM_PACKAGES_DIR: /__w/openvino/packages/ @@ -254,7 +271,7 @@ jobs: python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")' - python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' + python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 7' benchmark_app --help ovc --help diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index d4734805a2cb4d..a7547a2483dd16 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -87,7 +87,7 @@ jobs: python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")' - python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' + python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 7' benchmark_app --help opt_in_out --help ovc --help diff --git a/.github/workflows/job_gpu_tests.yml b/.github/workflows/job_gpu_tests.yml index 7ba71afec09748..7a5af97cdcde49 100644 --- a/.github/workflows/job_gpu_tests.yml +++ b/.github/workflows/job_gpu_tests.yml @@ -21,6 +21,8 @@ on: required: false default: '{"image": null}' +permissions: read-all + jobs: GPU: timeout-minutes: 80 diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index c6b185ec4c6a4a..bab441e3e27453 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -121,7 +121,7 @@ jobs: # - name: Python API Tests - # if: ${{ fromJSON(inputs.affected-components).Python_API.test && runner.arch != 'ARM64' }} # Ticket: 126380, 127101 + if: ${{ fromJSON(inputs.affected-components).Python_API.test }} run: | # for 'template' extension export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH @@ -246,6 +246,14 @@ jobs: env: TEST_DEVICE: CPU TEST_PRECISION: FP16 + + - name: JAX Layer Tests - JAX FE + if: ${{ fromJSON(inputs.affected-components).JAX_FE.test && runner.arch != 'ARM64' && runner.os != 'macOS' }} + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit_jax_fe --junitxml=${INSTALL_TEST_DIR}/TEST-jax_fe.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + JAX_TRACE_MODE: JAXPR - name: TensorFlow Lite Layer Tests - TFL FE if: fromJSON(inputs.affected-components).TFL_FE.test diff --git a/.github/workflows/job_samples_tests.yml b/.github/workflows/job_samples_tests.yml index c9c13569342784..e453210d58b13b 100644 --- a/.github/workflows/job_samples_tests.yml +++ b/.github/workflows/job_samples_tests.yml @@ -7,11 +7,11 @@ on: description: 'Machine on which the tests would run' type: string required: true - image: - description: 'Docker image in which the tests would run' + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' type: string required: false - default: null + default: '{"image": null}' affected-components: description: 'Components that are affected by changes in the commit defined by the Smart CI Action' type: string @@ -21,9 +21,9 @@ permissions: read-all jobs: Samples: + timeout-minutes: 30 runs-on: ${{ inputs.runner }} - container: - image: ${{ inputs.image }} + container: ${{ fromJSON(inputs.container) }} defaults: run: shell: bash @@ -66,6 +66,8 @@ jobs: run: brew install coreutils - name: Fetch setup_python action + # Python is already installed on Ubuntu within Dockerfile + if: runner.os != 'Linux' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | @@ -74,6 +76,8 @@ jobs: path: 'openvino' - name: Setup Python 3.11 + # Python is already installed on Ubuntu within Dockerfile + if: runner.os != 'Linux' uses: ./openvino/.github/actions/setup_python with: version: '3.11' diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml index 684abf8e026a45..da9e096e7abd8b 100644 --- a/.github/workflows/job_tokenizers.yml +++ b/.github/workflows/job_tokenizers.yml @@ -96,10 +96,21 @@ jobs: # Dependencies # - - name: Install python dependencies + - name: Install OpenVINO Python wheel (Linux and macOS) + if: runner.os != 'Windows' + run: | + # Find and install wheel + pushd ${INSTALL_DIR}/tools + wheel_name=$(find . -name 'openvino-*.whl') + python3 -m pip install $wheel_name + popd + + - name: Install OpenVINO Python wheel (Windows) + if: runner.os == 'Windows' run: | - # wheel packaging - python3 -m pip install -r ${{ env.OPENVINO_TOKENIZERS_REPO }}/requirements-build.txt + # Find and install wheel + $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\tools" -Filter openvino-*.whl | % { $_.FullName } + python3 -m pip install "$ovCoreWheelPath" # # Build @@ -108,20 +119,18 @@ jobs: - name: Build tokenizers wheel (Linux and macOS) if: runner.os != 'Windows' run: | - source ${INSTALL_DIR}/setupvars.sh - python -m build --wheel --outdir ${EXTENSION_BUILD_DIR} ${OPENVINO_TOKENIZERS_REPO} + # use OpenVINO wheel package only to build the extension + export OpenVINO_DIR=$(python3 -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')") + python -m pip wheel -v --no-deps --wheel-dir ${EXTENSION_BUILD_DIR} ${OPENVINO_TOKENIZERS_REPO} env: - CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF' CMAKE_BUILD_PARALLEL_LEVEL: '4' - CMAKE_GENERATOR: 'Unix Makefiles' - name: Build tokenizers wheel (Windows) if: runner.os == 'Windows' run: | - . "${{ env.INSTALL_DIR }}/setupvars.ps1" - python3 -m build --wheel --outdir ${env:EXTENSION_BUILD_DIR} ${env:OPENVINO_TOKENIZERS_REPO} + $env:OpenVINO_DIR=$(python3 -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')") + python3 -m pip wheel -v --no-deps --wheel-dir ${env:EXTENSION_BUILD_DIR} ${env:OPENVINO_TOKENIZERS_REPO} env: - CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF' CMAKE_BUILD_PARALLEL_LEVEL: '4' # diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 633a414d713ad7..4358af5707b077 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -224,7 +224,7 @@ jobs: run: | /usr/bin/python3.8 -m pip install -U pip /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - cmake -UPYTHON* \ + cmake -UPython* -U_Python* -UPYTHON* \ -UTBB* \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_PYTHON_PACKAGING=ON \ @@ -245,11 +245,9 @@ jobs: - name: CMake configure, build and install - OpenVINO JS API if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - run: - cmake -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -UTBB* -S ${OPENVINO_REPO} -B ${BUILD_DIR} - + run: | + cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR} cmake --build ${BUILD_DIR} --parallel - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake # @@ -317,7 +315,7 @@ jobs: uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-4-cores-16gb' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} JS_API: diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index d429398c05c810..9156fa711e752a 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -309,7 +309,7 @@ jobs: uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-16-cores-arm' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_arm64 }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} JS_API: diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index 48756022f6244b..cdae9c77af9b40 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -317,6 +317,7 @@ jobs: -DENABLE_OV_TF_LITE_FRONTEND=OFF \ -DENABLE_OV_PADDLE_FRONTEND=OFF \ -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -DENABLE_OV_JAX_FRONTEND=OFF \ -DENABLE_OV_ONNX_FRONTEND=OFF \ -DSELECTIVE_BUILD_STAT=${SELECTIVE_BUILD_STAT_DIR}/*.csv \ -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ diff --git a/.github/workflows/linux_riscv.yml b/.github/workflows/linux_riscv.yml index c9629e1cf22477..6c26bb8b3859f0 100644 --- a/.github/workflows/linux_riscv.yml +++ b/.github/workflows/linux_riscv.yml @@ -1,8 +1,5 @@ name: Linux RISC-V with Conan (Ubuntu 22.04, Python 3.10) on: - schedule: - # at 00:00 on Wednesday and Saturday - - cron: '0 0 * * 3,6' workflow_dispatch: pull_request: merge_group: @@ -23,6 +20,7 @@ jobs: runs-on: ubuntu-latest outputs: affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action @@ -43,15 +41,37 @@ jobs: skip_when_only_listed_labels_set: 'docs' skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' - Build: + Docker: needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_22_04_riscv + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Smart_CI, Docker] timeout-minutes: 150 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_riscv }} volumes: - /mount:/mount env: @@ -70,12 +90,6 @@ jobs: if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }} steps: - - name: Set apt retries - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - - name: Install git - run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: @@ -119,43 +133,16 @@ jobs: # Dependencies # - - name: Install build dependencies + - name: Install Python dependencies run: | # create build directory mkdir -p ${OPENVINO_BUILD_DIR} - # install compilers to build OpenVINO for RISC-V 64 - apt --assume-yes install gcc-riscv64-linux-gnu g++-riscv64-linux-gnu - apt --assume-yes install -yf gcc g++ python3-pip python3-venv python3-dev - # generic dependencies - apt --assume-yes install cmake ccache ninja-build fdupes patchelf - python3 -m venv ${OPENVINO_BUILD_DIR}/env source ${OPENVINO_BUILD_DIR}/env/bin/activate python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt python3 -m pip install conan - - name: Install RISC-V native debian packages - run: | - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted > riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy universe >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates universe >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy multiverse >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse >> riscv64-sources.list - echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security main restricted >> riscv64-sources.list - echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security universe >> riscv64-sources.list - echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security multiverse >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy main >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy universe >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main >> riscv64-sources.list - mv riscv64-sources.list /etc/apt/sources.list.d/ - dpkg --add-architecture riscv64 - apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/riscv64-sources.list - apt-get install -y --no-install-recommends libpython3-dev:riscv64 - - name: Create conan_toolchain.cmake file run: | source ${OPENVINO_BUILD_DIR}/env/bin/activate diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index fb298026f4b473..d52378bab8c95b 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -35,14 +35,14 @@ jobs: include: - SANITIZER: 'AddressAndLeak' SANITIZER_CMAKE_OPTION: '-DENABLE_SANITIZER=ON' - - SANITIZER: 'UndefinedBehavior' - SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' +# - SANITIZER: 'UndefinedBehavior' # Ticket: 144163 +# SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' # - SANITIZER: 'Thread' # Problems with protobuf # SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - CMAKE_BUILD_TYPE: 'Release' - CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_BUILD_TYPE: 'RelWithDebInfo' + CMAKE_GENERATOR: 'Ninja' GITHUB_WORKSPACE: '/__w/openvino/openvino' OPENVINO_REPO: /__w/openvino/openvino/openvino OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib @@ -135,7 +135,6 @@ jobs: -DENABLE_NCC_STYLE=OFF \ -DENABLE_TESTS=ON \ -DENABLE_STRICT_DEPENDENCIES=OFF \ - -DENABLE_SYSTEM_TBB=ON \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCPACK_GENERATOR=TGZ \ @@ -144,6 +143,7 @@ jobs: -DENABLE_OV_TF_LITE_FRONTEND=ON \ -DENABLE_OV_PADDLE_FRONTEND=ON \ -DENABLE_OV_PYTORCH_FRONTEND=ON \ + -DENABLE_OV_JAX_FRONTEND=ON \ -DENABLE_OV_ONNX_FRONTEND=ON \ -DENABLE_ONEDNN_FOR_GPU=OFF \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ @@ -159,9 +159,9 @@ jobs: - name: Cmake install - OpenVINO run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake --config ${{ env.CMAKE_BUILD_TYPE }} - name: Remove unused files to free space run: rm -rf ${BUILD_DIR}/* @@ -360,7 +360,7 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowLiteFrontend.xml - name: Transformations func tests - if: always() + if: ${{ 'false' }} # Ticket: 143900 run: | source ${INSTALL_DIR}/setupvars.sh @@ -389,7 +389,7 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - name: ov_subgraphs_dumper_tests tests - if: ${{ 'false' }} # Ticket: 134419 + if: always() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ diff --git a/.github/workflows/webassembly.yml b/.github/workflows/webassembly.yml index 9ce66a9fb02b14..902fb0dfcb00f0 100644 --- a/.github/workflows/webassembly.yml +++ b/.github/workflows/webassembly.yml @@ -20,6 +20,7 @@ jobs: runs-on: ubuntu-latest outputs: affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action @@ -40,14 +41,41 @@ jobs: skip_when_only_listed_labels_set: 'docs' skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' - Build: + - name: Show affected components + run: | + echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}" + shell: bash + + Docker: needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/webassembly + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Docker, Smart_CI] defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: emscripten/emsdk + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.webassembly }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -62,20 +90,12 @@ jobs: SCCACHE_AZURE_KEY_PREFIX: webassembly_Release if: "!needs.smart_ci.outputs.skip_workflow" steps: - - name: Install git - run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - - name: Install sccache - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - with: - version: "v0.7.5" - - name: emcmake cmake - configure run: | emcmake cmake \ diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index 96e6c44145ddd3..977f9aee91bcce 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -319,6 +319,7 @@ jobs: -DENABLE_OV_TF_LITE_FRONTEND=OFF ` -DENABLE_OV_PADDLE_FRONTEND=OFF ` -DENABLE_OV_PYTORCH_FRONTEND=OFF ` + -DENABLE_OV_JAX_FRONTEND=OFF ` -DENABLE_OV_ONNX_FRONTEND=OFF ` -DSELECTIVE_BUILD_STAT=${{ env.SELECTIVE_BUILD_STAT_DIR }}\*.csv ` -S ${{ env.OPENVINO_REPO }} ` diff --git a/CMakeLists.txt b/CMakeLists.txt index adfdc58a8de05b..494a3e4ce21d61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,10 @@ if(POLICY CMP0149) cmake_policy(SET CMP0149 NEW) endif() -project(OpenVINO DESCRIPTION "OpenVINO toolkit") +project(OpenVINO + DESCRIPTION "OpenVINO toolkit" + HOMEPAGE_URL "https://docs.openvino.ai/2024/home.html" + LANGUAGES C CXX) find_package(OpenVINODeveloperScripts REQUIRED PATHS "${OpenVINO_SOURCE_DIR}/cmake/developer_package" diff --git a/cmake/coverage.cmake b/cmake/coverage.cmake index d58200400c904f..061d6c493cdf41 100644 --- a/cmake/coverage.cmake +++ b/cmake/coverage.cmake @@ -112,6 +112,13 @@ if(ENABLE_OV_IR_FRONTEND) PREFIX "${OV_COVERAGE_BASE_DIRECTORY}") endif() +if(ENABLE_OV_JAX_FRONTEND) + ov_coverage_extract(INPUT "openvino" OUTPUT "jax_frontend" + PATTERNS "${OV_COVERAGE_BASE_DIRECTORY}/src/frontends/jax/*") + ov_coverage_genhtml(INFO_FILE "jax_frontend" + PREFIX "${OV_COVERAGE_BASE_DIRECTORY}") +endif() + if(ENABLE_OV_ONNX_FRONTEND) ov_coverage_extract(INPUT "openvino" OUTPUT "onnx_frontend" PATTERNS "${OV_COVERAGE_BASE_DIRECTORY}/src/frontends/onnx/*") diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 69dedb3e3bd527..219d464682b016 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -89,10 +89,10 @@ function(ov_download_tbb) # TODO: add target_path to be platform specific as well, to avoid following if # build oneTBB 2021.2.1 with Visual Studio 2019 (MSVC 14.21) RESOLVE_DEPENDENCY(TBB - ARCHIVE_WIN "oneapi-tbb-2021.2.2-win.zip" + ARCHIVE_WIN "oneapi-tbb-2021.2.5-win-trim.zip" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "103b19a8af288c6a7d83ed3f0d2239c4afd0dd189fc12aad1d34b3c9e78df94b" + SHA256 "a9384a25861946648db58f7f294c4aa2b1be7ae52748024fef3e13ca2762a1ba" USE_NEW_LOCATION TRUE) elseif(ANDROID AND X86_64) RESOLVE_DEPENDENCY(TBB @@ -104,10 +104,10 @@ function(ov_download_tbb) elseif(LINUX AND X86_64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17) # build oneTBB 2021.2.1 with gcc 4.8 (glibc 2.17) RESOLVE_DEPENDENCY(TBB - ARCHIVE_LIN "oneapi-tbb-2021.2.4-lin.tgz" + ARCHIVE_LIN "oneapi-tbb-2021.2.5-lin-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "6523661559a340e88131472ea9a595582c306af083e55293b7357d11b8015546" + SHA256 "9bea2c838df3085d292989d643523dc1cedce9b46d5a03eec90104151b49a180" USE_NEW_LOCATION TRUE) elseif(YOCTO_AARCH64) RESOLVE_DEPENDENCY(TBB @@ -119,34 +119,34 @@ function(ov_download_tbb) elseif(APPLE AND X86_64) # build oneTBB 2021.2.1 with OS version 11.4 RESOLVE_DEPENDENCY(TBB - ARCHIVE_MAC "oneapi-tbb-2021.2.1-mac.tgz" + ARCHIVE_MAC "oneapi-tbb-2021.2.5-mac-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "c57ce4b97116cd3093c33e6dcc147fb1bbb9678d0ee6c61a506b2bfe773232cb" + SHA256 "5a6035fcf7b9d3bd8183ecc31b3e2b2026a749152b5b879f8e4d147e09479efc" USE_NEW_LOCATION TRUE) elseif(WIN32 AND AARCH64) # build oneTBB 2021.2.1 with Visual Studio 2022 (MSVC 14.35) RESOLVE_DEPENDENCY(TBB - ARCHIVE_WIN "oneapi-tbb-2021.2.1-win-arm64.zip" + ARCHIVE_WIN "oneapi-tbb-2021.2.5-win-arm64-trim.zip" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "09fe7f5e7be589aa34ccd20fdfd7cad9e0afa89d1e74ecdb008a75d0af71d6e1" + SHA256 "c26b7593e1808c2dd15a768cd4ea1ee14aa0aa2dacb210b86e326ab7960d2473" USE_NEW_LOCATION TRUE) elseif(LINUX AND AARCH64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17) - # build oneTBB 2021.2.1 with gcc 4.8 (glibc 2.17) + # build oneTBB with glibc 2.17 RESOLVE_DEPENDENCY(TBB - ARCHIVE_LIN "oneapi-tbb-2021.2.5-lin-arm64.tgz" + ARCHIVE_LIN "oneapi-tbb-2021.13.0-rc1-lin-arm64-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "86b24e4db254136bf88f0a8195ae478494e34db109e08569b1c059b174494865" + SHA256 "7fe49525217de9536980a820d90645784216ad4a61e11799b8c95129dcdeeecf" USE_NEW_LOCATION TRUE) elseif(APPLE AND AARCH64) - # build oneTBB 2021.2.1 with export MACOSX_DEPLOYMENT_TARGET=11.0 + # build oneTBB with export MACOSX_DEPLOYMENT_TARGET=11.0 RESOLVE_DEPENDENCY(TBB - ARCHIVE_MAC "oneapi-tbb-2021.2.5-mac-arm64.tgz" + ARCHIVE_MAC "oneapi-tbb-2021.13.0-rc1-mac-arm64-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "bb1f84b1dcc50787f35c99b4b6ecea733d3068ca3467b5ebd2e369c4f7eccb53" + SHA256 "d3ce1c00e46a187baee459458e8d13d3421dc7242bff0c977b95d8d66d74441a" USE_NEW_LOCATION TRUE) else() message(WARNING "Prebuilt TBB is not available on current platform") diff --git a/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake b/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake index 904ad86e31d8f0..ee8111bd185d85 100644 --- a/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake +++ b/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake @@ -211,6 +211,8 @@ set(CMAKE_POLICY_DEFAULT_CMP0068 NEW) set(CMAKE_POLICY_DEFAULT_CMP0074 NEW) # CMake 3.13+: option() honors normal variables. set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) +# CMake 3.15+: export(PACKAGE) does not populate package registry by default. +set(CMAKE_POLICY_DEFAULT_CMP0090 NEW) # CMake 3.15: Modules FindPython3, FindPython2 and FindPython use LOCATION for lookup strategy set(CMAKE_POLICY_DEFAULT_CMP0094 NEW) # CMake 3.19+: An imported target missing its location property fails during generation. @@ -222,6 +224,11 @@ set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) # CMake 3.27+ :Visual Studio Generators select latest Windows SDK by default. set(CMAKE_POLICY_DEFAULT_CMP0149 NEW) +set(CMAKE_FIND_USE_PACKAGE_REGISTRY OFF CACHE BOOL "Disables search in user / system package registries") +set(CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY ON CACHE BOOL "Disables search in user package registries") +set(CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY ON CACHE BOOL "Disables search in system package registries") +set(CMAKE_EXPORT_PACKAGE_REGISTRY OFF CACHE BOOL "Disables package registry. Required for 3rd party projects like rapidjson, gflags") +set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY ON CACHE BOOL "Disables package registry. Required for 3rd party projects like rapidjson, gflags") set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "Don't warn about obsolete cmake versions in 3rdparty") set(CMAKE_WARN_ON_ABSOLUTE_INSTALL_DESTINATION ON CACHE BOOL "Warn about absolute paths in destination") diff --git a/cmake/developer_package/api_validator/api_validator.cmake b/cmake/developer_package/api_validator/api_validator.cmake index 0fd1f2c03a3343..ff6b0fbaa061e9 100644 --- a/cmake/developer_package/api_validator/api_validator.cmake +++ b/cmake/developer_package/api_validator/api_validator.cmake @@ -2,7 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # -if(WIN32) +macro(ov_search_api_validator) + if(NOT ENABLE_API_VALIDATOR) + return() + endif() + # CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION is only set when # Visual Studio generators are used, but we need it # when we use Ninja as well @@ -12,7 +16,7 @@ if(WIN32) set(CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION ${WINDOWS_SDK_VERSION}) message(STATUS "Use ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} Windows SDK version") else() - message(FATAL_ERROR "WindowsSDKVersion environment variable is not set,\ + message(FATAL_ERROR "WindowsSDKVersion environment variable is not set,\ can't find Windows SDK version. Try to use vcvarsall.bat script") endif() endif() @@ -41,6 +45,11 @@ can't find Windows SDK version. Try to use vcvarsall.bat script") message(STATUS "Found apivalidator: ${ONECORE_API_VALIDATOR}") endif() endif() +endmacro() + + +if(ENABLE_API_VALIDATOR) + ov_search_api_validator() endif() function(_ov_add_api_validator_post_build_step_recursive) diff --git a/cmake/developer_package/compile_flags/os_flags.cmake b/cmake/developer_package/compile_flags/os_flags.cmake index f846d8a0050afc..a49dce9bce7b50 100644 --- a/cmake/developer_package/compile_flags/os_flags.cmake +++ b/cmake/developer_package/compile_flags/os_flags.cmake @@ -75,11 +75,10 @@ macro(ov_dev_package_no_errors) endif() endif() - if (CMAKE_COMPILE_WARNING_AS_ERROR AND WIN32) + if(CMAKE_COMPILE_WARNING_AS_ERROR AND WIN32) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") if(CMAKE_VERSION VERSION_LESS 3.24) - string(REPLACE "/WX" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - string(REPLACE "/WX" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + ov_add_compiler_flags(/WX-) endif() string(REPLACE "/WX" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") endif() diff --git a/cmake/developer_package/compile_flags/sdl.cmake b/cmake/developer_package/compile_flags/sdl.cmake index ce47c9cd552f93..d399ec4f62fe0d 100644 --- a/cmake/developer_package/compile_flags/sdl.cmake +++ b/cmake/developer_package/compile_flags/sdl.cmake @@ -48,6 +48,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG OR OV_COMPILER_IS_INTEL_LLVM endif() elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") set(OV_C_CXX_FLAGS "${OV_C_CXX_FLAGS} /sdl /guard:cf") + set(OV_LINKER_FLAGS "${OV_LINKER_FLAGS} /guard:cf") endif() if(ENABLE_QSPECTRE) diff --git a/cmake/developer_package/cross_compile/python_helpers.cmake b/cmake/developer_package/cross_compile/python_helpers.cmake index a6c7de739375f1..050c99db9a852f 100644 --- a/cmake/developer_package/cross_compile/python_helpers.cmake +++ b/cmake/developer_package/cross_compile/python_helpers.cmake @@ -49,6 +49,19 @@ endfunction() # Wrapper for find_package(Python3) to allow cross-compilation macro(ov_find_python3 find_package_mode) + # Settings for FindPython3.cmake + if(NOT DEFINED Python3_USE_STATIC_LIBS) + set(Python3_USE_STATIC_LIBS OFF) + endif() + + if(NOT DEFINED Python3_FIND_VIRTUALENV) + set(Python3_FIND_VIRTUALENV FIRST) + endif() + + if(NOT DEFINED Python3_FIND_IMPLEMENTATIONS) + set(Python3_FIND_IMPLEMENTATIONS CPython PyPy) + endif() + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) set(python3_development_component Development.Module) else() diff --git a/cmake/developer_package/features.cmake b/cmake/developer_package/features.cmake index da2fbf6af4f5fe..d2fdcd4cf12049 100644 --- a/cmake/developer_package/features.cmake +++ b/cmake/developer_package/features.cmake @@ -5,8 +5,11 @@ include(options) include(target_flags) -set (CPACK_GENERATOR "TGZ" CACHE STRING "Cpack generator for OpenVINO") -list (APPEND OV_OPTIONS CPACK_GENERATOR) +if(WIN32) + set (CPACK_GENERATOR "ZIP" CACHE STRING "Cpack generator for OpenVINO") +else() + set (CPACK_GENERATOR "TGZ" CACHE STRING "Cpack generator for OpenVINO") +endif() ov_dependent_option (ENABLE_LTO "Enable Link Time Optimization" OFF "LINUX;NOT ARM;CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9" OFF) diff --git a/cmake/developer_package/frontends/frontends.cmake b/cmake/developer_package/frontends/frontends.cmake index 0b959322950106..87efaea0ceb387 100644 --- a/cmake/developer_package/frontends/frontends.cmake +++ b/cmake/developer_package/frontends/frontends.cmake @@ -319,7 +319,7 @@ macro(ov_add_frontend) if(OV_FRONTEND_LINKABLE_FRONTEND) set(export_set EXPORT OpenVINOTargets) set(archive_dest ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${dev_component} ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) - set(namelink NAMELINK_COMPONENT ${dev_component}) + set(namelink NAMELINK_COMPONENT ${OV_CPACK_COMP_LINKS} ${OV_CPACK_COMP_LINKS_EXCLUDE_ALL}) else() set(namelink NAMELINK_SKIP) endif() diff --git a/cmake/developer_package/ncc_naming_style/openvino.style b/cmake/developer_package/ncc_naming_style/openvino.style index 8bfa4164490482..6c82ccb9b8c55a 100644 --- a/cmake/developer_package/ncc_naming_style/openvino.style +++ b/cmake/developer_package/ncc_naming_style/openvino.style @@ -1,6 +1,6 @@ # custom OpenVINO values CppMethod: '^(operator\W+|[a-z_\d]+|signaling_NaN|quiet_NaN|OPENVINO_OP)$' -ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|float4_e2m1|numeric_limits|ngraph_error|stopwatch|unsupported_op)$' +ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|float4_e2m1|float8_e8m0|numeric_limits|ngraph_error|stopwatch|unsupported_op)$' StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair|stat)$' FunctionName: '^(operator\W+|[a-z_\d]+)|PrintTo$' Namespace: '^([a-z\d_]*|InferenceEngine)$' @@ -18,7 +18,7 @@ VariableReference: '^\w+$' EnumName: '^[A-Z][\w]+$' # excepts element_type -EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u2|u3|u4|u6|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|f4e2m1|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$' +EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u2|u3|u4|u6|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|f4e2m1|f8e8m0|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$' # TODO: align UsingDeclaration: '^.*$' TypedefName: '^.*$' diff --git a/cmake/developer_package/options.cmake b/cmake/developer_package/options.cmake index a54a8bed1d4e08..3ad42f156c706c 100644 --- a/cmake/developer_package/options.cmake +++ b/cmake/developer_package/options.cmake @@ -11,11 +11,13 @@ endif() macro(ov_option variable description value) option(${variable} "${description}" ${value}) list(APPEND OV_OPTIONS ${variable}) + set(OV_OPTIONS "${OV_OPTIONS}" CACHE INTERNAL "A list of OpenVINO cmake options") endmacro() macro(ov_dependent_option variable description def_value condition fallback_value) cmake_dependent_option(${variable} "${description}" ${def_value} "${condition}" ${fallback_value}) list(APPEND OV_OPTIONS ${variable}) + set(OV_OPTIONS "${OV_OPTIONS}" CACHE INTERNAL "A list of OpenVINO cmake options") endmacro() macro(ov_option_enum variable description value) @@ -29,6 +31,7 @@ macro(ov_option_enum variable description value) endif() list(APPEND OV_OPTIONS ${variable}) + set(OV_OPTIONS "${OV_OPTIONS}" CACHE INTERNAL "A list of OpenVINO cmake options") set(${variable} ${value} CACHE STRING "${description}") set_property(CACHE ${variable} PROPERTY STRINGS ${OPTION_ENUM_ALLOWED_VALUES}) diff --git a/cmake/developer_package/packaging/archive.cmake b/cmake/developer_package/packaging/archive.cmake index 19c6ac2ae5fc11..853b7649e88ddf 100644 --- a/cmake/developer_package/packaging/archive.cmake +++ b/cmake/developer_package/packaging/archive.cmake @@ -90,6 +90,12 @@ macro(ov_define_component_include_rules) # scripts unset(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL) unset(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # npu internal tools + unset(OV_CPACK_COMP_NPU_INTERNAL_EXCLUDE_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/common-libraries.cmake b/cmake/developer_package/packaging/common-libraries.cmake index c47c123ab4b31d..4ec96dc28b53e8 100644 --- a/cmake/developer_package/packaging/common-libraries.cmake +++ b/cmake/developer_package/packaging/common-libraries.cmake @@ -57,6 +57,9 @@ macro(ov_override_component_names) # merge C++ and C runtimes set(OV_CPACK_COMP_CORE_C "${OV_CPACK_COMP_CORE}") set(OV_CPACK_COMP_CORE_C_DEV "${OV_CPACK_COMP_CORE_DEV}") + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") endmacro() ov_override_component_names() @@ -104,6 +107,12 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # npu internal tools + set(OV_CPACK_COMP_NPU_INTERNAL_EXCLUDE_ALL EXCLUDE_FROM_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/debian/debian.cmake b/cmake/developer_package/packaging/debian/debian.cmake index bdb005ab6c3706..c7f49419111cea 100644 --- a/cmake/developer_package/packaging/debian/debian.cmake +++ b/cmake/developer_package/packaging/debian/debian.cmake @@ -60,6 +60,9 @@ macro(ov_override_component_names) # merge all C / C++ samples as a single samples component set(OV_CPACK_COMP_CPP_SAMPLES "samples") set(OV_CPACK_COMP_C_SAMPLES "${OV_CPACK_COMP_CPP_SAMPLES}") + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") endmacro() ov_override_component_names() @@ -111,6 +114,12 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # npu internal tools + set(OV_CPACK_COMP_NPU_INTERNAL_EXCLUDE_ALL EXCLUDE_FROM_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/npm.cmake b/cmake/developer_package/packaging/npm.cmake index 6d7ca6d04e680a..24453965125348 100644 --- a/cmake/developer_package/packaging/npm.cmake +++ b/cmake/developer_package/packaging/npm.cmake @@ -4,7 +4,7 @@ include(GNUInstallDirs) -# We have to specify RPATH, all runtime libs are in one dir +# We have to specify RPATH, all runtime libs are in one dir set(CMAKE_SKIP_INSTALL_RPATH OFF) # @@ -32,6 +32,20 @@ endmacro() ov_npm_cpack_set_dirs() +# +# Override CPack components name for NPM generator +# This is needed to change the granularity, i.e. merge several components +# into a single one +# + +macro(ov_override_component_names) + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") +endmacro() + +ov_override_component_names() + # # Override include / exclude rules for components # This is required to exclude some files from installation @@ -67,6 +81,12 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + unset(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL) + # symbolic links + unset(OV_CPACK_COMP_LINKS_EXCLUDE_ALL) + # npu internal tools + set(OV_CPACK_COMP_NPU_INTERNAL_EXCLUDE_ALL EXCLUDE_FROM_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/nsis.cmake b/cmake/developer_package/packaging/nsis.cmake index 9dcda7315dab61..f5f9a233e8b87f 100644 --- a/cmake/developer_package/packaging/nsis.cmake +++ b/cmake/developer_package/packaging/nsis.cmake @@ -87,6 +87,20 @@ endmacro() ov_nsis_cpack_set_dirs() +# +# Override CPack components name for NSIS generator +# This is needed to change the granularity, i.e. merge several components +# into a single one +# + +macro(ov_override_component_names) + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") +endmacro() + +ov_override_component_names() + # # Override include / exclude rules for components # This is required to exclude some files from installation @@ -122,6 +136,12 @@ macro(ov_define_component_include_rules) # scripts unset(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL) unset(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # npu internal tools + set(OV_CPACK_COMP_NPU_INTERNAL_EXCLUDE_ALL EXCLUDE_FROM_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/packaging.cmake b/cmake/developer_package/packaging/packaging.cmake index 41c0308acafba6..d62a8ca7fe2084 100644 --- a/cmake/developer_package/packaging/packaging.cmake +++ b/cmake/developer_package/packaging/packaging.cmake @@ -178,6 +178,10 @@ macro(ov_define_component_names) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES "install_dependencies") set(OV_CPACK_COMP_SETUPVARS "setupvars") + # symbolic_links + set(OV_CPACK_COMP_LINKS "core_dev_links") + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG "core_dev_pkgconfig") endmacro() ov_define_component_names() diff --git a/cmake/developer_package/packaging/rpm/rpm.cmake b/cmake/developer_package/packaging/rpm/rpm.cmake index 99f11730983f04..7c9fb4f22a372d 100644 --- a/cmake/developer_package/packaging/rpm/rpm.cmake +++ b/cmake/developer_package/packaging/rpm/rpm.cmake @@ -51,6 +51,9 @@ macro(ov_override_component_names) # merge all C / C++ samples as a single samples component set(OV_CPACK_COMP_CPP_SAMPLES "samples") set(OV_CPACK_COMP_C_SAMPLES "${OV_CPACK_COMP_CPP_SAMPLES}") + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") endmacro() ov_override_component_names() @@ -102,6 +105,12 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # npu internal tools + set(OV_CPACK_COMP_NPU_INTERNAL_EXCLUDE_ALL EXCLUDE_FROM_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/features.cmake b/cmake/features.cmake index bfa3a05481d77f..4063e2f8545ced 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -137,6 +137,7 @@ endif() ov_option(ENABLE_OV_PADDLE_FRONTEND "Enable PaddlePaddle FrontEnd" ON) ov_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON) ov_option(ENABLE_OV_PYTORCH_FRONTEND "Enable PyTorch FrontEnd" ON) +ov_option(ENABLE_OV_JAX_FRONTEND "Enable JAX FrontEnd" ON) ov_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON) ov_option(ENABLE_OV_TF_FRONTEND "Enable TensorFlow FrontEnd" ON) ov_option(ENABLE_OV_TF_LITE_FRONTEND "Enable TensorFlow Lite FrontEnd" ON) @@ -192,6 +193,8 @@ ov_dependent_option(ENABLE_JS "Enables JS API building" ON "NOT ANDROID;NOT EMSC ov_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF) +ov_dependent_option (ENABLE_API_VALIDATOR "Enables API Validator usage" ON "WIN32" OFF) + if(NOT BUILD_SHARED_LIBS AND ENABLE_OV_TF_FRONTEND) set(FORCE_FRONTENDS_USE_PROTOBUF ON) else() diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 90b01a196cec84..b97f8eea67c8a6 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -234,6 +234,19 @@ macro(ov_cpack_settings) set(ir_copyright "generic") endif() + if(ENABLE_OV_JAX_FRONTEND) + set(CPACK_COMPONENT_JAX_DESCRIPTION "OpenVINO JAX Frontend") + set(CPACK_COMPONENT_JAX_DEPENDS "${OV_CPACK_COMP_CORE}") + set(CPACK_DEBIAN_JAX_PACKAGE_NAME "libopenvino-jax-frontend-${cpack_name_ver}") + # since we JAX FE is linkable target, we need to call ldconfig (i.e. `def_triggers`) + set(CPACK_DEBIAN_JAX_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm};${def_triggers}") + ov_debian_add_lintian_suppression(jax + # we have different package name strategy; it suggests libopenvino-jax-frontend202230 + "package-name-doesnt-match-sonames") + list(APPEND frontends jax) + set(jax_copyright "generic") + endif() + if(ENABLE_OV_ONNX_FRONTEND) set(CPACK_COMPONENT_ONNX_DESCRIPTION "OpenVINO ONNX Frontend") set(CPACK_COMPONENT_ONNX_DEPENDS "${OV_CPACK_COMP_CORE}") diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index 6bbeba02bdc11f..4d06ebff496421 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -208,6 +208,15 @@ macro(ov_cpack_settings) set(ir_copyright "generic") endif() + if(ENABLE_OV_JAX_FRONTEND) + set(CPACK_COMPONENT_JAX_DESCRIPTION "OpenVINO JAX Frontend") + set(CPACK_RPM_JAX_PACKAGE_NAME "libopenvino-jax-frontend-${cpack_name_ver}") + set(CPACK_RPM_JAX_POST_INSTALL_SCRIPT_FILE "${def_triggers}") + set(CPACK_RPM_JAX_POST_UNINSTALL_SCRIPT_FILE "${def_triggers}") + _ov_add_package(frontend_packages jax) + set(jax_copyright "generic") + endif() + if(ENABLE_OV_ONNX_FRONTEND) set(CPACK_COMPONENT_ONNX_DESCRIPTION "OpenVINO ONNX Frontend") set(CPACK_RPM_ONNX_PACKAGE_NAME "libopenvino-onnx-frontend-${cpack_name_ver}") diff --git a/cmake/templates/OpenVINOConfig.cmake.in b/cmake/templates/OpenVINOConfig.cmake.in index ac527e969068ef..af8dd1596c4a39 100644 --- a/cmake/templates/OpenVINOConfig.cmake.in +++ b/cmake/templates/OpenVINOConfig.cmake.in @@ -10,6 +10,7 @@ # The following components are supported: # # * `Runtime`: OpenVINO C++ and C Core & Inference Runtime, frontend common +# * `JAX`: OpenVINO JAX frontend # * `ONNX`: OpenVINO ONNX frontend # * `Paddle`: OpenVINO Paddle frontend # * `PyTorch`: OpenVINO PyTorch frontend @@ -37,6 +38,9 @@ # # Frontend specific targets: # +# `openvino::frontend::jax` +# JAX FrontEnd target (optional) +# # `openvino::frontend::onnx` # ONNX FrontEnd target (optional) # @@ -63,6 +67,9 @@ # `OpenVINO_Runtime_FOUND` # OpenVINO C++ Core & Inference Runtime is available # +# `OpenVINO_Frontend_JAX_FOUND` +# OpenVINO JAX frontend is available +# # `OpenVINO_Frontend_ONNX_FOUND` # OpenVINO ONNX frontend is available # @@ -510,7 +517,7 @@ unset(_OV_ENABLE_OPENVINO_BUILD_SHARED) set(_ov_imported_libs openvino::runtime openvino::runtime::c openvino::frontend::onnx openvino::frontend::paddle openvino::frontend::tensorflow - openvino::frontend::pytorch openvino::frontend::tensorflow_lite) + openvino::frontend::pytorch openvino::frontend::tensorflow_lite openvino::frontend::jax) if(_ov_as_external_package) foreach(target IN LISTS _ov_imported_libs) @@ -549,6 +556,7 @@ set(${CMAKE_FIND_PACKAGE_NAME}_TensorFlow_FOUND @ENABLE_OV_TF_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_TensorFlowLite_FOUND @ENABLE_OV_TF_LITE_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_IR_FOUND @ENABLE_OV_IR_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_PyTorch_FOUND @ENABLE_OV_PYTORCH_FRONTEND@) +set(${CMAKE_FIND_PACKAGE_NAME}_JAX_FOUND @ENABLE_OV_JAX_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_ONNX_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_ONNX_FOUND}) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_Paddle_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_Paddle_FOUND}) @@ -556,6 +564,7 @@ set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_TensorFlow_FOUND ${${CMAKE_FIND_PACKAGE_ set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_TensorFlowLite_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_TensorFlowLite_FOUND}) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_IR_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_IR_FOUND}) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_PyTorch_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_PyTorch_FOUND}) +set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_JAX_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_JAX_FOUND}) # if no components specified, only Runtime is provided if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) @@ -591,5 +600,6 @@ unset(${CMAKE_FIND_PACKAGE_NAME}_ONNX_FOUND) unset(${CMAKE_FIND_PACKAGE_NAME}_TensorFlow_FOUND) unset(${CMAKE_FIND_PACKAGE_NAME}_TensorFlowLite_FOUND) unset(${CMAKE_FIND_PACKAGE_NAME}_PyTorch_FOUND) +unset(${CMAKE_FIND_PACKAGE_NAME}_JAX_FOUND) cmake_policy(POP) diff --git a/docs/articles_en/about-openvino/additional-resources.rst b/docs/articles_en/about-openvino/additional-resources.rst deleted file mode 100644 index cb8d0fc62f244a..00000000000000 --- a/docs/articles_en/about-openvino/additional-resources.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. {#resources} - -Additional Resources -==================== - - - -.. meta:: - :description: Learn more about OpenVINO from benchmark results, case studies - and lists of supported models, operations and devices. - -.. toctree:: - :maxdepth: 1 - :hidden: - - additional-resources/glossary - Legal and Responsible AI Information <./additional-resources/legal-information> - additional-resources/telemetry - Case Studies - - -:doc:`Performance Benchmarks ` contain results from benchmarking models with OpenVINO on Intel hardware. - -:doc:`Glossary ` contains terms used in OpenVINO. - -:doc:`Legal and Responsible AI Information ` provides trademark information and other legal statements. - -:doc:`OpenVINO™ Telemetry ` has detailed information on the telemetry data collection. - -`Case Studies `__ are articles about real-world examples of OpenVINO™ usage. - - - - diff --git a/docs/articles_en/about-openvino/compatibility-and-support.rst b/docs/articles_en/about-openvino/compatibility-and-support.rst index f2c467be34676e..f9ea2b2b34ba90 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support.rst @@ -1,5 +1,3 @@ -.. {#compatibility_and_support} - Compatibility and Support ========================= @@ -9,12 +7,15 @@ Compatibility and Support :hidden: compatibility-and-support/supported-devices + compatibility-and-support/supported-models compatibility-and-support/supported-operations-inference-devices compatibility-and-support/supported-operations-framework-frontend :doc:`Supported Devices ` - compatibility information for supported hardware accelerators. +:doc:`Supported Models ` - a list of selected models confirmed to work with given hardware. + :doc:`Supported Operations ` - a listing of framework layers supported by OpenVINO. :doc:`Supported Operations ` - a listing of layers supported by OpenVINO inference devices. diff --git a/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst b/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst index 0c1195063196e9..968c5b6571bca9 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst @@ -1,44 +1,45 @@ -Supported Models -======================== +AI Models that run with OpenVINO +===================================== -.. toctree:: - :maxdepth: 1 - :hidden: +This list of 500+ computer vision, LLM, and GenAI models includes pre-optimized OpenVINO +models/notebooks plus models from public model zoos (ONNX Model Zoo, Pytorch Model Zoo, and +HuggingFace). This list is not comprehensive and only includes models tested by Intel. - about-openvino/performance-benchmarks - about-openvino/compatibility-and-support - Release Notes - Additional Resources +**AI Models that run on Intel® Core Ultra ™ Processors with OpenVINO™ toolkit:** .. raw:: html - +
- + .. csv-table:: :class: modeldata stripe :name: supportedModelsTable :header-rows: 1 - :file: ../../_static/models.csv + :file: ../../_static/download/supported_models.csv + +| Note: +| The results as of June 17 2024, for OpenVINO version 2024.2. +| The validation process involved using the OpenVINO toolkit (natively or as a backend) to load + each model onto the designated hardware and successfully execute inference without encountering + any errors or failures. These successfully run models are indicated with a check mark below and + blanks are not tested. In the precision column where it is listed as optimum-intel default, + that corresponds to FP32 for small models and INT8 for models greater than 1B parameters. diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index ced358db18c044..cc0377f64857e5 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -205,10 +205,10 @@ You can also test performance for your system yourself, following the guide on * Intel® Distribution of OpenVINO™ toolkit performance results are based on release - 2024.2, as of June 17, 2024. + 2024.2, as of June 28, 2024. * OpenVINO Model Server performance results are based on release - 2024.1, as of April 26, 2024. + 2024.2, as of June 28, 2024. The results may not reflect all publicly available updates. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software, or service diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst index a56b623c574a34..73c9f255ff3dbb 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -1,13 +1,15 @@ Most Efficient Large Language Models for AI PC ============================================== -This page is regularly updated to help you identify the best-performing LLMs on the Intel® Core™ Ultra processor family and AI PCs. +This page is regularly updated to help you identify the best-performing LLMs on the +Intel® Core™ Ultra processor family and AI PCs. -The table below lists the key performance indicators for a selection of Large Language Models running on an Intel® Core™ Ultra 7-165H based system. +The table below lists the key performance indicators for a selection of Large Language Models, +running on an Intel® Core™ Ultra 7-165H based system, on built-in GPUs. -.. raw:: html +For complete information on the system config, see: +`Hardware Platforms [PDF] `__ - For complete information on the system config, see: Hardware Platforms [PDF] .. raw:: html @@ -59,7 +61,11 @@ The table below lists the key performance indicators for a selection of Large La :class: modeldata stripe :name: supportedModelsTable :header-rows: 1 - :file: ../../_static/llm_models.csv + :file: ../../_static/download/llm_models.csv +This page is regularly updated to help you identify the best-performing LLMs on the +Intel® Core™ Ultra processor family and AI PCs. +For complete information on the system config, see: +`Hardware Platforms [PDF] `__ \ No newline at end of file diff --git a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst index 868c828845a092..6c731dc374010b 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst @@ -17,7 +17,13 @@ Performance Information F.A.Q. .. dropdown:: Where can I find the models used in the performance benchmarks? - All models used are included in the GitHub repository of :doc:`Open Model Zoo <../../documentation/legacy-features/model-zoo>`. + All models used are included in the GitHub repository of + :doc:`Open Model Zoo <../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. .. dropdown:: Will there be any new models added to the list used for benchmarking? diff --git a/docs/articles_en/about-openvino/release-notes-openvino.rst b/docs/articles_en/about-openvino/release-notes-openvino.rst index 68c64e83a61aa0..c8c32126e3e10c 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino.rst @@ -24,7 +24,8 @@ OpenVINO Release Notes What's new +++++++++++++++++++++++++++++ -* More Gen AI coverage and framework integrations to minimize code changes. +* More :doc:`Gen AI <../learn-openvino/llm_inference_guide/genai-guide>` coverage and framework + integrations to minimize code changes. * Llama 3 optimizations for CPUs, built-in GPUs, and discrete GPUs for improved performance and efficient memory usage. @@ -35,7 +36,7 @@ What's new Python Custom Operation empowers users to implement their own specialized operations into any model. * Notebooks expansion to ensure better coverage for new models. Noteworthy notebooks added: - DynamiCrafter, YOLOv10, and Chatbot notebook with Phi-3. + DynamiCrafter, YOLOv10, Chatbot notebook with Phi-3, and QWEN2. * Broader Large Language Model (LLM) support and more model compression techniques. @@ -52,21 +53,22 @@ What's new * Model Serving Enhancements: - * OpenVINO Model Server (OVMS) now supports OpenAI-compatible API along with Continuous + * Preview: OpenVINO Model Server (OVMS) now supports OpenAI-compatible API along with Continuous Batching and PagedAttention, enabling significantly higher throughput for parallel inferencing, especially on Intel® Xeon® processors, when serving LLMs to many concurrent users. - * OpenVINO backend for Triton Server now supports built-in GPUs and discrete GPUs, in - addition to dynamic shapes support. + * OpenVINO backend for Triton Server now supports dynamic input shapes. * Integration of TorchServe through torch.compile OpenVINO backend for easy model deployment, provisioning to multiple instances, model versioning, and maintenance. - * Addition of the Generate API, a simplified API for text generation using large language - models with only a few lines of code. The API is available through the newly launched - OpenVINO GenAI package. + * Preview: addition of the :doc:`Generate API <../learn-openvino/llm_inference_guide/genai-guide>`, + a simplified API for text generation using large language models with only a few lines of + code. The API is available through the newly launched OpenVINO GenAI package. + * Support for Intel Atom® Processor X Series. For more details, see :doc:`System Requirements <./release-notes-openvino/system-requirements>`. * Preview: Support for Intel® Xeon® 6 processor. + OpenVINO™ Runtime +++++++++++++++++++++++++++++ @@ -77,8 +79,6 @@ Common efficient LLM weight compression. * Common OV headers have been optimized, improving binary compilation time and reducing binary size. -* Support for Intel Atom® Processor X Series has been added. For more details, see - :doc:`System Requirements <./release-notes-openvino/system-requirements>`. AUTO Inference Mode @@ -115,13 +115,14 @@ GPU Device Plugin * Stable Diffusion FP16 performance improved on Core Ultra platforms, with significant pipeline improvement for models with dynamic-shaped input. Memory usage of the pipeline has been reduced, as well. -* Improved performance of optimized permute_f_y kernel. +* Optimized permute_f_y kernel performance has been improved. NPU Device Plugin ----------------------------- -* TBD +* A new set of configuration options is now available. +* Performance increase has been unlocked, with the new `2408 NPU driver `__. OpenVINO Python API @@ -185,8 +186,8 @@ OpenVINO Model Server * OpenVINO Model server can be now used for text generation use cases using OpenAI compatible API. * Added support for continuous batching and PagedAttention algorithms for text generation with - fast and efficient in high concurrency load especially on Intel Xeon processors. Learn more - about it. + fast and efficient in high concurrency load especially on Intel Xeon processors. + `Learn more about it `__. Neural Network Compression Framework @@ -233,10 +234,10 @@ Known Issues | **Component: TBB** | ID: TBB-1400/ TBB-1401 | Description: -| In 2024.2, oneTBB 2021.2.x is used for Intel Distribution OpenVINO Ubuntu and Red Hat - archives, instead of system TBB/oneTBB, which improved the performance on some platforms - (EMR for example), but the latency of some models drop on SPR, user can build OpenVINO with - **-DSYSTEM_TBB=ON** to get better latency performance for these models. +| In 2024.2, oneTBB 2021.2.x is used for Intel Distribution of OpenVINO Ubuntu and Red Hat + archives, instead of system TBB/oneTBB. This improves performance on the new generation of + Xeon platforms but may increase latency of some models on the previous generation. You can + build OpenVINO with **-DSYSTEM_TBB=ON** to get better latency performance for these models. | **Component: python API** | ID: CVS-141744 diff --git a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst index 4d8381ec09bba3..a02d133453c8b9 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst @@ -19,6 +19,7 @@ CPU .. tab-item:: Supported Hardware + * Intel® Xeon® 6 processor (preview) * Intel Atom® Processor X Series * Intel Atom® processor with Intel® SSE4.2 support * Intel® Pentium® processor N4200/5, N3350/5, N3450/5 with Intel® HD Graphics diff --git a/docs/articles_en/documentation/legacy-features.rst b/docs/articles_en/documentation/legacy-features.rst index a470f9bcd9cc4b..489e6e77fe4191 100644 --- a/docs/articles_en/documentation/legacy-features.rst +++ b/docs/articles_en/documentation/legacy-features.rst @@ -59,6 +59,9 @@ offering. until they are fully removed. | :doc:`See the Open Model ZOO documentation ` | `Check the OMZ GitHub project `__ +| As for public model databases, `Hugging Face `__ has + become the recommended model source for OpenVINO. + diff --git a/docs/articles_en/documentation/legacy-features/install-dev-tools.rst b/docs/articles_en/documentation/legacy-features/install-dev-tools.rst index 6466eb8711a381..de2b35529aeafc 100644 --- a/docs/articles_en/documentation/legacy-features/install-dev-tools.rst +++ b/docs/articles_en/documentation/legacy-features/install-dev-tools.rst @@ -8,14 +8,18 @@ Install OpenVINO™ Development Tools :description: Learn how to install OpenVINO™ Development Tools on Windows, Linux, and macOS operating systems, using a PyPi package. -OpenVINO Development Tools is a set of utilities that make it easy to develop and optimize models and applications for OpenVINO. It provides the following tools: +OpenVINO Development Tools is a set of utilities that make it easy to develop and +optimize models and applications for OpenVINO. It provides the following tools: * Model conversion API * Benchmark Tool * Accuracy Checker and Annotation Converter * Model Downloader and other Open Model Zoo tools -The instructions on this page show how to install OpenVINO Development Tools. If you are a Python developer, it only takes a few simple steps to install the tools with PyPI. If you are developing in C/C++, OpenVINO Runtime must be installed separately before installing OpenVINO Development Tools. +The instructions on this page show how to install OpenVINO Development Tools. If you are a +Python developer, it only takes a few simple steps to install the tools with PyPI. If you +are developing in C/C++, OpenVINO Runtime must be installed separately before installing +OpenVINO Development Tools. In both cases, Python 3.8 - 3.11 needs to be installed on your machine before starting. @@ -28,16 +32,28 @@ In both cases, Python 3.8 - 3.11 needs to be installed on your machine before st For Python Developers ##################### -If you are a Python developer, follow the steps in the :ref:`Installing OpenVINO Development Tools ` section on this page to install it. Installing OpenVINO Development Tools will also install OpenVINO Runtime as a dependency, so you don’t need to install OpenVINO Runtime separately. This option is recommended for new users. +If you are a Python developer, follow the steps in the +:ref:`Installing OpenVINO Development Tools ` section on this page to +install it. Installing OpenVINO Development Tools will also install OpenVINO Runtime as +a dependency, so you don’t need to install OpenVINO Runtime separately. This option is +recommended for new users. .. _cpp_developers: For C/C++ Developers ####################### -If you are a C/C++ developer, you must first install OpenVINO Runtime separately to set up the C/C++ libraries, sample code, and dependencies for building applications with OpenVINO. These files are not included with the PyPI distribution. See the :doc:`Selector Tool <../../get-started/install-openvino>` page to install OpenVINO Runtime from an archive file for your operating system. +If you are a C/C++ developer, you must first install OpenVINO Runtime separately to set +up the C/C++ libraries, sample code, and dependencies for building applications with +OpenVINO. These files are not included with the PyPI distribution. See the +:doc:`Selector Tool <../../get-started/install-openvino>` page to install OpenVINO Runtime +from an archive file for your operating system. -Once OpenVINO Runtime is installed, you may install OpenVINO Development Tools for access to tools like ``mo``, Model Downloader, Benchmark Tool, and other utilities that will help you optimize your model and develop your application. Follow the steps in the :ref:`Installing OpenVINO Development Tools ` section on this page to install it. +Once OpenVINO Runtime is installed, you may install OpenVINO Development Tools for access +to tools like ``mo``, Model Downloader, Benchmark Tool, and other utilities that will help +you optimize your model and develop your application. Follow the steps in the +:ref:`Installing OpenVINO Development Tools ` section on this page +to install it. .. _install_dev_tools: @@ -45,14 +61,16 @@ Installing OpenVINO™ Development Tools ###################################### Follow these step-by-step instructions to install OpenVINO Development Tools on your computer. -There are two options to install OpenVINO Development Tools: installation into an existing environment with a deep learning framework that was used -for model training or creation; or installation into a new environment. +There are two options to install OpenVINO Development Tools: installation into an existing +environment with a deep learning framework that was used for model training or creation; +or installation into a new environment. Installation into an Existing Environment with the Source Deep Learning Framework +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -To install OpenVINO Development Tools (see the :ref:`Install the Package ` section of this article) into an existing environment -with the deep learning framework used for the model training or creation, run the following command: +To install OpenVINO Development Tools (see the :ref:`Install the Package ` +section of this article) into an existing environment with the deep learning framework used +for the model training or creation, run the following command: .. code-block:: sh @@ -62,13 +80,16 @@ with the deep learning framework used for the model training or creation, run th Installation in a New Environment +++++++++++++++++++++++++++++++++ -If you do not have an environment with a deep learning framework for the input model or you encounter any compatibility issues between OpenVINO -and your version of deep learning framework, you may install OpenVINO Development Tools with validated versions of frameworks into a new environment. +If you do not have an environment with a deep learning framework for the input model or you +encounter any compatibility issues between OpenVINO and your version of deep learning +framework, you may install OpenVINO Development Tools with validated versions of +frameworks into a new environment. Step 1. Set Up Python Virtual Environment ----------------------------------------- -Create a virtual Python environment to avoid dependency conflicts. To create a virtual environment, use the following command: +Create a virtual Python environment to avoid dependency conflicts. To create a virtual +environment, use the following command: .. tab-set:: @@ -117,7 +138,8 @@ Activate the newly created Python virtual environment by issuing this command: Step 3. Set Up and Update PIP to the Highest Version ---------------------------------------------------- -Make sure `pip` is installed in your environment and upgrade it to the latest version by issuing the following command: +Make sure `pip` is installed in your environment and upgrade it to the latest version by +issuing the following command: .. code-block:: sh @@ -129,7 +151,8 @@ Make sure `pip` is installed in your environment and upgrade it to the latest ve Step 4. Install the Package --------------------------- -To install and configure the components of the development package together with validated versions of specific frameworks, use the commands below. +To install and configure the components of the development package together with validated +versions of specific frameworks, use the commands below. .. code-block:: sh @@ -137,9 +160,11 @@ To install and configure the components of the development package together with where the ``extras`` parameter specifies the source deep learning framework for the input model -and is one or more of the following values separated with "," : ``onnx``, ``pytorch``, ``tensorflow``, ``tensorflow2``. +and is one or more of the following values separated with "," : ``onnx``, ``pytorch``, +``tensorflow``, ``tensorflow2``. -For example, to install and configure dependencies required for working with TensorFlow 2.x and ONNX models, use the following command: +For example, to install and configure dependencies required for working with TensorFlow 2.x +and ONNX models, use the following command: .. code-block:: sh @@ -148,9 +173,14 @@ For example, to install and configure dependencies required for working with Ten .. note:: - Model conversion API support for TensorFlow 1.x environment has been deprecated. Use the ``tensorflow2`` parameter to install a TensorFlow 2.x environment that can convert both TensorFlow 1.x and 2.x models. If your model isn't compatible with the TensorFlow 2.x environment, use the `tensorflow` parameter to install the TensorFlow 1.x environment. The TF 1.x environment is provided only for legacy compatibility reasons. + Model conversion API support for TensorFlow 1.x environment has been deprecated. Use the + ``tensorflow2`` parameter to install a TensorFlow 2.x environment that can convert both + TensorFlow 1.x and 2.x models. If your model isn't compatible with the TensorFlow 2.x + environment, use the `tensorflow` parameter to install the TensorFlow 1.x environment. + The TF 1.x environment is provided only for legacy compatibility reasons. -For more details on the openvino-dev PyPI package, see `pypi.org `__ . +For more details on the openvino-dev PyPI package, see +`pypi.org `__ . Step 5. Test the Installation ------------------------------ @@ -161,9 +191,13 @@ To verify the package is properly installed, run the command below (this may tak mo -h -You will see the help message for ``mo`` if installation finished successfully. If you get an error, refer to the :doc:`Troubleshooting Guide <../../get-started/troubleshooting-install-config>` for possible solutions. +You will see the help message for ``mo`` if installation finished successfully. If you get an +error, refer to the :doc:`Troubleshooting Guide <../../get-started/troubleshooting-install-config>` +for possible solutions. -Congratulations! You finished installing OpenVINO Development Tools with C/C++ capability. Now you can start exploring OpenVINO's functionality through example C/C++ applications. See the "What's Next?" section to learn more! +Congratulations! You finished installing OpenVINO Development Tools with C/C++ capability. +Now you can start exploring OpenVINO's functionality through example C/C++ applications. +See the "What's Next?" section to learn more! What's Next? ############ @@ -176,9 +210,12 @@ Get started with Python .. image:: ../../assets/images/get_started_with_python.gif :width: 400 -Try the `Python Quick Start Example <../../notebooks/vision-monodepth-with-output.html>`__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. +Try the `Python Quick Start Example <../../notebooks/vision-monodepth-with-output.html>`__ +to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook +inside your web browser. -Visit the :doc:`Tutorials <../../learn-openvino/interactive-tutorials-python>` page for more Jupyter Notebooks to get you started with OpenVINO, such as: +Visit the :doc:`Tutorials <../../learn-openvino/interactive-tutorials-python>` page for more +Jupyter Notebooks to get you started with OpenVINO, such as: * `OpenVINO Python API Tutorial <../../notebooks/openvino-api-with-output.html>`__ * `Basic image classification program with Hello Image Classification <../../notebooks/hello-world-with-output.html>`__ @@ -191,9 +228,11 @@ Get started with C++ :width: 400 -Try the :doc:`C++ Quick Start Example <../../learn-openvino/openvino-samples/get-started-demos>` for step-by-step instructions on building and running a basic image classification C++ application. +Try the :doc:`C++ Quick Start Example <../../learn-openvino/openvino-samples/get-started-demos>` +for step-by-step instructions on building and running a basic image classification C++ application. -Visit the :doc:`Samples <../../learn-openvino/openvino-samples>` page for other C++ example applications to get you started with OpenVINO, such as: +Visit the :doc:`Samples <../../learn-openvino/openvino-samples>` page for other C++ +example applications to get you started with OpenVINO, such as: * :doc:`Basic object detection with the Hello Reshape SSD C++ sample <../../learn-openvino/openvino-samples/hello-reshape-ssd>` * :doc:`Object classification sample <../../learn-openvino/openvino-samples/hello-classification>` @@ -201,10 +240,20 @@ Visit the :doc:`Samples <../../learn-openvino/openvino-samples>` page for other Learn OpenVINO Development Tools ++++++++++++++++++++++++++++++++ -* Explore a variety of pre-trained deep learning models in the :doc:`Open Model Zoo ` and deploy them in demo applications to see how they work. -* Want to import a model from another framework and optimize its performance with OpenVINO? Visit the :doc:`Convert a Model ` page. -* Accelerate your model's speed even further with quantization and other compression techniques using :doc:`Neural Network Compression Framework (NNCF) <../../openvino-workflow/model-optimization-guide/quantizing-models-post-training>`. -* Benchmark your model's inference speed with one simple command using the :doc:`Benchmark Tool <../../learn-openvino/openvino-samples/benchmark-tool>`. +* Explore a variety of pre-trained deep learning models in the + :doc:`Open Model Zoo ` and deploy them in demo applications to see how they work. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + +* Want to import a model from another framework and optimize its performance with OpenVINO? + Visit the :doc:`Convert a Model ` page. +* Accelerate your model's speed even further with quantization and other compression techniques + using :doc:`Neural Network Compression Framework (NNCF) <../../openvino-workflow/model-optimization-guide/quantizing-models-post-training>`. +* Benchmark your model's inference speed with one simple command using the + :doc:`Benchmark Tool <../../learn-openvino/openvino-samples/benchmark-tool>`. Additional Resources #################### diff --git a/docs/articles_en/documentation/legacy-features/model-zoo.rst b/docs/articles_en/documentation/legacy-features/model-zoo.rst index 281e217f75c9c4..f2b26f2a983efc 100644 --- a/docs/articles_en/documentation/legacy-features/model-zoo.rst +++ b/docs/articles_en/documentation/legacy-features/model-zoo.rst @@ -29,9 +29,29 @@ Model Zoo ../../omz_model_api_ovms_adapter -Open Model Zoo for OpenVINO™ toolkit delivers a wide variety of free, pre-trained deep learning models and demo applications that provide full application templates to help you implement deep learning in Python, C++, or OpenCV Graph API (G-API). Models and demos are available in the `Open Model Zoo GitHub repo `__ and licensed under Apache License Version 2.0. - -Browse through over 200 neural network models, both :doc:`public <../../omz_models_group_public>` and from :doc:`Intel <../../omz_models_group_intel>`, and pick the right one for your solution. Types include object detection, classification, image segmentation, handwriting recognition, text to speech, pose estimation, and others. The Intel models have already been converted to work with OpenVINO™ toolkit, while public models can easily be converted using the :doc:`OpenVINO Model Conversion API <../../openvino-workflow/model-preparation>` utility. - -Get started with simple :doc:`step-by-step procedures <../../learn-openvino/openvino-samples/get-started-demos>` to learn how to build and run demo applications or discover the :doc:`full set of demos <../../omz_demos>` and adapt them for implementing specific deep learning scenarios in your applications. +.. note:: + + Since the deprecation of Open Model Zoo, OpenVINO has significantly extended its presence on the + `Hugging Face `__ model repository. It is currently + the recommended source of optimized OpenVINO IR models. + +Open Model Zoo for OpenVINO™ toolkit delivers a wide variety of free, pre-trained deep learning +models and demo applications that provide full application templates to help you implement deep +learning in Python, C++, or OpenCV Graph API (G-API). Models and demos are available in the +`Open Model Zoo GitHub repo `__ +and licensed under Apache License Version 2.0. + +Browse through over 200 neural network models, both +:doc:`public <../../omz_models_group_public>` and from +:doc:`Intel <../../omz_models_group_intel>`, and pick the right one for your solution. +Types include object detection, classification, image segmentation, handwriting recognition, +text to speech, pose estimation, and others. The Intel models have already been converted +to work with OpenVINO™ toolkit, while public models can easily be converted using the +:doc:`OpenVINO Model Conversion API <../../openvino-workflow/model-preparation>` utility. + +Get started with simple +:doc:`step-by-step procedures <../../learn-openvino/openvino-samples/get-started-demos>` +to learn how to build and run demo applications or discover the +:doc:`full set of demos <../../omz_demos>` and adapt them for implementing specific deep +learning scenarios in your applications. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst index 6cc2de65a8a9a0..fb7e3004258253 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst @@ -53,7 +53,7 @@ For example, to add the description of the ``CustomReshape`` layer, which is an python3 generate_caffe_pb2.py --input_proto /src/caffe/proto/caffe.proto - where ``PATH_TO_CUSTOM_CAFFE` is the path to the root directory of custom Caffe. + where ``PATH_TO_CUSTOM_CAFFE`` is the path to the root directory of custom Caffe. 3. Now, Model Optimizer is able to load the model into memory and start working with your extensions if there are any. @@ -927,7 +927,7 @@ Q102. What does the message "Operation _contrib_box_nms is not supported ..." me .. _question-103: -Q103. What does the message "ModelOptimizer is not able to parse *.caffemodel" mean? +Q103. What does the message "ModelOptimizer is not able to parse "\*.caffemodel" mean? ##################################################################################################################################################### **A:** If a ``*.caffemodel`` file exists and is correct, the error occurred possibly because of the use of Python protobuf implementation. In some cases, error messages may appear during model parsing, for example: "``utf-8`` codec can't decode byte 0xe0 in position 4: invalid continuation byte in field: mo_caffe.SpatialTransformerParameter.transform_type". You can either use a newer Python version (3.8 - 3.11) or build the ``cpp`` implementation of ``protobuf`` yourself for your version of Python. For the complete instructions about building ``protobuf`` from sources, see the appropriate section in the :doc:`Converting Models with Model Optimizer <../legacy-conversion-api>` guide. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst index 71c28a5db9205d..45157106165732 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst @@ -42,8 +42,8 @@ To download the model, follow the instruction below: 2. Download the `TensorFlow MetaGraph with pre-trained weights `__. 3. Unpack it with a file archiver application. -Freezing the Model into a *.pb File -################################### +Freezing the Model into a "\*.pb File" +###################################### After unpacking the archives above, you have to freeze the model. This requires TensorFlow version 1, which is not available under Python 3.8, so you need Python 3.7 or lower. @@ -75,7 +75,7 @@ a fixed input length shape, thus the model is not reshapable. Refer to the :doc:`Using Shape Inference <../../../../../../openvino-workflow/running-inference/changing-input-shape>` guide. The second is that the frozen model still has two variables: ``previous_state_c`` and ``previous_state_h``, figure -with the frozen *.pb model is below. It means that the model keeps training these variables at each inference. +with the frozen \*.pb model is below. It means that the model keeps training these variables at each inference. .. image:: ../../../../../../assets/images/DeepSpeech-0.8.2.png diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst index 47a6342ff7c989..daebe992d8441c 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst @@ -5,7 +5,7 @@ Converting TensorFlow EfficientDet Models .. meta:: - :description: Learn how to convert an EfficientDet model + :description: Learn how to convert an EfficientDet model from TensorFlow to the OpenVINO Intermediate Representation. @@ -14,7 +14,7 @@ Converting TensorFlow EfficientDet Models The code described here has been **deprecated!** Do not use it to avoid working with a legacy solution. It will be kept for some time to ensure backwards compatibility, but **you should not use** it in contemporary applications. This guide describes a deprecated conversion method. The guide on the new and recommended method can be found in the :doc:`Python tutorials <../../../../../../learn-openvino/interactive-tutorials-python>`. - + This tutorial explains how to convert EfficientDet public object detection models to the Intermediate Representation (IR). .. _efficientdet-to-ir: @@ -55,9 +55,15 @@ The attribute ``image_size`` specifies the shape to be defined for the model con The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the ``RGB<->BGR`` conversion specifying the command-line parameter: ``--reverse_input_channels``. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the :doc:`Converting a Model to Intermediate Representation (IR) <../../[legacy]-setting-input-shapes>` guide. -OpenVINO toolkit provides samples that can be used to infer EfficientDet model. +OpenVINO toolkit provides samples that can be used to infer EfficientDet model. For more information, refer to the :doc:`Open Model Zoo Demos <../../../../../../omz_demos>`. +.. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format have are now + published on `Hugging Face `__. + + Interpreting Results of the TensorFlow Model and the IR ####################################################### diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst index 3eb5a93e6e0feb..8b1a04029962b1 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst @@ -89,6 +89,11 @@ Speech Recognition, Natural Language Processing and others. Refer to the links b * :doc:`OpenVINO Samples <../../../../../../learn-openvino/openvino-samples>` * :doc:`Open Model Zoo Demos <../../../../model-zoo>` +.. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + Feeding Input Images to the Samples ################################### diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst index 10946ce84100da..bf1668a89007c9 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst @@ -236,7 +236,7 @@ To convert DarkNet YOLOv1 and YOLOv2 models to the OpenVINO format, follow these Installing DarkFlow --------------------------------------------------------------- ++++++++++++++++++++++ You need DarkFlow to convert YOLOv1 and YOLOv2 models to TensorFlow. To install DarkFlow: @@ -259,7 +259,7 @@ You need DarkFlow to convert YOLOv1 and YOLOv2 models to TensorFlow. To install Converting a DarkNet YOLOv1 or YOLOv2 Model to TensorFlow --------------------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ To convert YOLOv1 or YOLOv2 model to TensorFlow, go to the root directory of the cloned DarkFlow repository, place the previously downloaded \*.cfg and \*.weights files in the current directory and run the following command: @@ -292,7 +292,7 @@ in ``built_graph`` subdirectory of the cloned DarkFlow repository. File ``.pb`` is a TensorFlow representation of the YOLO model. Converting a TensorFlow YOLOv1 or YOLOv2 Model to the IR ---------------------------------------------------------- +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Converted TensorFlow YOLO model is missing ``Region`` layer and its parameters. Original YOLO ``Region`` layer parameters are stored in the configuration ``/.cfg`` file under the ``[region]`` title. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst index b2be35f4452832..f742b4531f93d0 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst @@ -71,7 +71,7 @@ the ``mo.graph.port.Port`` class). The ``Port`` object has several attributes: * ``node`` - the instance of the ``Node`` object the port belongs to. * ``idx`` - the port number. Input and output ports are numbered independently, starting from ``0``. Thus, -:doc:`ReLU <../../../openvino-ir-format/operation-sets/operation-specs/activation/relu-1>` operation has one input port (with index ``0``) and one output port (with index ``0``). + :doc:`ReLU <../../../openvino-ir-format/operation-sets/operation-specs/activation/relu-1>` operation has one input port (with index ``0``) and one output port (with index ``0``). * ``type`` - the type of the port. Could be equal to either ``"in"`` or ``"out"``. * ``data`` - the object that should be used to get attributes of the corresponding data node. This object has methods ``get_shape()`` / ``set_shape()`` and ``get_value()`` / ``set_value()`` to get/set shape/value of the corresponding data node. For example, ``in_port.data.get_shape()`` returns an input shape of a tensor connected to input port ``in_port`` (``in_port.type == 'in'``), ``out_port.data.get_value()`` returns a value of a tensor produced from output port ``out_port`` (``out_port.type == 'out'``). diff --git a/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst b/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst index c6b257ae3f17ca..e582b9359d227c 100644 --- a/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst +++ b/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst @@ -21,8 +21,6 @@ Plus, enjoy `Jupyter notebooks /cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml`` file. -* Call the ``ov::Core::set_property()`` method from your application with the ``"CONFIG_FILE"`` key and the configuration file name as a value before loading the network that uses custom operations to the plugin: +* Include a section with your kernels into the automatically-loaded + ``/cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml`` file. +* Call the ``ov::Core::set_property()`` method from your application with the + ``"CONFIG_FILE"`` key and the configuration file name as a value before loading + the network that uses custom operations to the plugin: .. tab-set:: @@ -35,14 +42,21 @@ There are two options for using the custom operation configuration file: :fragment: [part0] -All OpenVINO samples, except the trivial ``hello_classification``, and most Open Model Zoo demos -feature a dedicated command-line option ``-c`` to load custom kernels. For example, to load custom operations for the classification sample, run the command below: +All OpenVINO samples, except the trivial ``hello_classification``, and most Open +Model Zoo demos feature a dedicated command-line option ``-c`` to load custom kernels. +For example, to load custom operations for the classification sample, run the command below: .. code-block:: cpp $ ./classification_sample -m /bvlc_alexnet_fp16.xml -i ./validation_set/daily/227x227/apron.bmp -d GPU -c /custom_layer_example.xml +.. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + + .. _config-file-format: Configuration File Format @@ -80,7 +94,8 @@ The ``CustomLayer`` node contains the entire configuration for a single custom o - Description * - ``name`` - (1) - - The name of the operation type to be used. This name should be identical to the type used in the IR. + - The name of the operation type to be used. This name should be identical to + the type used in the IR. * - ``type`` - (1) - Must be ``SimpleGPU`` . @@ -111,7 +126,8 @@ The ``Source`` node points to a single OpenCL source file. - Description * - ``filename`` - (1) - - Name of the file containing OpenCL source code. The path is relative to your executable. Multiple source nodes will have their sources concatenated in order. + - Name of the file containing OpenCL source code. The path is relative to your + executable. Multiple source nodes will have their sources concatenated in order. **Sub-nodes**: None @@ -129,16 +145,19 @@ the sources during compilation (JIT). - Description * - ``name`` - (1) - - The name of the defined JIT. For static constants, this can include the value as well, which is taken as a string. + - The name of the defined JIT. For static constants, this can include the value + as well, which is taken as a string. * - ``param`` - (0/1) - This parameter value is used as the value of this JIT definition. * - ``type`` - (0/1) - - The parameter type. Accepted values: ``int`` , ``float`` , and ``int[]`` , ``float[]`` for arrays. + - The parameter type. Accepted values: ``int`` , ``float`` , and ``int[]`` , + ``float[]`` for arrays. * - ``default`` - (0/1) - - The default value to be used if the specified parameters are missing from the operation in the OpenVINO IR. + - The default value to be used if the specified parameters are missing from the + operation in the OpenVINO IR. **Sub-nodes:** None @@ -197,7 +216,8 @@ The ``Tensor`` node configures a single input or output tensor. - 0-based index in the operation input/output ports in the OpenVINO IR * - ``format`` - (0/1) - - Data layout declaration for the tensor. Accepted values: ``BFYX`` , ``BYXF`` , ``YXFB`` , ``FYXB`` , and same values in all lowercase. Default value: ``BFYX``. + - Data layout declaration for the tensor. Accepted values: ``BFYX`` , ``BYXF`` , + ``YXFB`` , ``FYXB`` , and same values in all lowercase. Default value: ``BFYX``. CompilerOptions Node and Sub-Node Structure +++++++++++++++++++++++++++++++++++++++++++ @@ -231,10 +251,14 @@ queuing an OpenCL program for execution. - Description * - ``global`` ``local`` - (0/1) (0/1) - - An array of up to three integers or formulas for defining OpenCL work-sizes to be used during execution. The formulas can use the values of the B,F,Y,X dimensions and contain the operators: +,-,/,\*,%. All operators are evaluated in integer arithmetic. Default value: ``global=”B\*F\*Y\*X” local=””`` + - An array of up to three integers or formulas for defining OpenCL work-sizes to + be used during execution. The formulas can use the values of the B,F,Y,X + dimensions and contain the operators: +,-,/,\*,%. All operators are evaluated + in integer arithmetic. Default value: ``global=”B\*F\*Y\*X” local=””`` * - ``dim`` - (0/1) - - A tensor to take the work-size from. Accepted values: ``input N`` , ``output`` , where ``N`` is an index of input tensor starting with 0. Default value: ``output`` + - A tensor to take the work-size from. Accepted values: ``input N`` , ``output`` , + where ``N`` is an index of input tensor starting with 0. Default value: ``output`` **Sub-nodes**: None @@ -242,7 +266,8 @@ Example Configuration File ########################## The following code sample provides an example configuration file in XML -format. For information on the configuration file structure, see the `Configuration File Format <#config-file-format>`__. +format. For information on the configuration file structure, see the +`Configuration File Format <#config-file-format>`__. .. code-block:: xml :force: @@ -291,21 +316,27 @@ For an example, see `Example Kernel <#example-kernel>`__. * - ``_TYPE`` - The datatype of the tensor: ``float`` , ``half`` , or ``char`` * - ``_FORMAT_`` - - The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is concatenated to the defined name. You can use the tensor format to define codepaths in your code with ``#ifdef/#endif`` . + - The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is + concatenated to the defined name. You can use the tensor format to define + codepaths in your code with ``#ifdef/#endif`` . * - ``_LOWER_PADDING`` - - An array of padding elements used for the tensor dimensions before they start. Always ordered as BFYX. + - An array of padding elements used for the tensor dimensions before they start. + Always ordered as BFYX. * - ``_LOWER_PADDING_SIZE`` - The size of the ``_LOWER_PADDING`` array * - ``_UPPER_PADDING`` - - An array of padding elements used for the tensor dimensions after they end. Always ordered as BFYX. + - An array of padding elements used for the tensor dimensions after they end. + Always ordered as BFYX. * - ``_UPPER_PADDING_SIZE`` - The size of the ``_UPPER_PADDING`` array * - ``_PITCHES`` - - The offset (in elements) between adjacent elements in each dimension. Always ordered as BFYX. + - The offset (in elements) between adjacent elements in each dimension. + Always ordered as BFYX. * - ``_PITCHES_SIZE`` - The size of the ``_PITCHES`` array * - ``_OFFSET`` - - The number of elements from the start of the tensor to the first valid element, bypassing the lower padding. + - The number of elements from the start of the tensor to the first valid element, + bypassing the lower padding. All ```` values are automatically defined for every tensor bound to this operation, such as ``INPUT0``, ``INPUT1``, and ``OUTPUT0``, as shown @@ -345,7 +376,9 @@ Example Kernel .. _debugging-tips: .. note:: - As described in the previous section, all items such as the ``INPUT0_TYPE`` are actually defined as OpenCL (pre-)compiler inputs by OpenVINO for efficiency reasons. See the `Debugging Tips <#debugging-tips>`__ below for information on debugging the results. + As described in the previous section, all items such as the ``INPUT0_TYPE`` are + actually defined as OpenCL (pre-)compiler inputs by OpenVINO for efficiency reasons. + See the `Debugging Tips <#debugging-tips>`__ below for information on debugging the results. Debugging Tips ############## @@ -359,5 +392,6 @@ your output can be truncated to fit the buffer. Also, because of buffering, you actually get an entire buffer of output when the execution ends. -For more information, refer to the `printf Function `__. +For more information, refer to the +`printf Function `__. diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst index 135dc2516c0276..5c36e8b509ebd6 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst @@ -15,10 +15,12 @@ Step 1. Prerequisites Transformations PullReshapeThroughDequantization PullTransposeThroughDequantization LinOpSequenceFusion + ConvertSubtractConstant Prerequisites transformations are optional. The transformations prepare a model before running other low precision transformations. The transformations do not operate with dequantization operations or update precisions. Prerequisites transformations include: * :doc:`PullReshapeThroughDequantization ` * :doc:`PullTransposeThroughDequantization ` * :doc:`LinOpSequenceFusion ` +* :doc:`ConvertSubtractConstant ` diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst index 073ccc1c3d0da4..cfdd53d348a259 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst @@ -26,6 +26,7 @@ Step 2. Markup Transformations CreatePrecisionsDependentAttribute PropagateThroughPrecisionPreserved PropagateToInput + PropagateSharedValue UpdateSharedPrecisionPreserved This step defines the optimal ``FakeQuantize`` decomposition precisions for the best inference performance via operations markup with runtime attribute instances. Attributes are created for input and output ports and operations. Transformations do not change the operation output port precisions. A model markup low precision logic is decomposed and implemented into the following common markup transformations. The order of transformations is important: @@ -36,8 +37,9 @@ This step defines the optimal ``FakeQuantize`` decomposition precisions for the 4. :doc:`MarkupPerTensorQuantization ` 5. :doc:`MarkupAvgPoolPrecisionPreserved ` 6. :doc:`PropagatePrecisions ` -7. :doc:`AlignQuantizationIntervals ` -8. :doc:`AlignQuantizationParameters ` +7. :doc:`PropagateSharedValue ` +8. :doc:`AlignQuantizationIntervals ` +9. :doc:`AlignQuantizationParameters ` .. list-table:: :header-rows: 1 diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst index bd5c449877ad87..121d19001f2a51 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst @@ -20,7 +20,7 @@ Step 3. Main Transformations ConcatTransformation ConvolutionTransformation ConvolutionBackpropDataTransformation - DepthToSpaceTransformation + DepthToSpaceTransformation FakeQuantizeDecompositionTransformation FakeQuantizeTransformation InterpolateTransformation @@ -29,6 +29,7 @@ Step 3. Main Transformations MatMulTransformation MaxPoolTransformation MultiplyPartialTransformation + MultiplyTransformation MVNTransformation NormalizeL2Transformation PadTransformation @@ -44,6 +45,7 @@ Step 3. Main Transformations ShuffleChannelsTransformation SplitTransformation StridedSliceTransformation + SubtractTransformation TransposeTransformation UnsqueezeTransformation VariadicSplitTransformation @@ -67,6 +69,7 @@ Main transformations are the majority of low precision transformations. Transfor * :doc:`MatMulTransformation ` * :doc:`MaxPoolTransformation ` * :doc:`MultiplyPartialTransformation ` +* :doc:`MultiplyTransformation ` * :doc:`MVNTransformation ` * :doc:`NormalizeL2Transformation ` * :doc:`PadTransformation ` @@ -82,6 +85,7 @@ Main transformations are the majority of low precision transformations. Transfor * :doc:`ShuffleChannelsTransformation ` * :doc:`SplitTransformation ` * :doc:`StridedSliceTransformation ` +* :doc:`SubtractTransformation ` * :doc:`TransposeTransformation ` * :doc:`UnsqueezeTransformation ` * :doc:`VariadicSplitTransformation ` diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/compiled-model.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/compiled-model.rst index d3c7d5d9c3e5a7..4bdbfa0ab4a189 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/compiled-model.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/compiled-model.rst @@ -8,19 +8,20 @@ Compiled Model :description: Use the ov::CompiledModel class as the base class for a compiled model and to create an arbitrary number of ov::InferRequest objects. -ov::CompiledModel class functionality: +``ov::CompiledModel`` class functionality: -* Compile an ov::Model instance to a backend specific graph representation -* Create an arbitrary number of ov::InferRequest objects -* Hold some common resources shared between different instances of ov::InferRequest. For example: +* Compile an ``ov::Model`` instance to a backend specific graph representation +* Create an arbitrary number of ``ov::InferRequest`` objects +* Hold some common resources shared between different instances of ``ov::InferRequest``. For example: - * ov::ICompiledModel::m_task_executor task executor to implement asynchronous execution - * ov::ICompiledModel::m_callback_executor task executor to run an asynchronous inference request callback in a separate thread + * ``ov::ICompiledModel::m_task_executor`` task executor to implement asynchronous execution + * ``ov::ICompiledModel::m_callback_executor`` task executor to run an asynchronous inference request callback in a separate thread CompiledModel Class ################### -OpenVINO Plugin API provides the interface ov::ICompiledModel which should be used as a base class for a compiled model. Based on that, a declaration of an compiled model class can look as follows: +OpenVINO Plugin API provides the interface ``ov::ICompiledModel`` which should be used as a base class +for a compiled model. Based on that, a declaration of an compiled model class can look as follows: .. doxygensnippet:: src/plugins/template/src/compiled_model.hpp :language: cpp @@ -32,9 +33,12 @@ Class Fields The example class has several fields: -* ``m_request_id`` - Tracks a number of created inference requests, which is used to distinguish different inference requests during profiling via the Intel® Instrumentation and Tracing Technology (ITT) library. +* ``m_request_id`` - Tracks a number of created inference requests, which is used to distinguish + different inference requests during profiling via the Intel® Instrumentation and Tracing Technology (ITT) library. * ``m_cfg`` - Defines a configuration a compiled model was compiled with. -* ``m_model`` - Keeps a reference to transformed ``ov::Model`` which is used in OpenVINO reference backend computations. Note, in case of other backends with backend specific graph representation ``m_model`` has different type and represents backend specific graph or just a set of computational kernels to perform an inference. +* ``m_model`` - Keeps a reference to transformed ``ov::Model`` which is used in OpenVINO reference + backend computations. Note, in case of other backends with backend specific graph representation + ``m_model`` has different type and represents backend specific graph or just a set of computational kernels to perform an inference. * ``m_loaded_from_cache`` - Allows to understand that model was loaded from cache. CompiledModel Constructor @@ -51,7 +55,11 @@ The implementation ``compile_model()`` is fully device-specific. compile_model() +++++++++++++++ -The function accepts a const shared pointer to ``ov::Model`` object and applies OpenVINO passes using ``transform_model()`` function, which defines plugin-specific conversion pipeline. To support low precision inference, the pipeline can include Low Precision Transformations. These transformations are usually hardware specific. You can find how to use and configure Low Precisions Transformations in :doc:`Low Precision Transformations ` guide. +The function accepts a const shared pointer to ``ov::Model`` object and applies OpenVINO passes +using ``transform_model()`` function, which defines plugin-specific conversion pipeline. To support +low precision inference, the pipeline can include Low Precision Transformations. These +transformations are usually hardware specific. You can find how to use and configure Low Precisions +Transformations in :doc:`Low Precision Transformations ` guide. .. doxygensnippet:: src/plugins/template/src/compiled_model.cpp :language: cpp @@ -65,7 +73,8 @@ The function accepts a const shared pointer to ``ov::Model`` object and applies export_model() ++++++++++++++ -The implementation of the method should write all data to the ``model_stream``, which is required to import a backend specific graph later in the ``Plugin::import_model`` method: +The implementation of the method should write all data to the ``model_stream``, which is required +to import a backend specific graph later in the ``Plugin::import_model`` method: .. doxygensnippet:: src/plugins/template/src/compiled_model.cpp :language: cpp @@ -80,14 +89,25 @@ The method creates an synchronous inference request and returns it. :language: cpp :fragment: [compiled_model:create_sync_infer_request] -While the public OpenVINO API has a single interface for inference request, which can be executed in synchronous and asynchronous modes, a plugin library implementation has two separate classes: +While the public OpenVINO API has a single interface for inference request, which can be executed +in synchronous and asynchronous modes, a plugin library implementation has two separate classes: * :doc:`Synchronous inference request `, which defines pipeline stages and runs them synchronously in the ``infer`` method. -* :doc:`Asynchronous inference request `, which is a wrapper for a synchronous inference request and can run a pipeline asynchronously. Depending on a device pipeline structure, it can have one or several stages: - - * For single-stage pipelines, there is no need to define this method and create a class derived from ov::IAsyncInferRequest. For single stage pipelines, a default implementation of this method creates ov::IAsyncInferRequest wrapping a synchronous inference request and runs it asynchronously in the ``m_request_executor`` executor. - * For pipelines with multiple stages, such as performing some preprocessing on host, uploading input data to a device, running inference on a device, or downloading and postprocessing output data, schedule stages on several task executors to achieve better device use and performance. You can do it by creating a sufficient number of inference requests running in parallel. In this case, device stages of different inference requests are overlapped with preprocessing and postprocessing stage giving better performance. +* :doc:`Asynchronous inference request `, which is a wrapper for a synchronous + inference request and can run a pipeline asynchronously. Depending on a device pipeline structure, + it can have one or several stages: + + * For single-stage pipelines, there is no need to define this method and create a class derived + from ``ov::IAsyncInferRequest``. For single stage pipelines, a default implementation of this + method creates ``ov::IAsyncInferRequest`` wrapping a synchronous inference request and runs + it asynchronously in the ``m_request_executor`` executor. + * For pipelines with multiple stages, such as performing some preprocessing on host, uploading + input data to a device, running inference on a device, or downloading and postprocessing output + data, schedule stages on several task executors to achieve better device use and performance. + You can do it by creating a sufficient number of inference requests running in parallel. + In this case, device stages of different inference requests are overlapped with preprocessing + and postprocessing stage giving better performance. .. important:: diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst index 3705f0e4a285d1..ff3bdf878d1b13 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst @@ -5,7 +5,7 @@ Plugin Testing .. meta:: - :description: Use the openvino::funcSharedTests library, which includes + :description: Use the openvino::funcSharedTests library, which includes a predefined set of functional tests and utilities to verify a plugin. @@ -22,13 +22,13 @@ Test definitions are split into tests class declaration (see ``src/tests/functio 2. **Single layer tests** (``single_layer_tests`` sub-folder). This groups of tests checks that a particular single layer can be inferenced on a device. An example of test instantiation based on test definition from ``openvino::funcSharedTests`` library: * From the declaration of convolution test class we can see that it's a parametrized GoogleTest based class with the ``convLayerTestParamsSet`` tuple of parameters: - -.. doxygensnippet:: src/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/convolution.hpp + +.. doxygensnippet:: src/tests/functional/shared_test_classes/include/shared_test_classes/single_op/convolution.hpp :language: cpp :fragment: [test_convolution:definition] * Based on that, define a set of parameters for ``Template`` plugin functional test instantiation: - + .. doxygensnippet:: src/plugins/template/tests/functional/shared_tests_instances/single_layer_tests/convolution.cpp :language: cpp :fragment: [test_convolution:declare_parameters] @@ -41,7 +41,7 @@ Test definitions are split into tests class declaration (see ``src/tests/functio 3. **Sub-graph tests** (``subgraph_tests`` sub-folder). This group of tests is designed to tests small patterns or combination of layers. E.g. when a particular topology is being enabled in a plugin e.g. TF ResNet-50, there is no need to add the whole topology to test tests. In opposite way, a particular repetitive subgraph or pattern can be extracted from ``ResNet-50`` and added to the tests. The instantiation of the sub-graph tests is done in the same way as for single layer tests. -.. note:: +.. note:: Such sub-graphs or patterns for sub-graph tests should be added to ``openvino::ov_models`` library first (this library is a pre-defined set of small ``ov::Model``) and re-used in sub-graph tests after. @@ -55,7 +55,7 @@ Test definitions are split into tests class declaration (see ``src/tests/functio To use these tests for your own plugin development, link the ``openvino::funcSharedTests`` library to your test binary and instantiate required test cases with desired parameters values. .. note:: - + A plugin may contain its own tests for use cases that are specific to hardware or need to be extensively tested. To build test binaries together with other build artifacts, use the ``make all`` command. For details, see :doc:`Build Plugin Using CMake `. @@ -67,7 +67,7 @@ OpenVINO Plugin tests are open for contribution. Add common test case definitions applicable for all plugins to the ``openvino::funcSharedTests`` target within the OpenVINO repository. Then, any other plugin supporting corresponding functionality can instantiate the new test. .. note:: - + When implementing a new subgraph test, add new single-layer tests for each operation of the subgraph if such test does not exist. diff --git a/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst b/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst index c179e628f6d0db..840b93adddc75f 100644 --- a/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst +++ b/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst @@ -18,14 +18,17 @@ Overview of Transformations API transformation-api/graph-rewrite-pass transformation-api/patterns-python-api -OpenVINO Transformation mechanism allows to develop transformation passes to modify ``ov::Model``. You can use this mechanism to apply additional optimizations to the original Model or transform unsupported subgraphs and operations to new operations supported by the plugin. +OpenVINO Transformation mechanism allows to develop transformation passes to modify ``ov::Model``. +You can use this mechanism to apply additional optimizations to the original Model or transform +unsupported subgraphs and operations to new operations supported by the plugin. This guide contains all the necessary information to start implementing OpenVINO™ transformations. Working with Model ################## Before moving to the transformation part, it is important to say a few words about the functions which allow modifying ``ov::Model``. -This section extends the :doc:`model representation guide <../../openvino-workflow/running-inference/integrate-openvino-with-your-application/model-representation>` and introduces an API for ``ov::Model`` manipulation. +This section extends the :doc:`model representation guide <../../openvino-workflow/running-inference/integrate-openvino-with-your-application/model-representation>` +and introduces an API for ``ov::Model`` manipulation. Working with node input and output ports ++++++++++++++++++++++++++++++++++++++++ @@ -33,7 +36,8 @@ Working with node input and output ports Each OpenVINO operation has ``ov::Node`` input and output ports, except for ``Parameter`` and ``Constant`` types. The terms ``node`` and ``operation`` are used interchangeably in OpenVINO, but this article will maintain consistency in their use. -Every port is associated with a node, allowing access to the node it belongs to, including its shape, type, all consumers for output ports and the producer node for input ports. +Every port is associated with a node, allowing access to the node it belongs to, including +its shape, type, all consumers for output ports and the producer node for input ports. Take a look at the code example: @@ -44,7 +48,8 @@ Take a look at the code example: Node replacement ++++++++++++++++ -OpenVINO™ provides two ways for node replacement: via OpenVINO™ helper function and directly via port methods. We are going to review both of them. +OpenVINO™ provides two ways for node replacement: via OpenVINO™ helper function and directly +via port methods. We are going to review both of them. Let's start with OpenVINO™ helper functions. The most popular function is ``ov::replace_node(old_node, new_node)``. @@ -56,7 +61,8 @@ Let's review a replacement case where a Negative operation is replaced with Mult :language: cpp :fragment: [ov:replace_node] -``ov::replace_node`` has a constraint that number of output ports for both nodes must be the same. Otherwise, the attempt to replace the nodes will result in an exception. +``ov::replace_node`` has a constraint that number of output ports for both nodes must be the same. +Otherwise, the attempt to replace the nodes will result in an exception. The alternative way to do the same replacement is the following: @@ -109,7 +115,8 @@ Transformation conditional compilation Transformation library has two internal macros to support conditional compilation feature. -* ``MATCHER_SCOPE(region)`` - allows to disable the MatcherPass if matcher isn't used. The region name should be unique. This macro creates a local variable ``matcher_name`` which you should use as a matcher name. +* ``MATCHER_SCOPE(region)`` - allows to disable the MatcherPass if matcher isn't used. The region + name should be unique. This macro creates a local variable ``matcher_name`` which you should use as a matcher name. * ``RUN_ON_MODEL_SCOPE(region)`` - allows to disable run_on_model pass if it isn't used. The region name should be unique. .. _transformation_writing_essentials: @@ -122,22 +129,29 @@ To develop a transformation, follow these transformation rules: 1. Friendly Names +++++++++++++++++ -Each ``ov::Node`` has a unique name and a friendly name. In transformations, only the friendly name matters because it represents the name from the model's perspective. -To prevent losing the friendly name when replacing a node with another node or a subgraph, the original friendly name is set to the last node in the replacing subgraph. See the example below. +Each ``ov::Node`` has a unique name and a friendly name. In transformations, only the friendly +name matters because it represents the name from the model's perspective. +To prevent losing the friendly name when replacing a node with another node or a subgraph, +the original friendly name is set to the last node in the replacing subgraph. See the example below. .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp :language: cpp :fragment: [ov:replace_friendly_name] -In more complicated cases, when a replaced operation has several outputs and additional consumers are added to its outputs, the decision on how to set the friendly name is determined by an agreement. +In more complicated cases, when a replaced operation has several outputs and additional +consumers are added to its outputs, the decision on how to set the friendly name is determined by an agreement. 2. Runtime Info +++++++++++++++ -Runtime info is a map ``std::map`` located inside the ``ov::Node`` class. It represents additional attributes of the ``ov::Node``. -These attributes, which can be set by users or plugins, need to be preserved when executing a transformation that changes ``ov::Model``, as they are not automatically propagated. -In most cases, transformations have the following types: 1:1 (replace node with another node), 1:N (replace node with a sub-graph), N:1 (fuse sub-graph into a single node), N:M (any other transformation). -Currently, there is no mechanism that automatically detects transformation types, so this runtime information needs to be propagated manually. See the example below: +Runtime info is a map ``std::map`` located inside the ``ov::Node`` class. +It represents additional attributes of the ``ov::Node``. +These attributes, which can be set by users or plugins, need to be preserved when executing +a transformation that changes ``ov::Model``, as they are not automatically propagated. +In most cases, transformations have the following types: 1:1 (replace node with another node), +1:N (replace node with a sub-graph), N:1 (fuse sub-graph into a single node), N:M (any other transformation). +Currently, there is no mechanism that automatically detects transformation types, so this +runtime information needs to be propagated manually. See the example below: .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp @@ -146,12 +160,16 @@ Currently, there is no mechanism that automatically detects transformation types When a transformation has multiple fusions or decompositions, ``ov::copy_runtime_info`` must be called multiple times for each case. -.. note:: ``copy_runtime_info`` removes ``rt_info`` from destination nodes. If you want to keep it, specify them in source nodes as following: ``copy_runtime_info({a, b, c}, {a, b})`` +.. note:: + + ``copy_runtime_info`` removes ``rt_info`` from destination nodes. If you want to keep it, + specify them in source nodes as following: ``copy_runtime_info({a, b, c}, {a, b})`` 3. Constant Folding +++++++++++++++++++ -If your transformation inserts constant sub-graphs that need to be folded, do not forget to use ``ov::pass::ConstantFolding()`` after your transformation or call constant folding directly for operation. +If your transformation inserts constant sub-graphs that need to be folded, do not forget +to use ``ov::pass::ConstantFolding()`` after your transformation or call constant folding directly for operation. The example below shows how constant subgraph can be constructed. .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp @@ -174,8 +192,12 @@ Common mistakes in transformations In transformation development process: * Do not use deprecated OpenVINO™ API. Deprecated methods are marked with ``OPENVINO_DEPRECATED`` macro in their definition. -* Do not pass ``shared_ptr`` as input for another node if the type of the node is unknown or if it has multiple outputs. Instead, use explicit output ports. -* If you replace a node with another node that produces different shape, note that the new shape will not be propagated until the first ``validate_nodes_and_infer_types`` call for ``ov::Model``. If you are using ``ov::pass::Manager``, it will automatically call this method after each transformation execution. +* Do not pass ``shared_ptr`` as input for another node if the type of the node is unknown + or if it has multiple outputs. Instead, use explicit output ports. +* If you replace a node with another node that produces different shape, note that + the new shape will not be propagated until the first ``validate_nodes_and_infer_types`` + call for ``ov::Model``. If you are using ``ov::pass::Manager``, it will automatically call + this method after each transformation execution. * Do not forget to call the ``ov::pass::ConstantFolding`` pass if your transformation creates constant subgraphs. * Use latest OpSet if you are not developing downgrade transformation pass. * When developing a callback for ``ov::pass::MatcherPass``, do not change nodes that come after the root node in the topological order. @@ -185,9 +207,11 @@ In transformation development process: Using pass manager ################## -``ov::pass::Manager`` is a container class that can store a list of transformations and execute them. The main idea of this class is to have a high-level representation for grouped list of transformations. -It can register and apply any `transformation pass <#transformations_types>`__ on a model. -In addition, ``ov::pass::Manager`` has extended debug capabilities (find more information in the `how to debug transformations <#how-to-debug-transformations>`__ section). +``ov::pass::Manager`` is a container class that can store a list of transformations and execute them. +The main idea of this class is to have a high-level representation for grouped list of transformations. +It can register and apply any `transformation pass <#transformations-types>`__ on a model. +In addition, ``ov::pass::Manager`` has extended debug capabilities (find more information +in the `how to debug transformations <#how-to-debug-transformations>`__ section). The example below shows basic usage of ``ov::pass::Manager`` @@ -206,7 +230,8 @@ Another example shows how multiple matcher passes can be united into single Grap How to debug transformations ############################ -If you are using ``ov::pass::Manager`` to run sequence of transformations, you can get additional debug capabilities by using the following environment variables: +If you are using ``ov::pass::Manager`` to run sequence of transformations, you can get +additional debug capabilities by using the following environment variables: .. code-block:: cpp @@ -214,7 +239,9 @@ If you are using ``ov::pass::Manager`` to run sequence of transformations, you c OV_ENABLE_VISUALIZE_TRACING=1 - enables visualization after each transformation. By default, it saves dot and svg files. -.. note:: Make sure that you have dot installed on your machine; otherwise, it will silently save only dot file without svg file. +.. note:: + + Make sure that you have dot installed on your machine; otherwise, it will silently save only dot file without svg file. See Also ######## diff --git a/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst b/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst index a0cc4488ef15cc..4c9c3ab6f05c76 100644 --- a/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst +++ b/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst @@ -21,7 +21,7 @@ Such a model is called a Low Precision IR and can be generated in two ways: * By :doc:`quantizing regular IR with the Neural Network Compression Framework (NNCF) <../../openvino-workflow/model-optimization>` * Using model conversion of a model pre-trained for Low Precision inference: TensorFlow models (``.pb`` model file with ``FakeQuantize`` operations), quantized TensorFlow Lite models and ONNX quantized models. -TensorFlow and ONNX quantized models can be prepared by `Neural Network Compression Framework `__. + TensorFlow and ONNX quantized models can be prepared by `Neural Network Compression Framework `__. For an operation to be executed in INT8, it must have `FakeQuantize` operations as inputs. For more details, see the :doc:`specification of FakeQuantize operation `. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst index bfa55a5a798ce6..c15b31b9051e20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst @@ -5,8 +5,8 @@ Broadcast Rules For Elementwise Operations .. meta:: - :description: Learn about Numpy and PDPD broadcast types in OpenVINO, that - enable performing element-wise operation for inputs of arbitrary + :description: Learn about Numpy and PDPD broadcast types in OpenVINO, that + enable performing element-wise operation for inputs of arbitrary number of dimensions. The purpose of this document is to provide a set of common rules which are applicable for ops using broadcasting. @@ -40,81 +40,82 @@ Rules Numpy examples ############## -* ``A: Shape(,) -> scalar`` - ``B: Shape(,) -> scalar`` +* ``A: Shape(,) -> scalar`` + ``B: Shape(,) -> scalar`` ``Result: Shape(,) -> scalar`` -* ``A: Shape(2, 3)`` - ``B: Shape( 1)`` +* ``A: Shape(2, 3)`` + ``B: Shape( 1)`` ``Result: Shape(2, 3)`` -* ``A: Shape( 3)`` - ``B: Shape(2, 3)`` +* ``A: Shape( 3)`` + ``B: Shape(2, 3)`` ``Result: Shape(2, 3)`` -* ``A: Shape(2, 3, 5)`` - ``B: Shape(,) -> scalar`` +* ``A: Shape(2, 3, 5)`` + ``B: Shape(,) -> scalar`` ``Result: Shape(2, 3, 5)`` -* ``A: Shape(2, 1, 5)`` +* ``A: Shape(2, 1, 5)`` ``B: Shape(1, 4, 5)`` ``Result: Shape(2, 4, 5)`` -* ``A: Shape( 6, 5)`` - ``B: Shape(2, 1, 5)`` +* ``A: Shape( 6, 5)`` + ``B: Shape(2, 1, 5)`` ``Result: Shape(2, 6, 5)`` -* ``A: Shape(2, 1, 5)`` - ``B: Shape( 4, 1)`` - ``Result: Shape(2, 4, 5)`` +* ``A: Shape(2, 1, 5)`` + ``B: Shape( 4, 1)`` + ``Result: Shape(2, 4, 5)`` -* ``A: Shape(3, 2, 1, 4)`` - ``B: Shape( 5, 4)`` +* ``A: Shape(3, 2, 1, 4)`` + ``B: Shape( 5, 4)`` ``Result: Shape(3, 2, 5, 4)`` -* ``A: Shape( 1, 5, 3)`` - ``B: Shape(5, 2, 1, 3)`` +* ``A: Shape( 1, 5, 3)`` + ``B: Shape(5, 2, 1, 3)`` ``Result: Shape(5, 2, 5, 3)`` -* ``A: Shape(3)`` - ``B: Shape(2)`` +* ``A: Shape(3)`` + ``B: Shape(2)`` ``Result: broadcast won't happen due to dimensions mismatch`` -* ``A: Shape(3, 1, 5)`` - ``B: Shape(4, 4, 5)`` +* ``A: Shape(3, 1, 5)`` + ``B: Shape(4, 4, 5)`` ``Result: broadcast won't happen due to dimensions mismatch on the leftmost axis`` PDPD examples ############# -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape( 3, 4 ) with axis = 1`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape( 3, 4 ) with axis = 1`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape( 3, 1 ) with axis = 1`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape( 3, 1 ) with axis = 1`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape( 4, 5) with axis=-1(default) or axis=2`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape( 4, 5) with axis=-1(default) or axis=2`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape(1, 3 ) with axis = 0`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape(1, 3 ) with axis = 0`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape(,)`` - ``Result: Shape(2, 3, 4, 5)`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape(,)`` + ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` +* ``A: Shape(2, 3, 4, 5)`` ``B: Shape( 5) with axis=-1(default) or axis = 3`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(8, 1, 6, 1)`` - ``B: Shape( 7, 1, 5) with axis = 1`` +* ``A: Shape(8, 1, 6, 1)`` + ``B: Shape( 7, 1, 5) with axis = 1`` ``Result: broadcast won't happen due to dimensions mismatch, only B to A broadcast is supported for PDPD`` -Bidirectional Broadcast Rules + +Bidirectional Broadcast Rules ############################# Description @@ -138,23 +139,23 @@ Rules Bidirectional examples ++++++++++++++++++++++ -* ``A: Shape(5)`` - ``B: Shape(1)`` +* ``A: Shape(5)`` + ``B: Shape(1)`` ``Result: Shape(5)`` -* ``A: Shape(2, 3)`` - ``B: Shape( 3)`` +* ``A: Shape(2, 3)`` + ``B: Shape( 3)`` ``Result: Shape(2, 3)`` -* ``A: Shape(3, 1)`` - ``B: Shape(3, 4)`` +* ``A: Shape(3, 1)`` + ``B: Shape(3, 4)`` ``Result: Shape(3, 4)`` -* ``A: Shape(3, 4)`` - ``B: Shape(,) -> scalar`` +* ``A: Shape(3, 4)`` + ``B: Shape(,) -> scalar`` ``Result: Shape(3, 4)`` -* ``A: Shape( 3, 1)`` - ``B: Shape(2, 1, 6)`` +* ``A: Shape( 3, 1)`` + ``B: Shape(2, 1, 6)`` ``Result: Shape(2, 3, 6)`` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst index 250ef955bb41a8..8f37b9dbdf8e9f 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst @@ -22,6 +22,8 @@ Operation Specifications Assign-6 Atan-1 Atanh-3 + AUGRUCell + AUGRUSequence AvgPool-1 AvgPool-14 BatchNormInference-1 @@ -107,7 +109,7 @@ Operation Specifications Interpolate-1 Interpolate-4 Interpolate-11 - Inverse-14 + Inverse-14 IsFinite-10 IsInf-10 IsNaN-10 @@ -130,6 +132,7 @@ Operation Specifications MaxPool-1 MaxPool-8 MaxPool-14 + Shape Calculation Rules for Pooling Operators Maximum-1 Minimum-1 Mish-4 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst index 57a4e6f28091e8..bd2184251d0a26 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst @@ -5,7 +5,7 @@ Tanh .. meta:: - :description: Learn about Tanh-1 - an element-wise, arithmetic operation, which + :description: Learn about Tanh-1 - an element-wise, arithmetic operation, which can be performed on a single tensor in OpenVINO. **Versioned name**: *Tanh-1* @@ -34,7 +34,7 @@ For each element from the input tensor calculates corresponding element in the o **Outputs**: * **1**: The result of element-wise *Tanh* operation. A tensor of type *T* and the same shape as input tensor. -**Types** + **Types** * *T*: any numeric type. @@ -44,7 +44,7 @@ For each element from the input tensor calculates corresponding element in the o *Example 1* .. code-block:: xml - :force: + :force: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst index 7dd1a2d1659a0a..f53693792133f8 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst @@ -5,46 +5,46 @@ If .. meta:: - :description: Learn about If-8 - an element-wise, condition operation, which + :description: Learn about If-8 - an element-wise, condition operation, which can be performed on multiple tensors in OpenVINO. **Versioned name**: *If-8* **Category**: *Condition* -**Short description**: *If* operation contains two internal networks(subgraphs) such as ``then_body`` and ``else_body``, -and performs one of them depending on ``cond`` value. If ``cond`` is ``True``, ``then_body`` is executed. If ``cond`` is ``False``, -the operation executes the ``else_body`` subgraph. +**Short description**: *If* operation contains two internal networks(subgraphs) such as ``then_body`` and ``else_body``, +and performs one of them depending on ``cond`` value. If ``cond`` is ``True``, ``then_body`` is executed. If ``cond`` is ``False``, +the operation executes the ``else_body`` subgraph. **Detailed description** -*If* must not contain empty subgraphs. Each of them must have at least one operation ``Result``. +*If* must not contain empty subgraphs. Each of them must have at least one operation ``Result``. Also the number of outputs from *If* always must be greater than zero and equal to the number of outputs from each subgraph. **If attributes**: * **Subgraphs**: - ``then_body``/``else_body`` are subgraphs that are executed depending on the ``cond`` value. - The subgraph is described operation by operation as a typical IR network. + ``then_body``/``else_body`` are subgraphs that are executed depending on the ``cond`` value. + The subgraph is described operation by operation as a typical IR network. The subgraph has inputs (``Parameter`` operations) and outputs (``Result`` operations). - * **Subgraph's inputs** - inputs to the subgraph which associated with *If* inputs via *port_map*. + * **Subgraph's inputs** - inputs to the subgraph which associated with *If* inputs via *port_map*. The subgraph can have any number of inputs (even zero). * **Subgraph's outputs** - outputs from the subgraph which associated with *If* outputs via *port_map*. - The subgraph must contain at least one output. Each *If* output is associated with one output from the subgraph. - Therefore the number of ``then_body`` outputs is equal to the number of outputs from *If* and + The subgraph must contain at least one output. Each *If* output is associated with one output from the subgraph. + Therefore the number of ``then_body`` outputs is equal to the number of outputs from *If* and the number of ``else_body`` outputs. The type of the subgraph output and the type of the associated output from *If* must be equal. * **Port maps**: - *port_map* is a set of rules to map input or output data tensors of *If* operation onto the subgraph data tensors. - The ``port_map`` entries can be ``input`` and ``output``. Each entry describes a corresponding mapping rule. + *port_map* is a set of rules to map input or output data tensors of *If* operation onto the subgraph data tensors. + The ``port_map`` entries can be ``input`` and ``output``. Each entry describes a corresponding mapping rule. *If* has two *port_maps*: ``then_port_map`` for ``then_body`` and ``else_port_map`` for ``else_body``. * **Port map attributes**: @@ -59,18 +59,18 @@ Also the number of outputs from *If* always must be greater than zero and equal * *internal_layer_id* - * **Description**: *internal_layer_id* is a ``Parameter`` or ``Result`` operation ID inside - the subgraph to map to. + * **Description**: *internal_layer_id* is a ``Parameter`` or ``Result`` operation ID inside + the subgraph to map to. * **Range of values**: IDs of the ``Parameter`` or ``Result`` operations in the subgraph * **Type**: ``unsigned int`` * **Default value**: None - * **Required**: *yes* + * **Required**: *yes* **If Inputs** -* **cond**: A scalar or 1D tensor with 1 element of ``boolean`` type specifying which subgraph to execute. -``True`` value means to execute the ``then_body``, ``False`` - ``else_body``. *Required*. +* **cond**: A scalar or 1D tensor with 1 element of ``boolean`` type specifying which subgraph to execute. + ``True`` value means to execute the ``then_body``, ``False`` - ``else_body``. *Required*. * **Multiple other inputs**: Tensors of different types and shapes. *Optional*. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst index 4f494d910fa014..3e9ba293596d2a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst @@ -5,8 +5,8 @@ ExperimentalDetectronDetectionOutput .. meta:: - :description: Learn about ExperimentalDetectronDetectionOutput-6 - an object - detection operation, which can be performed on four required + :description: Learn about ExperimentalDetectronDetectionOutput-6 - an object + detection operation, which can be performed on four required input tensors in OpenVINO. **Versioned name**: *ExperimentalDetectronDetectionOutput-6* @@ -42,7 +42,7 @@ refined boxes according to the formulas: ``ctr_y = y0 + 0.5f * box_h`` * ``dx``, ``dy``, ``d_log_w`` and ``d_log_h`` are deltas calculated according to the formulas below, and ``deltas_tensor`` is a -second input: + second input: ``dx = deltas_tensor[roi_idx, 4 * class_idx + 0] / deltas_weights[0]`` @@ -104,7 +104,7 @@ second input: * **Description**: *class_agnostic_box_regression* attribute is a flag that specifies whether to delete background classes or not. * **Range of values**: - + * ``true`` means background classes should be deleted * ``false`` means background classes should not be deleted * **Type**: ``boolean`` @@ -149,7 +149,7 @@ second input: .. code-block:: xml :force: - + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst index d3850dbb4296ec..4a3b947603c15c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst @@ -5,8 +5,8 @@ ExperimentalDetectronGenerateProposalsSingleImage .. meta:: - :description: Learn about ExperimentalDetectronGenerateProposalsSingleImage-6 - - an object detection operation, which can be performed on four + :description: Learn about ExperimentalDetectronGenerateProposalsSingleImage-6 - + an object detection operation, which can be performed on four required input tensors. **Versioned name**: *ExperimentalDetectronGenerateProposalsSingleImage-6* @@ -64,7 +64,7 @@ ExperimentalDetectronGenerateProposalsSingleImage * **1**: A 1D tensor of type *T* with 3 elements ``[image_height, image_width, scale_height_and_width]`` providing input image size info. **Required.** * **2**: A 2D tensor of type *T* with shape ``[height * width * number_of_channels, 4]`` providing anchors. **Required.** * **3**: A 3D tensor of type *T* with shape ``[number_of_channels * 4, height, width]`` providing deltas for anchors. -Height and width for third and fourth inputs should be equal. **Required.** + Height and width for third and fourth inputs should be equal. **Required.** * **4**: A 3D tensor of type *T* with shape ``[number_of_channels, height, width]`` providing proposals scores. **Required.** **Outputs** @@ -80,7 +80,7 @@ Height and width for third and fourth inputs should be equal. **Required.** .. code-block:: xml :force: - + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst index d3ac83442e3c2d..c5de7a037bbbe9 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst @@ -5,7 +5,7 @@ PSROIPooling .. meta:: - :description: Learn about PSROIPooling-1 - an object detection operation, + :description: Learn about PSROIPooling-1 - an object detection operation, which can be performed on two required input tensors. **Versioned name**: *PSROIPooling-1* @@ -73,10 +73,10 @@ ROIs coordinates are specified in absolute values for the average mode and in no **Inputs**: -* **1**: 4D input tensor with shape ``[N, C, H, W]`` and type *T* with feature maps. **Required.** +* **1**: 4D input tensor with shape ``[N, C, H, W]`` and type *T* with feature maps. **Required.** -* **2**: 2D input tensor with shape ``[num_boxes, 5]``. It contains a list of five element tuples that describe a region of interest: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** -Batch indices must be in the range of ``[0, N-1]``. +* **2**: 2D input tensor with shape ``[num_boxes, 5]``. It contains a list of five element tuples that describe a region of interest: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** + Batch indices must be in the range of ``[0, N-1]``. **Outputs**: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst index 1da1e33079c106..6663b048fdc6de 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst @@ -1,4 +1,5 @@ .. {#openvino_docs_ops_detection_ROIAlignRotated_15} + ROIAlignRotated =============== @@ -56,9 +57,9 @@ Each ROI box's center is shifted by [-0.5, -0.5] before pooling to achive better * *spatial_scale* * **Description**: *spatial_scale* is a multiplicative spatial scale factor to that is applied to the ROI box(height, weight and center vector) before pooling. - WARNING! - Spatial scale is also applied to the center point of the ROI box. It means that scaling does not only change the size of the ROI box, but also its position. - For example, if the spatial scale is 2.0, ROI box center is [0.5, 0.5], box width is 1.0 and box height is 1.0, then after scaling the ROI box center will be [1.0, 1.0], box width will be 2.0 and box height will be 2.0. + WARNING! + Spatial scale is also applied to the center point of the ROI box. It means that scaling does not only change the size of the ROI box, but also its position. + For example, if the spatial scale is 2.0, ROI box center is [0.5, 0.5], box width is 1.0 and box height is 1.0, then after scaling the ROI box center will be [1.0, 1.0], box width will be 2.0 and box height will be 2.0. * **Range of values**: a positive floating-point number * **Type**: ``float`` * **Required**: *yes* diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst index 26377ca0aebfcc..7deb444d388ac8 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst @@ -5,7 +5,7 @@ ROIPooling .. meta:: - :description: Learn about ROIPooling-1 - an object detection operation, + :description: Learn about ROIPooling-1 - an object detection operation, which can be performed on two required input tensors. **Versioned name**: *ROIPooling-1* @@ -61,10 +61,10 @@ The box height and width have different representation based on **method** attri **Inputs**: -* **1**: 4D input tensor of shape ``[N, C, H, W]`` with feature maps of type *T*. **Required.** +* **1**: 4D input tensor of shape ``[N, C, H, W]`` with feature maps of type *T*. **Required.** -* **2**: 2D input tensor of shape ``[NUM_ROIS, 5]`` describing region of interest box consisting of 5 element tuples of type *T*: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** -Batch indices must be in the range of ``[0, N-1]``. +* **2**: 2D input tensor of shape ``[NUM_ROIS, 5]`` describing region of interest box consisting of 5 element tuples of type *T*: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** + Batch indices must be in the range of ``[0, N-1]``. **Outputs**: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst index 470fa5b7099006..6968d556734d20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst @@ -51,7 +51,7 @@ Interpolate * *pads_begin* * **Description**: *pads_beg* specify the number of pixels to add to the beginning of the image being interpolated. -This is a scalar that specifies padding for each spatial dimension. + This is a scalar that specifies padding for each spatial dimension. * **Range of values**: list of non-negative integer numbers * **Type**: ``int`` * **Default value**: 0 @@ -60,7 +60,7 @@ This is a scalar that specifies padding for each spatial dimension. * *pads_end* * **Description**: *pads_end* specify the number of pixels to add to the beginning of the image being interpolated. -This is a scalar that specifies padding for each spatial dimension. + This is a scalar that specifies padding for each spatial dimension. * **Range of values**: list of non-negative integer numbers * **Type**: ``int`` * **Default value**: 0 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst index 7572f7c1bc97ac..202bcde7c0c60c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst @@ -22,7 +22,12 @@ Interpolate * **Range of values**: one of ``nearest``, ``linear``, ``linear_onnx``, ``cubic`` * **Type**: string * **Required**: *yes* - **Note**: Only 2D, 3D, 4D, 5D tensors with ``axes = {0, 1}``, ``axes = {0, 1, 2}``, ``axes = {2, 3}``, ``axes = {2, 3, 4}`` respectively are supported for ``"mode" == "linear_onnx"``. + + .. note:: + + Only 2D, 3D, 4D, 5D tensors with ``axes = {0, 1}``, ``axes = {0, 1, 2}``, + ``axes = {2, 3}``, ``axes = {2, 3, 4}`` respectively are supported for + ``"mode" == "linear_onnx"``. * *shape_calculation_mode* diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst index 375833ec7ee18e..0e1e206d821ad0 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst @@ -72,7 +72,7 @@ There are several combinations of these two inputs ``(trip_count, execution cond Loop operation description in the IR has regular sections: ``input`` and ``output``. They connect Loop body to the outer graph and specify condition(s). Loop operation description in the IR also has several special sections: ``body``, ``port_map`` and ``back_edges`` similar to the ones from the TensorIterator operation but having some important features described below. -1. The body operation getting an input from the main graph should have an entry in the ``port_map`` section of the Loop operation. These edges connect input ports of the Loop with the body ``Parameter``s. +1. The body operation getting an input from the main graph should have an entry in the ``port_map`` section of the Loop operation. These edges connect input ports of the Loop with the body ``Parameter``\ s. 2. Input tensors to the Loop can be sliced along a specified axis, the Loop can iterates over all sliced parts. The corresponding ``input`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to slice. Therefore, inputs to the Loop operation corresponding to ``input`` entries in the ``port_map`` without ``axis`` attribute are used "as is" (without slicing). 3. The body operation producing tensor to be used in the subsequent iterations (like in RNN models) should have a back edge described in the ``back_edges`` section of the operation. The back edge connects the respective body ``Parameter`` and ``Result`` operations. For such a case the Loop operation node provides input for the first iteration, while corresponding Loop operation output produces the tensor computed during the last iteration. 4. Output tensors produced by a particular body operation across all iterations can be concatenated and returned as a Loop operation output (this is a "scan output" according to the ONNX* Loop operation `specification `__ ). The corresponding ``output`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to concatenate. Therefore, outputs from operations corresponding to ``output`` entries in the ``port_map`` without ``axis`` attribute are returned "as is" (without concatenation). @@ -114,7 +114,7 @@ Loop operation description in the IR also has several special sections: ``body`` * *axis* * **Description**: if *axis* is specified for ``output`` entry, then it is an axis to concatenate the body ``Result`` output across all iterations. - If *axis* is specified for ``input`` entry, then it is an axis to iterate through, it triggers the slicing of the input tensor. + If *axis* is specified for ``input`` entry, then it is an axis to iterate through, it triggers the slicing of the input tensor. * **Range of values**: an integer. Negative value means counting dimension from the end. * **Type**: ``int`` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst index ebe248309a122d..cd1026cb052b18 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst @@ -13,7 +13,7 @@ Gather **Category**: *Data movement* **Short description**: *Gather* operation takes slices of data of the first input tensor according to the indices - specified with the second input tensor and axis from the third input. Semantics of this operation is identical to +specified with the second input tensor and axis from the third input. Semantics of this operation is identical to TensorFlow `Gather `__ operation. **Detailed description** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst index b2bb5bf0235c60..9bc5dc50a51fb6 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst @@ -195,7 +195,7 @@ Example 7 with indices out of the range: **Outputs** * **1**: The resulting tensor of type *T* that consists of elements from ``data`` tensor gathered by ``indices``. The shape -of the output tensor is ``data.shape[:axis] + indices.shape[batch_dims:] + data.shape[axis + 1:]`` + of the output tensor is ``data.shape[:axis] + indices.shape[batch_dims:] + data.shape[axis + 1:]`` **Types** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst index 69eafbb10f7c26..175e838ab0b576 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst @@ -33,20 +33,20 @@ For instance, in a 3D tensor case, the update corresponding to the ``[i][j][k]`` **Inputs**: -* **1**: ``data`` tensor of arbitrary rank ``r`` and of type *T*. **Required.** +* **1**: ``data`` tensor of arbitrary rank ``r`` and of type *T*. **Required.** -* **2**: ``indices`` tensor with indices of type *T_IND*. The rank of the tensor is equal to the rank of ``data`` tensor. All index values are expected to be within bounds ``[0, s - 1]`` along axis of size ``s``. If multiple indices point to the -same output location then the order of updating the values is undefined. If an index points to non-existing output -tensor element or is negative then exception is raised. **Required.** +* **2**: ``indices`` tensor with indices of type *T_IND*. The rank of the tensor is equal to the rank of ``data`` tensor. All index values are expected to be within bounds ``[0, s - 1]`` along axis of size ``s``. If multiple indices point to the + same output location then the order of updating the values is undefined. If an index points to non-existing output + tensor element or is negative then exception is raised. **Required.** -* **3**: ``updates`` tensor of shape equal to the shape of ``indices`` tensor and of type *T*. **Required.** +* **3**: ``updates`` tensor of shape equal to the shape of ``indices`` tensor and of type *T*. **Required.** -* **4**: ``axis`` tensor with scalar or 1D tensor with one element of type *T_AXIS* specifying axis for scatter. -The value can be in range ``[-r, r - 1]`` where ``r`` is the rank of ``data``. **Required.** +* **4**: ``axis`` tensor with scalar or 1D tensor with one element of type *T_AXIS* specifying axis for scatter. + The value can be in range ``[-r, r - 1]`` where ``r`` is the rank of ``data``. **Required.** **Outputs**: -* **1**: tensor with shape equal to ``data`` tensor of the type *T*. +* **1**: tensor with shape equal to ``data`` tensor of the type *T*. **Types** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst index cc5105a4f11697..df46e4afb30c01 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst @@ -1,6 +1,6 @@ .. {#openvino_docs_pooling_shape_rules} -Shape calculation rules for Pooling Operators +Shape Calculation Rules for Pooling Operators ============================================= .. meta:: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst index 9aef4e981cf46f..768b440f626ea0 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst @@ -107,7 +107,7 @@ Particular cases: -= + .. code-block:: xml :force: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/scaled-dot-product-attention.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/scaled-dot-product-attention.rst index 922048a4f18754..2d18b9d3a6488c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/scaled-dot-product-attention.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/scaled-dot-product-attention.rst @@ -60,11 +60,11 @@ omitting training-related parameter. * **4**: ``attention_mask`` - two options available. ``attention_mask`` is ignored if ``causal`` is set to ``True``. **Optional.** - * at least 3 dimensional tensor of type *T* or ``boolean`` and shape ``[N, ..., L, S]``. + * at least 2 dimensional tensor of type *T* or ``boolean`` and shape numpy-broadcastable to ``[N, ..., L, S]``. See :doc:`Numpy Broadcast Rules <../../broadcast-rules>` for broadcast details. * a scalar of type *T* with value ``0``. Scalar zero value signals that applying an attention mask is not necessary (similar to specifying attention_mask=None in the provided pseudo-code). -* **5**: ``scale`` a scalar tensor of type *T*, an alternative scale factor instead of 1/sqrt(query.shape[-1]) used by default in the pseudo-code above. **Optional.** +* **5**: ``scale`` a scalar or single element 1D tensor of type *T*, an alternative scale factor instead of 1/sqrt(query.shape[-1]) used by default in the pseudo-code above. **Optional.** **Outputs** @@ -78,7 +78,7 @@ omitting training-related parameter. **Dimensions** -* ``N, ...`` - one or more batch dimensions. Each batch dimension should be either constant across the input tensors (query, key, and value), indicating that they have the same batch size, or they should be broadcastable to the same value. +* ``N, ...`` - one or more batch dimensions. Each batch dimension should be either constant across the input tensors (query, key, and value), indicating that they have the same batch size, or they should be numpy-broadcastable to the same value. See :doc:`Numpy Broadcast Rules <../../broadcast-rules>` for broadcast details. * ``S`` - source sequence length @@ -192,29 +192,29 @@ Other batch dimensions ``...`` are optional. - 1 + 4 6 - 5 + 10 -1 80 - 2 (repeat 2 times) - 2 (repeat 3 times) - 2 (repeat 5 times) + 1 + 6 + 10 -1 80 - 4 - 3 - 10 + 1 + 1 + 1 -1 80 1 - 2 + 1 1 -1 -1 @@ -231,3 +231,42 @@ Other batch dimensions ``...`` are optional. + +*Example 5: With attention mask broadcasting* + +.. code-block:: xml + :force: + + + + + + + 2 + 16 + 80 + + + 2 + 32 + 80 + + + 2 + 32 + 80 + + + 2 + 1 + 1 + + + + + 2 + 16 + 80 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst index 1ccce710ee0f41..ae6db8fb5353a4 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst @@ -18,7 +18,7 @@ Squeeze * If only the first input is provided, every dimension that is equal to 1 will be removed from it. * With the second input provided, each value is an index of a dimension from the first tensor that is to be removed. Specified dimension should be equal to 1, otherwise it will be ignored and copied as is. -Dimension indices can be specified directly, or by negative indices (counting dimensions from the end). + Dimension indices can be specified directly, or by negative indices (counting dimensions from the end). Note: Updated behavior since 2024.3, request of squeezing dimension not equal to 1 is expected to be ignored instead of causing an error. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst index 328dd37d0f2c44..662dbd5d282e83 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst @@ -5,8 +5,8 @@ NonMaxSuppression .. meta:: - :description: Learn about NonMaxSuppression-1 - a sorting and maximization - operation, which can be performed on two required and three + :description: Learn about NonMaxSuppression-1 - a sorting and maximization + operation, which can be performed on two required and three optional input tensors. **Versioned name**: *NonMaxSuppression-1* @@ -52,24 +52,24 @@ class must not exceed ``max_output_boxes_per_class``. **Inputs**: -* **1**: ``boxes`` - floating-point tensor of shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** +* **1**: ``boxes`` - floating-point tensor of shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** -* **2**: ``scores`` - floating-point tensor of shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** +* **2**: ``scores`` - floating-point tensor of shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** -* **3**: ``max_output_boxes_per_class`` - integer scalar tensor specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. +* **3**: ``max_output_boxes_per_class`` - integer scalar tensor specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. -* **4**: ``iou_threshold`` - floating-point scalar tensor specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. +* **4**: ``iou_threshold`` - floating-point scalar tensor specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. -* **5**: ``score_threshold`` - floating-point scalar tensor specifying minimum score to consider box for the processing. Optional with default value 0. +* **5**: ``score_threshold`` - floating-point scalar tensor specifying minimum score to consider box for the processing. Optional with default value 0. **Outputs**: -* **1**: ``selected_indices`` - integer tensor of shape ``[min(num_boxes, max_output_boxes_per_class * num_classes), 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. -The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. +* **1**: ``selected_indices`` - integer tensor of shape ``[min(num_boxes, max_output_boxes_per_class * num_classes), 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. + The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. **Example** -.. code-block:: cpp +.. code-block:: cpp diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst index 161a68b255b1c9..72a5631194f2cf 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst @@ -60,20 +60,20 @@ class must not exceed ``max_output_boxes_per_class``. **Inputs**: -* **1**: ``boxes`` - tensor of type *T* and shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** +* **1**: ``boxes`` - tensor of type *T* and shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** -* **2**: ``scores`` - tensor of type *T* and shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** +* **2**: ``scores`` - tensor of type *T* and shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** -* **3**: ``max_output_boxes_per_class`` - scalar tensor of type *T_MAX_BOXES* specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. +* **3**: ``max_output_boxes_per_class`` - scalar tensor of type *T_MAX_BOXES* specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. -* **4**: ``iou_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. +* **4**: ``iou_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. -* **5**: ``score_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying minimum score to consider box for the processing. Optional with default value 0. +* **5**: ``score_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying minimum score to consider box for the processing. Optional with default value 0. **Outputs**: -* **1**: ``selected_indices`` - tensor of type *T_IND* and shape ``[min(num_boxes, max_output_boxes_per_class) * num_batches * num_classes, 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. -The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. +* **1**: ``selected_indices`` - tensor of type *T_IND* and shape ``[min(num_boxes, max_output_boxes_per_class) * num_batches * num_classes, 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. + The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. **Types** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-15.rst new file mode 100644 index 00000000000000..9f0392c8e2d038 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-15.rst @@ -0,0 +1,184 @@ +.. {#openvino_docs_ops_sparse_EmbeddingBagOffsets_15} + +EmbeddingBagOffsets +====================== + + +.. meta:: + :description: Learn about EmbeddingBagOffsets-15 - a sparse operation, which + can be performed on three required and two optional input tensors. + +**Versioned name**: *EmbeddingBagOffsets-15* + +**Category**: *Sparse* + +**Short description**: Computes sums or means of "bags" of embeddings, without instantiating the intermediate embeddings. + +**Detailed description**: + +Operation EmbeddingBagOffsets is an implementation of ``torch.nn.EmbeddingBag`` with indices and offsets inputs being 1D tensors. + +For each index in ``indices`` this operator gathers values from ``emb_table`` embedding table. Then values at indices in the range of the same bag (based on ``offset`` input) are reduced according to ``reduction`` attribute. + +Values in ``offsets`` define starting index in ``indices`` tensor of each "bag", +e.g. ``offsets`` with value ``[0, 3, 4, 4, 6]`` define 5 "bags" containing ``[3, 1, 0, 2, num_indices-6]`` elements corresponding to ``[indices[0:3], indices[3:4], empty_bag, indices[4:6], indices[6:]]`` slices of indices per bag. + +EmbeddingBagOffsets is an equivalent to following NumPy snippet: + +.. code-block:: py + + def embedding_bag_offsets( + emb_table: np.ndarray, + indices: np.ndarray, + offsets: np.ndarray, + default_index: Optional[int] = None, + per_sample_weights: Optional[np.ndarray] = None, + reduction: Literal["sum", "mean"] = "sum", + ): + assert ( + reduction == "sum" or per_sample_weights is None + ), "Attribute per_sample_weights is only supported in sum reduction." + if per_sample_weights is None: + per_sample_weights = np.ones_like(indices) + embeddings = [] + for emb_idx, emb_weight in zip(indices, per_sample_weights): + embeddings.append(emb_table[emb_idx] * emb_weight) + previous_offset = offsets[0] + bags = [] + offsets = np.append(offsets, len(indices)) + for bag_offset in offsets[1:]: + bag_size = bag_offset - previous_offset + if bag_size != 0: + embedding_bag = embeddings[previous_offset:bag_offset] + reduced_bag = np.add.reduce(embedding_bag) + if reduction == "mean": + reduced_bag = reduced_bag / bag_size + bags.append(reduced_bag) + else: + # Empty bag case + if default_index is not None and default_index != -1: + bags.append(emb_table[default_index]) + else: + bags.append(np.zeros(emb_table.shape[1:])) + previous_offset = bag_offset + return np.stack(bags, axis=0) + + +**Attributes**: + +* *reduction* + + * **Description**: reduction mode. + * **Range of values**: + + * sum - compute weighted sum, using corresponding values of ``per_sample_weights`` as weights if provided. + * mean - compute average of values in bag. Input ``per_sample_weights`` is not supported and will raise exception. + + * **Type**: ``string`` + * **Default value**: sum + * **Required**: *no* + +**Inputs**: + +* **1**: ``emb_table`` tensor containing the embedding lookup table of the module of shape ``[num_emb, emb_dim1, emb_dim2, ...]`` and of type *T*. **Required.** +* **2**: ``indices`` tensor of shape ``[num_indices]`` and of type *T_IND*. **Required.** +* **3**: ``offsets`` tensor of shape ``[batch]`` and of type *T_IND* containing the starting index positions of each "bag" in ``indices``. Maximum value of offsets cannot be greater than length of ``indices``. **Required.** +* **4**: ``default_index`` scalar of type *T_IND* containing default index in embedding table to fill empty "bags". If set to ``-1`` or not provided, empty "bags" are filled with zeros. Reverse indexing using negative values is not supported. **Optional.** +* **5**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T*. Supported only when *reduction* attribute is set to ``"sum"``. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. **Optional.** + +**Outputs**: + +* **1**: tensor of shape ``[batch, emb_dim1, emb_dim2, ...]`` and of type *T* containing embeddings for each bag. + +**Types** + +* *T*: any numeric type. +* *T_IND*: ``int32`` or ``int64``. + +**Example** + +*Example 1: per_sample_weights are provided, default_index is set to 0 to fill empty bag with values gathered form emb_table on given index.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 4 + + + 3 + + + + 4 + + + + + 3 + 2 + + + + +*Example 2: per_sample_weights are provided, default_index is set to -1 to fill empty bag with 0.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 4 + + + 3 + + + + 4 + + + + + 3 + 2 + + + + +*Example 3: Example of reduction set to mean.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 4 + + + 3 + + + + + 3 + 2 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst index 0a0cb67afb0f06..c3eb163b16d98f 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst @@ -14,7 +14,48 @@ EmbeddingBagOffsetsSum **Short description**: Computes sums of "bags" of embeddings, without instantiating the intermediate embeddings. -**Detailed description**: This is the second case of the PyTorch `EmbeddingBag `__ , it has indices in two 1D tensors provided as 2nd and 3rd inputs. For each index in ``indices`` this operator gets values from ``data`` embedding table and sums all values belonging to each bag. Values in ``offsets`` define starting index in ``indices`` tensor of each "bag", e.g. ``offsets`` with value ``[0,3,4,4,6]`` define 5 "bags" containing ``[3,1,0,2,n-6]`` elements. +**Detailed description**: + +Operation EmbeddingBagOffsets is an implementation of ``torch.nn.EmbeddingBag`` with indices and offsets inputs being 1D tensors. + +For each index in ``indices`` this operator gathers values from ``emb_table`` embedding table. Then values at indices in the range of the same bag (based on ``offset`` input) are reduced according to ``reduction`` attribute. + +Values in ``offsets`` define starting index in ``indices`` tensor of each "bag", +e.g. ``offsets`` with value ``[0, 3, 4, 4, 6]`` define 5 "bags" containing ``[3, 1, 0, 2, num_indices-6]`` elements corresponding to ``[indices[0:3], indices[3:4], empty_bag, indices[4:6], indices[6:]]`` slices of indices per bag. + +EmbeddingBagOffsetsSum is an equivalent to following NumPy snippet: + +.. code-block:: py + + def embedding_bag_offsets( + emb_table: np.ndarray, + indices: np.ndarray, + offsets: np.ndarray, + default_index: Optional[int] = None, + per_sample_weights: Optional[np.ndarray] = None, + ): + if per_sample_weights is None: + per_sample_weights = np.ones_like(indices) + embeddings = [] + for emb_idx, emb_weight in zip(indices, per_sample_weights): + embeddings.append(emb_table[emb_idx] * emb_weight) + previous_offset = offsets[0] + bags = [] + offsets = np.append(offsets, len(indices)) + for bag_offset in offsets[1:]: + bag_size = bag_offset - previous_offset + if bag_size != 0: + embedding_bag = embeddings[previous_offset:bag_offset] + reduced_bag = np.add.reduce(embedding_bag) + bags.append(reduced_bag) + else: + # Empty bag case + if default_index is not None and default_index != -1: + bags.append(emb_table[default_index]) + else: + bags.append(np.zeros(emb_table.shape[1:])) + previous_offset = bag_offset + return np.stack(bags, axis=0) **Attributes**: EmbeddingBagOffsetsSum operation has no attributes. @@ -22,9 +63,9 @@ EmbeddingBagOffsetsSum * **1**: ``emb_table`` tensor containing the embedding lookup table of the module of shape ``[num_emb, emb_dim1, emb_dim2, ...]`` and of type *T*. **Required.** * **2**: ``indices`` tensor of shape ``[num_indices]`` and of type *T_IND*. **Required.** -* **3**: ``offsets`` tensor of shape ``[batch]`` and of type *T_IND* containing the starting index positions of each "bag" in ``indices``. **Required.** -* **4**: ``default_index`` scalar of type *T_IND* containing default index in embedding table to fill empty "bags". If not provided empty "bags" are filled with zeros. **Optional.** -* **5**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T*. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. +* **3**: ``offsets`` tensor of shape ``[batch]`` and of type *T_IND* containing the starting index positions of each "bag" in ``indices``. Maximum value of offsets cannot be greater than length of ``indices``. **Required.** +* **4**: ``default_index`` scalar of type *T_IND* containing default index in embedding table to fill empty "bags". If set to ``-1`` or not provided, empty "bags" are filled with zeros. Reverse indexing using negative values is not supported. **Optional.** +* **5**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T*. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. **Optional.** **Outputs**: @@ -37,7 +78,9 @@ EmbeddingBagOffsetsSum **Example** -.. code-block:: cpp +*Example 1: per_sample_weights are provided, default_index is set to 0 to fill empty bag with values gathered form emb_table on given index.* + +.. code-block:: xml @@ -52,7 +95,7 @@ EmbeddingBagOffsetsSum 3 - + 4 @@ -64,4 +107,31 @@ EmbeddingBagOffsetsSum +*Example 2: per_sample_weights are provided, default_index is set to -1 to fill empty bag with 0.* + +.. code-block:: xml + + + + 5 + 2 + + + 4 + + + 3 + + + + 4 + + + + + 3 + 2 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-15.rst new file mode 100644 index 00000000000000..2892d49759f667 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-15.rst @@ -0,0 +1,131 @@ +.. {#openvino_docs_ops_sparse_EmbeddingBagPacked_15} + +EmbeddingBagPacked +===================== + + +.. meta:: + :description: Learn about EmbeddingBagPacked-15 - a sparse operation, which + can be performed on two required and one optional input tensor. + +**Versioned name**: *EmbeddingBagPacked-15* + +**Category**: *Sparse* + +**Short description**: Computes sums or means of "bags" of embeddings, without instantiating the intermediate embeddings. + +**Detailed description**: + +Operation EmbeddingBagPacked is an implementation of ``torch.nn.EmbeddingBag`` with indices input being 2D tensor of shape ``[batch, indices_per_bag]``. +Operation is equivalent to *gather_op = Gather(emb_table, indices, axis=0)* followed by reduction: + + * *sum* - *ReduceSum(Multiply(gather_op, Unsqueeze(per_sample_weights, -1)), axis=1)*, + * *mean* - *ReduceMean(gather_op, axis=1)*. + +**Attributes**: + +* *reduction* + + * **Description**: reduction mode. + * **Range of values**: + + * sum - compute weighted sum, using corresponding values of ``per_sample_weights`` as weights if provided. + * mean - compute average of values in bag. Input ``per_sample_weights`` is not supported and will raise exception. + + * **Type**: ``string`` + * **Default value**: sum + * **Required**: *no* + +**Inputs**: + +* **1**: ``emb_table`` tensor containing the embedding lookup table of the module of shape ``[num_emb, emb_dim1, emb_dim2, ...]`` and of type *T*. **Required.** +* **2**: ``indices`` tensor of shape ``[batch, indices_per_bag]`` and of type *T_IND*. **Required.** +* **3**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T* supported only in ``sum`` mode. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. **Optional.** + +**Outputs**: + +* **1**: tensor of shape ``[batch, emb_dim1, emb_dim2, ...]`` and of type *T* containing embeddings for each bag. + +**Types** + +* *T*: any numeric type. +* *T_IND*: ``int32`` or ``int64``. + +**Example** + +*Example 1: reduction set to sum, per_sample_weights are not provided.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 3 + 2 + + + + + 3 + 2 + + + + +*Example 2: reduction set to sum and per_sample_weights are provided.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 3 + 2 + + + 3 + 2 + + + + + 3 + 2 + + + + +*Example 3: reduction set to mean, per_sample_weights are not provided.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 3 + 2 + + + + + 3 + 2 + + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst index 9ef623ca7755eb..b6cad12be869ac 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst @@ -14,7 +14,10 @@ EmbeddingBagPackedSum **Short description**: Computes sums of "bags" of embeddings, without instantiating the intermediate embeddings. -**Detailed description**: This is the first case of the PyTorch `EmbeddingBag `__ , it has indices in the tensor of format ``[batch, indices_per_bag]``. If 3rd input is not provided, this operation is equivalent to *Gather* followed by *ReduceSum(axis=0)*. However, *EmbeddingBagPackedSum* is much more time and memory efficient than using a chain of these operations. +**Detailed description**: + +Operation EmbeddingBagPackedSum is an implementation of ``torch.nn.EmbeddingBag`` in ``sum`` mode, which indices input being 2D tensor of shape ``[batch, indices_per_bag]``. +Operation is equivalent to *ReduceSum(Multiply(Gather(emb_table, indices, axis=0), Unsqueeze(per_sample_weights, -1)), axis=1)*. **Attributes**: EmbeddingBagPackedSum operation has no attributes. @@ -35,7 +38,7 @@ EmbeddingBagPackedSum **Example** -.. code-block:: cpp +.. code-block:: xml @@ -47,13 +50,13 @@ EmbeddingBagPackedSum 3 2 - + 3 2 - + 3 2 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-pack-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-pack-15.rst new file mode 100644 index 00000000000000..77f44f8d687741 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-pack-15.rst @@ -0,0 +1,147 @@ +.. {#openvino_docs_ops_type_StringTensorPack_15} + +StringTensorPack +=================== + + +.. meta:: + :description: Learn about StringTensorPack-15 - operation which packs a concatenated batch of strings into a batched string tensor. + +**Versioned name**: *StringTensorPack-15* + +**Category**: *Type* + +**Short description**: *StringTensorPack* transforms a concatenated strings data (encoded as 1D tensor of u8 element type) into +a string tensor using *begins* and *ends* indices. + +**Detailed description** + +Consider inputs: + +* *begins* = [0, 5] +* *ends* = [5, 13] +* *symbols* = "IntelOpenVINO" + +*StringTensorPack* uses indices from ``begins`` and ``ends`` to transform concatenated string ``symbols`` into ``output``, +a string tensor. The ``output.shape`` is equal to ``begins.shape`` and ``ends.shape``, +and in this case ``output`` holds values ``["Intel", "OpenVINO"]``. + +When defining *begins* and *ends*, the notation ``[a, b)`` is used. This means that the range starts with ``a`` and includes all values up to, +but not including, ``b``. That is why in the example given the length of "IntelOpenVINO" is 12, but *ends* vector contains 13. The shapes of ``begins`` and ``ends`` are required to be equal. + +**Inputs** + +* **1**: ``begins`` - ND tensor of non-negative integer numbers of type *T_IDX*, containing indices of each string's beginnings. **Required.** + +* **2**: ``ends`` - ND tensor of non-negative integer numbers of type *T_IDX*, containing indices of each string's endings. **Required.** + +* **3**: ``symbols`` - 1D tensor of concatenated strings data encoded in utf-8 bytes, of type *u8*. **Required.** + +**Outputs** + +* **1**: ``output`` - ND string tensor of the same shape as *begins* and *ends*. + +**Types** + +* *T_IDX*: ``int32`` or ``int64``. + +**Examples** + +*Example 1: 1D begins and ends* + +.. code-block:: xml + :force: + + + + + 2 + + + 2 + + + 13 + + + + + 2 + + + + +*Example 2: input with an empty string* + +.. code-block:: xml + :force: + + + + + 2 + + + 2 + + + 13 + + + + + 5 + + + + +*Example 3: skipped symbols* + +.. code-block:: xml + :force: + + + + + 2 + + + 2 + + + 13 + + + + + 5 + + + + +*Example 4: 2D begins and ends* + +.. code-block:: xml + :force: + + + + + 2 + 2 + + + 2 + 2 + + + 21 + + + + + 2 + 2 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-unpack-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-unpack-15.rst new file mode 100644 index 00000000000000..8ae8a8f8234b21 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-unpack-15.rst @@ -0,0 +1,133 @@ +.. {#openvino_docs_ops_type_StringTensorUnpack_15} + +StringTensorUnpack +=================== + + +.. meta:: + :description: Learn about StringTensorUnpack-15 - operation which unpacks a batch of strings into three tensors. + +**Versioned name**: *StringTensorUnpack-15* + +**Category**: *Type* + +**Short description**: *StringTensorUnpack* operation transforms a given batch of strings into three tensors - two storing begin +and end indices of the strings and another containing the concatenated string data, respectively. + +**Detailed description** + +Consider an ``input`` string tensor containing values ``["Intel", "OpenVINO"]``. + +The operator will transform the tensor into three outputs: + +* *begins* = [0, 5] + * ``begins[0]`` is equal to 0, because the first string starts at the beginning index. + * ``begins[1]`` is equal to 5, because length of the string "Intel" is equal to 5. + * ``begins.shape`` is equal to [2], because the ``input`` is a batch of 2 strings. + +* *ends* = [5, 13] + * ``ends[0]`` is equal to 5, because length of the string "Intel" is equal to 5. + * ``ends[1]`` is equal to 13, because length of the string "OpenVINO" is 8, and it needs to be summed up with length of the string "Intel". + * ``ends.shape`` is equal to ``[2]``, because the ``input`` is a batch of 2 strings. + +* *symbols* = "IntelOpenVINO" + * ``symbols`` contains concatenated string data encoded in utf-8 bytes, interpretable using ``begins`` and ``ends``. + * ``symbols.shape`` is equal to ``[13]``, because it's the length of concatenated ``input`` strings. + +When defining *begins* and *ends*, the notation ``[a, b)`` is used. This means that the range starts with ``a`` and includes all values up to, +but not including, ``b``. That is why in the example given the length of "IntelOpenVINO" is 12, but *ends* vector contains 13. + +**Inputs** + +* **1**: ``data`` - ND tensor of type *string*. **Required.** + +**Outputs** + +* **1**: ``begins`` - ND tensor of non-negative integer numbers of type *int32* and of the same shape as ``data`` input. + +* **2**: ``ends`` - ND tensor of non-negative integer numbers of type *int32* and of the same shape as ``data`` input. + +* **3**: ``symbols`` - 1D tensor of concatenated strings data encoded in utf-8 bytes, of type *u8* and size equal to the sum of the lengths of each string from the ``data`` input. + +**Examples** + +*Example 1: 1D input* + +For ``input = ["Intel", "OpenVINO"]`` + +.. code-block:: xml + :force: + + + + + 2 + + + + + 2 + + + 2 + + + 13 + + + + +*Example 2: input with an empty string* + +For ``input = ["OMZ", "", "GenAI", " ", "2024"]`` + +.. code-block:: xml + :force: + + + + + 5 + + + + + 2 + + + 2 + + + 13 + + + + +*Example 3: 2D input* + +For ``input = [["Intel", "OpenVINO"], ["OMZ", "GenAI"]]`` + +.. code-block:: xml + :force: + + + + + 2 + 2 + + + + + 2 + 2 + + + 2 + 2 + + + 21 + + + diff --git a/docs/articles_en/get-started/install-openvino.rst b/docs/articles_en/get-started/install-openvino.rst index 166df8eb104cd7..f8fefc4abacece 100644 --- a/docs/articles_en/get-started/install-openvino.rst +++ b/docs/articles_en/get-started/install-openvino.rst @@ -21,7 +21,7 @@ Install OpenVINO™ 2024.2 - + OpenVINO 2024.2, described here, is not a Long-Term-Support version! All currently supported versions are: @@ -50,7 +50,7 @@ All currently supported versions are: | \* **Of the Linux systems, version 22.04 includes drivers for NPU.** | **For Windows, CPU inference on ARM64 is not supported.** -.. dropdown:: Effortless GenAI integration with OpenVINO GenAI Flavor** +.. dropdown:: Effortless GenAI integration with OpenVINO GenAI Flavor A new OpenVINO GenAI Flavor streamlines application development by providing LLM-specific interfaces for easy integration of language models, handling tokenization and diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst index 058b93f3b9dd75..004f74f74043fd 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst @@ -252,13 +252,8 @@ You can also try the following: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. -* See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. +* See pre-trained deep learning models on `Hugging Face `__ * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Take a glance at the OpenVINO `product home page `__ . - - - - - diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst index 294d966df9b37c..c64c27678b4fa4 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst @@ -25,10 +25,8 @@ Install OpenVINO™ Runtime on Linux from an Archive File Debian9 armhf V n/a n/a Ubuntu20 arm64 V n/a n/a CentOS7 x86_64 V V n/a - Ubuntu18 x86_64 V V n/a - Ubuntu20 x86_64 V V n/a + Ubuntu20 x86_64 V V V Ubuntu22 x86_64 V V V - Ubuntu24 x86_64 V V V RHEL8 x86_64 V V n/a =================== ===== ===== ===== @@ -60,11 +58,6 @@ Install OpenVINO™ Runtime on Linux from an Archive File * GCC 9.3.0 - .. tab-item:: Ubuntu 18.04 - :sync: ubuntu-18 - - * GCC 7.5.0 - .. tab-item:: RHEL 8 :sync: rhel-8 @@ -74,7 +67,7 @@ Install OpenVINO™ Runtime on Linux from an Archive File :sync: centos-7 * GCC 8.3.1 - Use the following instructions to install it: + Use the following instructions to install it: Install GCC 8.3.1 via devtoolset-8 @@ -91,8 +84,6 @@ Install OpenVINO™ Runtime on Linux from an Archive File gcc -v - - Installing OpenVINO Runtime ############################################################ @@ -117,7 +108,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -132,9 +123,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu24_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu24_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu24_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 @@ -142,9 +133,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu22_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu22_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: Ubuntu 20.04 :sync: ubuntu-20 @@ -152,20 +143,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 - - - .. tab-item:: Ubuntu 18.04 - :sync: ubuntu-18 - - .. code-block:: sh - - - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu18_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu18_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: RHEL 8 :sync: rhel-8 @@ -173,18 +153,18 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_rhel8_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_rhel8_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_rhel8_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_rhel8_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: CentOS 7 :sync: centos-7 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_centos7_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_centos7_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_centos7_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_centos7_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: ARM 64-bit @@ -192,26 +172,25 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_arm64.tgz -O openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_arm64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_arm64.tgz -O openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_arm64 /opt/intel/openvino_2024.2.0 .. tab-item:: ARM 32-bit :sync: arm-32 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_debian9_2024.1.0.15008.f4afc983258_armhf.tgz -O openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_debian9_2024.1.0.15008.f4afc983258_armhf /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_debian9_2024.2.0.15519.5c0f38f83f6_armhf.tgz -O openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_debian9_2024.2.0.15519.5c0f38f83f6_armhf /opt/intel/openvino_2024.2.0 5. Install required system dependencies on Linux. To do this, OpenVINO provides a script in the extracted installation directory. Run the following command: .. code-block:: sh - cd /opt/intel/openvino_2024.1.0 + cd /opt/intel/openvino_2024.2.0 sudo -E ./install_dependencies/install_openvino_dependencies.sh 6. (Optional) Install *numpy* Python Library: @@ -220,11 +199,11 @@ Step 1: Download and Install the OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.1.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.2.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.1.0 + cd /opt/intel/openvino_2024.2.0 python3 -m pip install -r ./python/requirements.txt 7. For simplicity, it is useful to create a symbolic link as below: @@ -233,7 +212,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /opt/intel - sudo ln -s openvino_2024.1.0 openvino_2024 + sudo ln -s openvino_2024.2.0 openvino_2024 .. note:: If you have already installed a previous release of OpenVINO 2024, a symbolic link to the ``openvino_2024`` folder may already exist. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst index 2421ec54c32354..03902a8edb7dc6 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst @@ -63,7 +63,7 @@ Step 1: Install OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -73,9 +73,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/macos/m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/macos/m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: ARM, 64-bit :sync: arm-64 @@ -83,9 +83,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/macos/m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_arm64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_arm64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/macos/m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_arm64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_arm64 /opt/intel/openvino_2024.2.0 5. (Optional) Install *numpy* Python Library: @@ -94,11 +94,11 @@ Step 1: Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.1.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.2.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.1.0 + cd /opt/intel/openvino_2024.2.0 python3 -m pip install -r ./python/requirements.txt 6. For simplicity, it is useful to create a symbolic link as below: @@ -106,7 +106,7 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - sudo ln -s /opt/intel/openvino_2024.1.0 /opt/intel/openvino_2024 + sudo ln -s /opt/intel/openvino_2024.2.0 /opt/intel/openvino_2024 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst index ffa5e612ab1571..c9e32e907837b0 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst @@ -78,19 +78,19 @@ Step 1: Download and Install OpenVINO Core Components ``C:\Program Files (x86)\Intel`` is the recommended folder. You may also use a different path if desired or if you don't have administrator privileges on your computer. -2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. +2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. If you prefer using command-lines, run the following commands in the command prompt window you opened: .. code-block:: sh cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/windows/w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64.zip --output openvino_2024.1.0.zip --output openvino_2024.0.0.zip + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/windows/w_openvino_toolkit_windows_2024.2.0.15519.5c0f38f83f6_x86_64.zip --output openvino_2024.2.0.zip --output openvino_2024.2.0.zip .. note:: - A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.1.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. + A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.2.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. 3. Use your favorite tool to extract the archive file, rename the extracted folder, and move it to the ``C:\Program Files (x86)\Intel`` directory. @@ -99,9 +99,9 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh - tar -xf openvino_2024.1.0.zip - ren w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64 openvino_2024.1.0 - move openvino_2024.1.0 "C:\Program Files (x86)\Intel" + tar -xf openvino_2024.2.0.zip + ren w_openvino_toolkit_windows_2024.2.0.15519.5c0f38f83f6_x86_64 openvino_2024.2.0 + move openvino_2024.2.0 "C:\Program Files (x86)\Intel" 4. (Optional) Install *numpy* Python Library: @@ -110,11 +110,11 @@ Step 1: Download and Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.1.0\python`` folder: + You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.2.0\python`` folder: .. code-block:: sh - cd "C:\Program Files (x86)\Intel\openvino_2024.1.0" + cd "C:\Program Files (x86)\Intel\openvino_2024.2.0" python -m pip install -r .\python\requirements.txt @@ -123,7 +123,7 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh cd C:\Program Files (x86)\Intel - mklink /D openvino_2024 openvino_2024.1.0 + mklink /D openvino_2024 openvino_2024.2.0 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst b/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst index a5f5abeaf34db9..11bd8d90b6b641 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst @@ -96,7 +96,14 @@ Now that you've installed OpenVINO Runtime, you can try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. -* See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. +* See pre-trained deep learning models in our + :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Check out the OpenVINO `product home page `__. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst index 6aa5240098da10..11c2425d231dd3 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst @@ -1,9 +1,6 @@ -.. {#openvino_docs_install_guides_installing_openvino_docker} - Install Intel® Distribution of OpenVINO™ toolkit from a Docker Image ======================================================================= - .. meta:: :description: Learn how to use a prebuilt Docker image or create an image manually to install OpenVINO™ Runtime on Linux and Windows operating systems. @@ -33,7 +30,7 @@ The Docker CI repository includes guides on how to To start using Dockerfiles the following conditions must be met: -- Linux OS or Windows (under :ref:`Windows Subsystem for Linux (WSL2) `) +- Linux OS or Windows (under :ref:`Windows Subsystem for Linux (WSL2) `) - Installed docker engine or compatible container engine - Permissions to run containers (sudo or docker group membership) @@ -42,10 +39,14 @@ To start using Dockerfiles the following conditions must be met: OpenVINO's `Docker `__ and :doc:`Bare Metal <../install-openvino>` distributions are identical, so the documentation applies to both. + Note that starting with OpenVINO 2024.4, Ubuntu docker images will no longer be provided + and will be replaced by Debian-based ones. + .. note:: OpenVINO development environment in a docker container is also available in the - `notebook repository `__. It can be implemented in + `notebook repository `__. + It can be implemented in `OpenShift RedHat OpenData Science (RHODS) `__. More information about Docker CI for Intel® Distribution of OpenVINO™ toolset can be found diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst index 5927d11b18e8c8..cd6714149e7211 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst @@ -9,16 +9,30 @@ For a quickstart guide, refer to the :doc:`GenAI API Guide <../../learn-openvino To see GenAI in action, check the Jupyter notebooks: `LLM-powered Chatbot `__ and -`LLM Instruction-following pipeline `__ +`LLM Instruction-following pipeline `__. -The OpenVINO GenAI flavor is available for installation via Archive and PyPI distributions: +The OpenVINO GenAI flavor is available for installation via PyPI and Archive distributions: + +PyPI Installation +############################### + +To install the GenAI flavor of OpenVINO via PyPI, follow the standard :doc:`installation steps `, +but use the *openvino-genai* package instead of *openvino*: + +.. code-block:: python + + python -m pip install openvino-genai Archive Installation ############################### +The OpenVINO GenAI archive package includes the OpenVINO™ Runtime and :doc:`Tokenizers <../../learn-openvino/llm_inference_guide/ov-tokenizers>`. To install the GenAI flavor of OpenVINO from an archive file, follow the standard installation steps for your system but instead of using the vanilla package file, download the one with OpenVINO GenAI: +Linux +++++++++++++++++++++++++++ + .. tab-set:: .. tab-item:: x86_64 @@ -31,32 +45,69 @@ but instead of using the vanilla package file, download the one with OpenVINO Ge .. code-block:: sh - - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu24_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu24_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu22_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + .. tab-item:: Ubuntu 20.04 + :sync: ubuntu-20 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu20_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + + .. tab-item:: ARM 64-bit + :sync: arm-64 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu20_2024.2.0.0_arm64.tar.gz -O openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + +Windows +++++++++++++++++++++++++++ + +.. code-block:: sh + + cd /Downloads + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/windows/openvino_genai_windows_2024.2.0.0_x86_64.zip --output openvino_genai_2024.2.0.0.zip + +macOS +++++++++++++++++++++++++++ + +.. tab-set:: + + .. tab-item:: x86, 64-bit + :sync: x86-64 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/macos/openvino_genai_macos_12_6_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + .. tab-item:: ARM, 64-bit + :sync: arm-64 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/macos/openvino_genai_macos_11_0_2024.2.0.0_arm64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + Here are the full guides: :doc:`Linux `, :doc:`Windows `, and :doc:`macOS `. -PyPI Installation -############################### - -To install the GenAI flavor of OpenVINO via PyPI, follow the standard :doc:`installation steps `, -but use the *openvino-genai* package instead of *openvino*: -.. code-block:: python - - python -m pip install openvino-genai diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst b/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst index 3074bd8135e4e7..828b5205169fea 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst @@ -102,6 +102,12 @@ Now that you've installed OpenVINO Runtime, you can try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. * See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Check out the OpenVINO `product home page `__ . diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst b/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst index afa00cd6551e85..0ff1b95c8eb212 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst @@ -66,8 +66,6 @@ Step 1: Set up the environment CORE_IMAGE_EXTRA_INSTALL:append = " openvino-inference-engine-samples" # Include OpenVINO Python API package in the target image. CORE_IMAGE_EXTRA_INSTALL:append = " openvino-inference-engine-python3" - # Include model conversion API in the target image. - CORE_IMAGE_EXTRA_INSTALL:append = " openvino-model-optimizer" Step 2: Build a Yocto Image with OpenVINO Packages ################################################## @@ -102,9 +100,6 @@ If the image build is successful, it will return the list of packages as below: openvino-inference-engine-python3 openvino-inference-engine-samples openvino-inference-engine-src - openvino-model-optimizer - openvino-model-optimizer-dbg - openvino-model-optimizer-dev Additional Resources #################### diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst b/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst index a5559e937e50d6..cbb25eac817f97 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst @@ -226,6 +226,12 @@ You can also try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. * See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Take a glance at the OpenVINO `product home page `__ . diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst b/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst index 20166da049fee7..ee0b2633299c66 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst @@ -170,6 +170,12 @@ You can also try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. * See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Take a glance at the OpenVINO `product home page `__ . diff --git a/docs/articles_en/learn-openvino.rst b/docs/articles_en/learn-openvino.rst index e955c1e6f8abbf..19e96db09bd543 100644 --- a/docs/articles_en/learn-openvino.rst +++ b/docs/articles_en/learn-openvino.rst @@ -6,7 +6,7 @@ Learn OpenVINO .. meta:: :description: Explore learning materials, including interactive - Python learn-openvino/interactive-tutorials-python and sample console applications that explain + Python tutorials and sample console applications that explain how to use OpenVINO features. @@ -26,7 +26,7 @@ accommodating different learning needs, which means you should find it useful if as well as an experienced user. | :doc:`Tutorials ` -| A collection of interactive Python learn-openvino/interactive-tutorials-python. It introduces you to the OpenVINO™ toolkit explaining how to use the Python API and tools for optimized deep learning inference. The learn-openvino/interactive-tutorials-python are available in Jupyter notebooks and can be run in your browser. No installation required. +| A collection of interactive Python tutorials. It introduces you to the OpenVINO™ toolkit explaining how to use the Python API and tools for optimized deep learning inference. The tutorials are available in Jupyter notebooks and can be run in your browser. No installation required. | :doc:`OpenVINO Samples ` | The OpenVINO samples (Python and C++) are simple console applications that show how to use specific OpenVINO API features. They can assist you in executing tasks such as loading a model, running inference, querying particular device capabilities, etc. diff --git a/docs/articles_en/learn-openvino/llm_inference_guide.rst b/docs/articles_en/learn-openvino/llm_inference_guide.rst index 874e53899e1cfd..6db776a3c1f5fb 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide.rst @@ -26,19 +26,36 @@ conversion to advanced use cases. The advantages of using OpenVINO for LLM deployment: -* **OpenVINO offers optimized LLM inference**; provides a full C/C++ API, leading to faster operation than Python-based runtimes; includes a Python API for rapid development, with the option for further optimization in C++. - -* **Compatible with diverse hardware**, supports CPUs, GPUs, and neural accelerators across ARM and x86/x64 architectures, integrated Intel® Processor Graphics, discrete Intel® Arc™ A-Series Graphics, and discrete Intel® Data Center GPU Flex Series; features automated optimization to maximize performance on target hardware. - -* **Requires fewer dependencies** than frameworks like Hugging Face and PyTorch, resulting in a smaller binary size and reduced memory footprint, making deployments easier and updates more manageable. - -* **Provides compression and precision management techniques** such as 8-bit and 4-bit weight compression, including embedding layers, and storage format reduction. This includes fp16 precision for non-compressed models and int8/int4 for compressed models, like GPTQ models from Hugging Face. - -* **Supports a wide range of deep learning models and architectures** including text, image, and audio generative models like Llama 2, MPT, OPT, Stable Diffusion, Stable Diffusion XL. This enables the development of multimodal applications, allowing for write-once, deploy-anywhere capabilities. - -* **Enhances inference capabilities**: fused inference primitives such as Scaled Dot Product Attention, Rotary Positional Embedding, Group Query Attention, and Mixture of Experts. It also offers advanced features like in-place KV-cache, dynamic quantization, KV-cache quantization and encapsulation, dynamic beam size configuration, and speculative sampling. - -* **Provides stateful model optimization**: models from the Hugging Face Transformers are converted into a stateful form, optimizing inference performance and memory usage in long-running text generation tasks by managing past KV-cache tensors more efficiently internally. This feature is automatically activated for many supported models, while unsupported ones remain stateless. Learn more about the :doc:`Stateful models and State API <../openvino-workflow/running-inference/stateful-models>`. +* **OpenVINO offers optimized LLM inference**: + provides a full C/C++ API, leading to faster operation than Python-based runtimes; includes a + Python API for rapid development, with the option for further optimization in C++. +* **Compatible with diverse hardware**: + supports CPUs, GPUs, and neural accelerators across ARM and x86/x64 architectures, integrated + Intel® Processor Graphics, discrete Intel® Arc™ A-Series Graphics, and discrete Intel® Data + Center GPU Flex Series; features automated optimization to maximize performance on target + hardware. +* **Requires fewer dependencies**: + than frameworks like Hugging Face and PyTorch, resulting in a smaller binary size and reduced + memory footprint, making deployments easier and updates more manageable. +* **Provides compression and precision management techniques**: + such as 8-bit and 4-bit weight compression, including embedding layers, and storage format + reduction. This includes fp16 precision for non-compressed models and int8/int4 for compressed + models, like GPTQ models from `Hugging Face `__. +* **Supports a wide range of deep learning models and architectures**: + including text, image, and audio generative models like Llama 2, MPT, OPT, Stable Diffusion, + Stable Diffusion XL. This enables the development of multimodal applications, allowing for + write-once, deploy-anywhere capabilities. +* **Enhances inference capabilities**: + fused inference primitives such as Scaled Dot Product Attention, Rotary Positional Embedding, + Group Query Attention, and Mixture of Experts. It also offers advanced features like in-place + KV-cache, dynamic quantization, KV-cache quantization and encapsulation, dynamic beam size + configuration, and speculative sampling. +* **Provides stateful model optimization**: + models from the Hugging Face Transformers are converted into a stateful form, optimizing + inference performance and memory usage in long-running text generation tasks by managing past + KV-cache tensors more efficiently internally. This feature is automatically activated for many + supported models, while unsupported ones remain stateless. Learn more about the + :doc:`Stateful models and State API <../openvino-workflow/running-inference/stateful-models>`. OpenVINO offers three main paths for Generative AI use cases: @@ -121,4 +138,5 @@ approaches. - Best suited for high-performance, resource-optimized production environments * - Model Serving - Paid service, based on CPU/GPU usage with Hugging Face - - Free code solution, run script for own server; costs may incur for cloud services like AWS but generally cheaper than Hugging Face rates + - Free code solution, run script for own server; costs may incur for cloud services + like AWS but generally cheaper than Hugging Face rates diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst index 821f16062c271b..e1b0a170848ec3 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst @@ -54,9 +54,10 @@ will not work with these instructions, make sure to #include int main(int argc, char* argv[]) { - std::string model_path = argv[1]; - ov::genai::LLMPipeline pipe(model_path, "CPU");//target device is CPU - std::cout << pipe.generate("The Sun is yellow because"); //input context + std::string model_path = argv[1]; + ov::genai::LLMPipeline pipe(model_path, "CPU"); + std::cout << pipe.generate("The Sun is yellow because"); + } The `LLMPipeline` is the main object used for decoding. You can construct it directly from the folder with the converted model. It will automatically load the main model, tokenizer, detokenizer, @@ -74,6 +75,17 @@ below, where a lambda function outputs words to the console immediately upon gen .. tab-set:: + .. tab-item:: Python + :sync: py + + .. code-block:: python + + import openvino_genai as ov_genai + pipe = ov_genai.LLMPipeline(model_path, "CPU") + + streamer = lambda x: print(x, end='', flush=True) + pipe.generate("The Sun is yellow because", streamer=streamer) + .. tab-item:: C++ .. code-block:: cpp @@ -85,14 +97,43 @@ below, where a lambda function outputs words to the console immediately upon gen std::string model_path = argv[1]; ov::genai::LLMPipeline pipe(model_path, "CPU"); - auto streamer = [](std::string word) { std::cout << word << std::flush; }; - std::cout << pipe.generate("The Sun is yellow because", streamer); + auto streamer = [](std::string word) { + std::cout << word << std::flush; + // Return flag indicating whether generation should be stopped. + // false means continue generation. + return false; + }; + pipe.generate("The Sun is yellow because", ov::genai::streamer(streamer)); } You can also create your custom streamer for more sophisticated processing: .. tab-set:: + .. tab-item:: Python + :sync: py + + .. code-block:: python + + import openvino_genai as ov_genai + + class CustomStreamer(ov_genai.StreamerBase): + def __init__(self, tokenizer): + ov_genai.StreamerBase.__init__(self) + self.tokenizer = tokenizer + def put(self, token_id) -> bool: + # Decode tokens and process them. + # Streamer returns a flag indicating whether generation should be stopped. + # In Python, `return` can be omitted. In that case, the function will return None + # which will be converted to False, meaning that generation should continue. + # return stop_flag + def end(self): + # Decode tokens and process them. + + pipe = ov_genai.LLMPipeline(model_path, "CPU") + pipe.generate("The Sun is yellow because", streamer=CustomStreamer()) + + .. tab-item:: C++ .. code-block:: cpp @@ -101,20 +142,31 @@ You can also create your custom streamer for more sophisticated processing: class CustomStreamer: publict StreamerBase { public: - void put(int64_t token) {/* decode tokens and do process them*/}; - - void end() {/* decode tokens and do process them*/}; + bool put(int64_t token) { + bool stop_flag = false; + /* + custom decoding/tokens processing code + tokens_cache.push_back(token); + std::string text = m_tokenizer.decode(tokens_cache); + ... + */ + return stop_flag; // Flag indicating whether generation should be stopped. If True, generation stops. + }; + + void end() { + /* custom finalization */ + }; }; int main(int argc, char* argv[]) { - CustomStreamer custom_streamer; + auto custom_streamer = std::make_shared(); std::string model_path = argv[1]; - ov::LLMPipeline pipe(model_path, "CPU"); - cout << pipe.generate("The Sun is yellow because", custom_streamer); + ov::genai::LLMPipeline pipe(model_path, "CPU"); + pipe.generate("The Sun is yellow because", ov::genai::streamer(custom_streamer)); } -Optimizing the Chat Scenario +Using GenAI in Chat Scenario ################################ For chat scenarios where inputs and outputs represent a conversation, maintaining KVCache across inputs @@ -131,16 +183,15 @@ mark a conversation session, as you can see in these simple examples: import openvino_genai as ov_genai pipe = ov_genai.LLMPipeline(model_path) - config = {'num_groups': 3, 'group_size': 5, 'diversity_penalty': 1.1} - pipe.set_generation_cofnig(config) + pipe.set_generation_cofnig({'max_new_tokens': 100) pipe.start_chat() while True: -     print('question:') -     prompt = input() + print('question:') + prompt = input() if prompt == 'Stop!': -         break -     print(pipe(prompt)) + break + print(pipe.generate(prompt)) pipe.finish_chat() @@ -153,14 +204,18 @@ mark a conversation session, as you can see in these simple examples: std::string prompt; std::string model_path = argv[1]; - ov::LLMPipeline pipe(model_path, "CPU"); + ov::genai::LLMPipeline pipe(model_path, "CPU"); + + ov::genai::GenerationConfig config = pipe.get_generation_config(); + config.max_new_tokens = 100; + pipe.set_generation_cofnig(config) pipe.start_chat(); for (size_t i = 0; i < questions.size(); i++) { std::cout << "question:\n"; std::getline(std::cin, prompt); - std::cout << pipe(prompt) << std::endl>>; + std::cout << pipe.generate(prompt) << std::endl; } pipe.finish_chat(); } @@ -171,60 +226,43 @@ Optimizing Generation with Grouped Beam Search Leverage grouped beam search decoding and configure generation_config for better text generation quality and efficient batch processing in GenAI applications. -Use grouped beam search decoding: +Specify generation_config to use grouped beam search: .. tab-set:: - .. tab-item:: C++ - - .. code-block:: cpp - - int main(int argc, char* argv[]) { - std::string model_path = argv[1]; - ov::LLMPipeline pipe(model_path, "CPU"); - ov::GenerationConfig config = pipe.get_generation_config(); - config.max_new_tokens = 256; - config.num_groups = 3; - config.group_size = 5; - config.diversity_penalty = 1.0f; + .. tab-item:: Python + :sync: py - cout << pipe.generate("The Sun is yellow because", config); - } + .. code-block:: python -Specify generation_config to use grouped beam search: + import openvino_genai as ov_genai + pipe = ov_genai.LLMPipeline(model_path, "CPU") + config = pipe.get_generation_config() + config.max_new_tokens = 256 + config.num_beam_groups = 3 + config.num_beams = 15 + config.diversity_penalty = 1.0 + pipe.generate("The Sun is yellow because", config) -.. tab-set:: .. tab-item:: C++ + :sync: cpp .. code-block:: cpp int main(int argc, char* argv[]) { - std::string prompt; - std::string model_path = argv[1]; - ov::LLMPipeline pipe(model_path, "CPU"); - - ov::GenerationConfig config = pipe.get_generation_config(); + ov::genai::LLMPipeline pipe(model_path, "CPU"); + ov::genai::GenerationConfig config = pipe.get_generation_config(); config.max_new_tokens = 256; - config.num_groups = 3; - config.group_size = 5; + config.num_beam_groups = 3; + config.num_beams = 15; config.diversity_penalty = 1.0f; - auto streamer = [](std::string word) { std::cout << word << std::flush; }; - - pipe.start_chat(); - for (size_t i = 0; i < questions.size(); i++) { - - std::cout << "question:\n"; - cout << prompt << endl; - - auto answer = pipe(prompt, config, streamer); - // no need to print answer, streamer will do that - } - pipe.finish_chat(); + cout << pipe.generate("The Sun is yellow because", config); } + Comparing with Hugging Face Results ####################################### @@ -237,6 +275,7 @@ Compare and analyze results with those generated by Hugging Face models. .. code-block:: python from transformers import AutoTokenizer, AutoModelForCausalLM + import openvino_genai as ov_genai tokenizer = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0") model = AutoModelForCausalLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0") @@ -249,12 +288,8 @@ Compare and analyze results with those generated by Hugging Face models. hf_output = tokenizer.decode(hf_encoded_output[0, encoded_prompt.shape[1]:]) print(f'hf_output: {hf_output}') - import sys - sys.path.append('build-Debug/') - import py_generate_pipeline as genai # set more friendly module name - - pipe = genai.LLMPipeline('text_generation/causal_lm/TinyLlama-1.1B-Chat-v1.0/pytorch/dldt/FP16/') - ov_output = pipe(prompt, max_new_tokens=max_new_tokens) + pipe = ov_genai.LLMPipeline('TinyLlama-1.1B-Chat-v1.0') + ov_output = pipe.generate(prompt, max_new_tokens=max_new_tokens) print(f'ov_output: {ov_output}') assert hf_output == ov_output diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst index a0edebf83d1749..4ee10dd9b10637 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst @@ -3,14 +3,16 @@ Run LLMs with Hugging Face and Optimum Intel ===================================================== -The steps below show how to load and infer LLMs from Hugging Face using Optimum Intel. -They also show how to convert models into OpenVINO IR format so they can be optimized -by NNCF and used with other OpenVINO tools. +The steps below show how to load and infer LLMs from +`Hugging Face `__ using Optimum Intel. They also show how to +convert models into OpenVINO IR format so they can be optimized by NNCF and used with +other OpenVINO tools. Prerequisites ############################################################ -* Create a Python environment by following the instructions on the :doc:`Install OpenVINO PIP <../../get-started/install-openvino>` page. +* Create a Python environment by following the instructions on the + :doc:`Install OpenVINO PIP <../../get-started/install-openvino>` page. * Install the necessary dependencies for Optimum Intel: .. code-block:: console @@ -20,7 +22,8 @@ Prerequisites Loading a Hugging Face Model to Optimum Intel ############################################################ -To start using OpenVINO as a backend for Hugging Face, change the original Hugging Face code in two places: +To start using OpenVINO as a backend for Hugging Face, change the original Hugging Face +code in two places: .. code-block:: diff @@ -225,10 +228,15 @@ OpenVINO runtime provides a set of optimizations for more efficient LLM inferenc includes **Dynamic quantization** of activations of 4/8-bit quantized MatMuls and **KV-cache quantization**. -* **Dynamic quantization** enables quantization of activations of MatMul operations that have 4 or 8-bit quantized weights (see :doc:`LLM Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`). - It improves inference latency and throughput of LLMs, though it may cause insignificant deviation in generation accuracy. Quantization is performed in a - group-wise manner, with configurable group size. It means that values in a group share quantization parameters. Larger group sizes lead to faster inference but lower accuracy. Recommended group size values are ``32``, ``64``, or ``128``. To enable Dynamic quantization, use the corresponding - inference property as follows: +* **Dynamic quantization** enables quantization of activations of MatMul operations + that have 4 or 8-bit quantized weights (see + :doc:`LLM Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`). + It improves inference latency and throughput of LLMs, though it may cause + insignificant deviation in generation accuracy. Quantization is performed in a group-wise + manner, with configurable group size. It means that values in a group share quantization + parameters. Larger group sizes lead to faster inference but lower accuracy. Recommended + group size values are ``32``, ``64``, or ``128``. To enable Dynamic quantization, use + the corresponding inference property as follows: .. code-block:: python @@ -238,9 +246,12 @@ includes **Dynamic quantization** of activations of 4/8-bit quantized MatMuls an ov_config={"DYNAMIC_QUANTIZATION_GROUP_SIZE": "32", "PERFORMANCE_HINT": "LATENCY"} ) -* **KV-cache quantization** allows lowering the precision of Key and Value cache in LLMs. This helps reduce memory consumption during inference, improving latency and throughput. KV-cache can be quantized into the following precisions: - ``u8``, ``bf16``, ``f16``. If ``u8`` is used, KV-cache quantization is also applied in a group-wise manner. Thus, it can use ``DYNAMIC_QUANTIZATION_GROUP_SIZE`` value if defined. - Otherwise, the group size ``32`` is used by default. KV-cache quantization can be enabled as follows: +* **KV-cache quantization** allows lowering the precision of Key and Value cache in LLMs. + This helps reduce memory consumption during inference, improving latency and throughput. + KV-cache can be quantized into the following precisions: ``u8``, ``bf16``, ``f16``. + If ``u8`` is used, KV-cache quantization is also applied in a group-wise manner. Thus, + it can use ``DYNAMIC_QUANTIZATION_GROUP_SIZE`` value if defined. Otherwise, the group + size ``32`` is used by default. KV-cache quantization can be enabled as follows: .. code-block:: python diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst index 12850aa5ab31f7..c1923e543e3b64 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst @@ -3,28 +3,41 @@ Run LLMs with Base OpenVINO =============================== -To run Generative AI models using native OpenVINO APIs you need to follow regular **Convert -> Optimize -> Deploy** path with a few simplifications. +To run Generative AI models using native OpenVINO APIs you need to follow regular +**Convert -> Optimize -> Deploy** path with a few simplifications. -To convert a model from Hugging Face, you can use Optimum-Intel export feature that allows you to export model in the OpenVINO format without invoking conversion API and tools directly. -In this case, the conversion process is a bit more simplified. You can still use a regular conversion path if the model comes from outside of Hugging Face ecosystem, i.e., in source framework format (PyTorch, etc.) +To convert a model from `Hugging Face `__, you can use +Optimum-Intel export feature that allows you to export model in the OpenVINO format without +invoking conversion API and tools directly. In this case, the conversion process is a bit +more simplified. You can still use a regular conversion path if the model comes from +outside of Hugging Face ecosystem, i.e., in source framework format (PyTorch, etc.) -Model optimization can be performed within Hugging Face or directly using NNCF as described in :doc:`Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`. +Model optimization can be performed within Hugging Face or directly using NNCF as described in +:doc:`Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`. .. note:: - It is recommended to use models in 4-bit precision, as maintaining the model in its original precision may result in significantly decreased performance. + It is recommended to use models in 4-bit precision, as maintaining the model in its + original precision may result in significantly decreased performance. -Inference code that uses native API cannot benefit from Hugging Face pipelines. You need to write your custom code or take it from the available examples. Below are some examples of popular Generative AI scenarios: +Inference code that uses native API cannot benefit from Hugging Face pipelines. +You need to write your custom code or take it from the available examples. Below are +some examples of popular Generative AI scenarios: -* In case of LLMs for text generation, you need to handle tokenization, inference and token selection loop, and de-tokenization. If token selection involves beam search, it also needs to be written. -* For image generation models, you need to make a pipeline that includes several model inferences: inference for source (e.g., text) encoder models, inference loop for diffusion process and inference for the decoding part. Scheduler code is also required. +* In case of LLMs for text generation, you need to handle tokenization, inference and + token selection loop, and de-tokenization. If token selection involves beam search, + it also needs to be written. +* For image generation models, you need to make a pipeline that includes several model + inferences: inference for source (e.g., text) encoder models, inference loop for + diffusion process and inference for the decoding part. Scheduler code is also required. To write such pipelines, you can follow the examples provided as part of OpenVINO: * `OpenVINO Latent Consistency Model C++ image generation pipeline `__ * `OpenVINO Stable Diffusion (with LoRA) C++ image generation pipeline `__ -To perform inference, models must be first converted to OpenVINO IR format using Hugging Face Optimum-Intel API. +To perform inference, models must be first converted to OpenVINO IR format using +Hugging Face Optimum-Intel API. An inference pipeline for a text generation LLM is set up in the following stages: @@ -179,4 +192,3 @@ Additional Resources * :doc:`Stateful Models Low-Level Details <../../openvino-workflow/running-inference/stateful-models>` * :doc:`Working with Textual Data <../../openvino-workflow/running-inference/string-tensors>` - diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst b/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst index 571743701ce01a..d670c9940b707c 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst @@ -338,7 +338,7 @@ Additional Resources * `OpenVINO Tokenizers repo `__ * `OpenVINO Tokenizers Notebook `__ -* `Text generation C++ samples that support most popular models like LLaMA 2 `__ +* `Text generation C++ samples that support most popular models like LLaMA 2 `__ * `OpenVINO GenAI Repo `__ diff --git a/docs/articles_en/openvino-workflow/deployment-locally/local-distribution-libraries.rst b/docs/articles_en/openvino-workflow/deployment-locally/local-distribution-libraries.rst index 9889f15c0ecbd9..90796e37e9abf3 100644 --- a/docs/articles_en/openvino-workflow/deployment-locally/local-distribution-libraries.rst +++ b/docs/articles_en/openvino-workflow/deployment-locally/local-distribution-libraries.rst @@ -10,20 +10,34 @@ Libraries for Local Distribution needed to deploy the application. -With local distribution, each C or C++ application/installer has its own copies of OpenVINO Runtime binaries. However, OpenVINO has a scalable plugin-based architecture, which means that some components can be loaded in runtime only when they are really needed. This guide helps you understand what minimal set of libraries is required to deploy the application. - -Local distribution is also suitable for OpenVINO binaries built from source using `Build instructions `__, -but this guide assumes that OpenVINO Runtime is built dynamically. For `Static OpenVINO Runtime `__, select the required OpenVINO capabilities at the CMake configuration stage using `CMake Options for Custom Compilation `__, then build and link the OpenVINO components to the final application. +With local distribution, each C or C++ application/installer has its own copies of OpenVINO Runtime binaries. +However, OpenVINO has a scalable plugin-based architecture, which means that some components +can be loaded in runtime only when they are really needed. This guide helps you understand +what minimal set of libraries is required to deploy the application. + +Local distribution is also suitable for OpenVINO binaries built from source using +`Build instructions `__, +but this guide assumes that OpenVINO Runtime is built dynamically. +For `Static OpenVINO Runtime `__, +select the required OpenVINO capabilities at the CMake configuration stage using +`CMake Options for Custom Compilation `__, +then build and link the OpenVINO components to the final application. .. note:: - The steps below are independent of the operating system and refer to the library file name without any prefixes (like ``lib`` on Unix systems) or suffixes (like ``.dll`` on Windows OS). Do not put ``.lib`` files on Windows OS to the distribution because such files are needed only at a linker stage. + The steps below are independent of the operating system and refer to the library file name + without any prefixes (like ``lib`` on Unix systems) or suffixes (like ``.dll`` on Windows OS). + Do not put ``.lib`` files on Windows OS to the distribution because such files are needed + only at a linker stage. Library Requirements for C++ and C Languages ############################################ -Regardless of the programming language of an application, the ``openvino`` library must always be included in its final distribution. This core library manages all inference and frontend plugins. The ``openvino`` library depends on the TBB libraries which are used by OpenVINO Runtime to optimally saturate devices with computations. +Regardless of the programming language of an application, the ``openvino`` library must always +be included in its final distribution. This core library manages all inference and frontend plugins. +The ``openvino`` library depends on the TBB libraries which are used by OpenVINO Runtime +to optimally saturate devices with computations. If your application is in C language, you need to additionally include the ``openvino_c`` library. @@ -48,7 +62,9 @@ For each inference device, OpenVINO Runtime has its own plugin library: Depending on which devices are used in the app, the corresponding libraries should be included in the distribution package. -As shown in the picture above, some plugin libraries may have OS-specific dependencies which are either backend libraries or additional supports files with firmware, etc. Refer to the table below for details: +As shown in the picture above, some plugin libraries may have OS-specific dependencies +which are either backend libraries or additional supports files with firmware, etc. +Refer to the table below for details: .. tab-set:: @@ -114,7 +130,8 @@ Libraries for Execution Modes The ``HETERO``, ``MULTI``, ``BATCH``, and ``AUTO`` execution modes can also be used by the application explicitly or implicitly. Use the following recommendation scheme to decide whether to add the appropriate libraries to the distribution package: -- If :doc:`AUTO <../running-inference/inference-devices-and-modes/auto-device-selection>` is used explicitly in the application or ov::Core::compile_model is used without specifying a device, put ``openvino_auto_plugin`` to the distribution. +- If :doc:`AUTO <../running-inference/inference-devices-and-modes/auto-device-selection>` is used + explicitly in the application or ``ov::Core::compile_model`` is used without specifying a device, put ``openvino_auto_plugin`` to the distribution. .. note:: @@ -122,9 +139,12 @@ Use the following recommendation scheme to decide whether to add the appropriate If you are not sure which inference devices are available on the target system, put all inference plugin libraries in the distribution. If ov::device::priorities is used for `AUTO` to specify a limited device list, grab the corresponding device plugins only. -- If :doc:`MULTI <../running-inference/inference-devices-and-modes/multi-device>` is used explicitly, put ``openvino_auto_plugin`` in the distribution. -- If :doc:`HETERO <../running-inference/inference-devices-and-modes/hetero-execution>` is either used explicitly or ov::hint::performance_mode is used with GPU, put ``openvino_hetero_plugin`` in the distribution. -- If :doc:`BATCH <../running-inference/inference-devices-and-modes/automatic-batching>` is either used explicitly or ``ov::hint::performance_mode`` is used with GPU, put ``openvino_batch_plugin`` in the distribution. +- If :doc:`MULTI <../running-inference/inference-devices-and-modes/multi-device>` is used explicitly, + put ``openvino_auto_plugin`` in the distribution. +- If :doc:`HETERO <../running-inference/inference-devices-and-modes/hetero-execution>` is either + used explicitly or ``ov::hint::performance_mode`` is used with GPU, put ``openvino_hetero_plugin`` in the distribution. +- If :doc:`BATCH <../running-inference/inference-devices-and-modes/automatic-batching>` is either + used explicitly or ``ov::hint::performance_mode`` is used with GPU, put ``openvino_batch_plugin`` in the distribution. Frontend Libraries for Reading Models +++++++++++++++++++++++++++++++++++++ @@ -138,46 +158,58 @@ OpenVINO Runtime uses frontend libraries dynamically to read models in different - ``openvino_paddle_frontend`` is used to read the Paddle file format. - ``openvino_pytorch_frontend`` is used to convert PyTorch model via ``openvino.convert_model`` API. -Depending on the model format types that are used in the application in ov::Core::read_model, select the appropriate libraries. +Depending on the model format types that are used in the application in ``ov::Core::read_model``, select the appropriate libraries. .. note:: - To optimize the size of the final distribution package, it is recommended to convert models to OpenVINO IR by using :doc:`model conversion API <../model-preparation>`. This way you do not have to keep TensorFlow, TensorFlow Lite, ONNX, PaddlePaddle, and other frontend libraries in the distribution package. + To optimize the size of the final distribution package, it is recommended to convert models + to OpenVINO IR by using :doc:`model conversion API <../model-preparation>`. This way you + do not have to keep TensorFlow, TensorFlow Lite, ONNX, PaddlePaddle, and other frontend + libraries in the distribution package. Examples #################### -**CPU + OpenVINO IR in C application** +.. dropdown:: CPU + OpenVINO IR in C application -In this example, the application is written in C, performs inference on CPU, and reads models stored in the OpenVINO IR format. + In this example, the application is written in C, performs inference on CPU, and reads models stored in the OpenVINO IR format. -The following libraries are used: ``openvino_c``, ``openvino``, ``openvino_intel_cpu_plugin``, and ``openvino_ir_frontend``. + The following libraries are used: ``openvino_c``, ``openvino``, ``openvino_intel_cpu_plugin``, and ``openvino_ir_frontend``. -- The ``openvino_c`` library is a main dependency of the application. The app links against this library. -- The ``openvino`` library is used as a private dependency for ``openvino_c`` and is also used in the deployment. -- ``openvino_intel_cpu_plugin`` is used for inference. -- ``openvino_ir_frontend`` is used to read source models. + - The ``openvino_c`` library is a main dependency of the application. The app links against this library. + - The ``openvino`` library is used as a private dependency for ``openvino_c`` and is also used in the deployment. + - ``openvino_intel_cpu_plugin`` is used for inference. + - ``openvino_ir_frontend`` is used to read source models. -**MULTI execution on GPU and CPU in `tput` mode** +.. dropdown:: MULTI execution on GPU and CPU in tput mode -In this example, the application is written in C++, performs inference :doc:`simultaneously on GPU and CPU devices <../running-inference/inference-devices-and-modes/multi-device>` with the ov::hint::PerformanceMode::THROUGHPUT property set, and reads models stored in the ONNX format. + In this example, the application is written in C++, performs inference + :doc:`simultaneously on GPU and CPU devices <../running-inference/inference-devices-and-modes/multi-device>` with + the ``ov::hint::PerformanceMode::THROUGHPUT`` property set, and reads models stored in the ONNX format. -The following libraries are used: ``openvino``, ``openvino_intel_gpu_plugin``, ``openvino_intel_cpu_plugin``, ``openvino_auto_plugin``, ``openvino_auto_batch_plugin``, and ``openvino_onnx_frontend``. + The following libraries are used: ``openvino``, ``openvino_intel_gpu_plugin``, ``openvino_intel_cpu_plugin``, + ``openvino_auto_plugin``, ``openvino_auto_batch_plugin``, and ``openvino_onnx_frontend``. -- The ``openvino`` library is a main dependency of the application. The app links against this library. -- ``openvino_intel_gpu_plugin`` and ``openvino_intel_cpu_plugin`` are used for inference. -- ``openvino_auto_plugin`` is used for Multi-Device Execution. -- ``openvino_auto_batch_plugin`` can be also put in the distribution to improve the saturation of :doc:`Intel® GPU <../running-inference/inference-devices-and-modes/gpu-device>` device. If there is no such plugin, :doc:`Automatic Batching <../running-inference/inference-devices-and-modes/automatic-batching>` is turned off. -- ``openvino_onnx_frontend`` is used to read source models. + - The ``openvino`` library is a main dependency of the application. The app links against this library. + - ``openvino_intel_gpu_plugin`` and ``openvino_intel_cpu_plugin`` are used for inference. + - ``openvino_auto_plugin`` is used for Multi-Device Execution. + - ``openvino_auto_batch_plugin`` can be also put in the distribution to improve the saturation + of :doc:`Intel® GPU <../running-inference/inference-devices-and-modes/gpu-device>` device. + If there is no such plugin, :doc:`Automatic Batching <../running-inference/inference-devices-and-modes/automatic-batching>` is turned off. + - ``openvino_onnx_frontend`` is used to read source models. -**Auto-Device Selection between GPU and CPU** +.. dropdown:: Auto-Device Selection between GPU and CPU -In this example, the application is written in C++, performs inference with the :doc:`Automatic Device Selection <../running-inference/inference-devices-and-modes/auto-device-selection>` mode, limiting device list to GPU and CPU, and reads models :doc:`created using C++ code <../running-inference/integrate-openvino-with-your-application/model-representation>`. + In this example, the application is written in C++, performs inference + with the :doc:`Automatic Device Selection <../running-inference/inference-devices-and-modes/auto-device-selection>` + mode, limiting device list to GPU and CPU, and reads models + :doc:`created using C++ code <../running-inference/integrate-openvino-with-your-application/model-representation>`. -The following libraries are used: ``openvino``, ``openvino_auto_plugin``, ``openvino_intel_gpu_plugin``, and ``openvino_intel_cpu_plugin``. + The following libraries are used: ``openvino``, ``openvino_auto_plugin``, ``openvino_intel_gpu_plugin``, and ``openvino_intel_cpu_plugin``. -- The ``openvino`` library is a main dependency of the application. The app links against this library. -- ``openvino_auto_plugin`` is used to enable Automatic Device Selection. -- ``openvino_intel_gpu_plugin`` and ``openvino_intel_cpu_plugin`` are used for inference. AUTO selects between CPU and GPU devices according to their physical existence on the deployed machine. -- No frontend library is needed because ``ov::Model`` is created in code. + - The ``openvino`` library is a main dependency of the application. The app links against this library. + - ``openvino_auto_plugin`` is used to enable Automatic Device Selection. + - ``openvino_intel_gpu_plugin`` and ``openvino_intel_cpu_plugin`` are used for inference. AUTO + selects between CPU and GPU devices according to their physical existence on the deployed machine. + - No frontend library is needed because ``ov::Model`` is created in code. diff --git a/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst b/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst index ed7604117a1a14..93adbf17b044f7 100644 --- a/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst +++ b/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst @@ -241,7 +241,7 @@ Build OpenVINO with conditional compilation enabled: cd %OPENVINO_HOME% md build_cc cd build_cc - cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Debug -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DENABLE_FASTER_BUILD=ON -DENABLE_SANITIZER=OFF -DTHREADING=TBB -DBUILD_SHARED_LIBS=OFF -DENABLE_PROFILING_ITT=ON -DSELECTIVE_BUILD=COLLECT -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DCMAKE_INSTALL_PREFIX=install .. + cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Debug -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DENABLE_FASTER_BUILD=ON -DENABLE_SANITIZER=OFF -DTHREADING=TBB -DBUILD_SHARED_LIBS=OFF -DENABLE_PROFILING_ITT=ON -DSELECTIVE_BUILD=COLLECT -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DCMAKE_INSTALL_PREFIX=install .. cmake --build . --config Debug @@ -274,7 +274,7 @@ Generate final optimal binaries size of OpenVINO package md build cd build - cmake -G "Visual Studio 16 2019" -A x64 -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_FASTER_BUILD=ON -DENABLE_PROFILING_ITT=OFF -DSELECTIVE_BUILD=ON -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DSELECTIVE_BUILD_STAT=%OPENVINO_HOME%\cc_data\*.csv -DBUILD_SHARED_LIBS=OFF -DENABLE_LTO=ON -DENABLE_ONEDNN_FOR_GPU=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_PROFILING_FIRST_INFERENCE=OFF .. + cmake -G "Visual Studio 16 2019" -A x64 -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_FASTER_BUILD=ON -DENABLE_PROFILING_ITT=OFF -DSELECTIVE_BUILD=ON -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DSELECTIVE_BUILD_STAT=%OPENVINO_HOME%\cc_data\*.csv -DBUILD_SHARED_LIBS=OFF -DENABLE_LTO=ON -DENABLE_ONEDNN_FOR_GPU=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_PROFILING_FIRST_INFERENCE=OFF .. cmake --build . --config Release diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst index ae4d03877f18ac..7a8a11a24bf56a 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/filter-pruning.rst @@ -228,8 +228,7 @@ To restore the model from checkpoint you should use the following API: :language: python :fragment: [load_checkpoint] -For more details on saving/loading checkpoints in the NNCF, see the following -`documentation `__. +For more details, see the following `documentation `__. Deploying pruned model ###################### diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/quantization-aware-training-tensorflow.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/quantization-aware-training-tensorflow.rst index 41a2ea615214a8..57abad31a42354 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/quantization-aware-training-tensorflow.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/compressing-models-during-training/quantization-aware-training-tensorflow.rst @@ -91,7 +91,8 @@ To restore the model from checkpoint, use the following API: :fragment: [load_checkpoint] -For more details on saving/loading checkpoints in the NNCF, see the corresponding `NNCF documentation `__. +For more details on saving/loading checkpoints in the NNCF, see the corresponding +`NNCF documentation `__. Deploying quantized model ######################### diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst index 7ce2893107e626..a393a0925cba3c 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst @@ -13,27 +13,32 @@ memory footprint, a crucial factor for Large Language Models (LLMs). Unlike full model quantization, where weights and activations are quantized, weight compression in `Neural Network Compression Framework (NNCF) `__ -only targets the model's weights. This approach -allows the activations to remain as floating-point numbers, preserving most -of the model's accuracy while improving its speed and reducing -its size. +only targets the model's weights. This approach allows the activations to remain as +floating-point numbers, preserving most of the model's accuracy while improving its +speed and reducing its size. The reduction in size is especially noticeable with larger models, for instance the 7 billion parameter Llama 2 model can be reduced -from about 25GB to 4GB using 4-bit weight compression. With smaller models (i.e. less than 1B parameters), -weight compression may result in more accuracy reduction than with larger models. +from about 25GB to 4GB using 4-bit weight compression. With smaller models (i.e. less +than 1B parameters), weight compression may result in more accuracy reduction than +with larger models. LLMs and other models that require extensive memory to store the weights during inference can benefit from weight compression as it: -* enables inference of exceptionally large models that cannot be accommodated in the device memory; +* enables inference of exceptionally large models that cannot be accommodated in the + device memory; -* reduces storage and memory overhead, making models more lightweight and less resource intensive for deployment; +* reduces storage and memory overhead, making models more lightweight and less resource + intensive for deployment; -* improves inference speed by reducing the latency of memory access when computing the operations with weights, for example, Linear layers. The weights are smaller and thus faster to load from memory; +* improves inference speed by reducing the latency of memory access when computing the + operations with weights, for example, Linear layers. The weights are smaller and thus + faster to load from memory; -* unlike quantization, does not require sample data to calibrate the range of activation values. +* unlike quantization, does not require sample data to calibrate the range of + activation values. Currently, `NNCF `__ provides weight quantization to 8 and 4-bit integer data types as a compression @@ -50,7 +55,8 @@ Transformer-based models. Models with 8-bit compressed weights are performant on vast majority of supported CPU and GPU platforms. -The code snippet below shows how to do 8-bit quantization of the model weights represented in OpenVINO IR using NNCF: +The code snippet below shows how to do 8-bit quantization of the model weights represented +in OpenVINO IR using NNCF: .. tab-set:: @@ -70,7 +76,8 @@ where INT4 is considered as the primary precision and INT8 is the backup one. It usually results in a smaller model size and lower inference latency, although the accuracy degradation could be higher, depending on the model. -The code snippet below shows how to do 4-bit quantization of the model weights represented in OpenVINO IR using NNCF: +The code snippet below shows how to do 4-bit quantization of the model weights represented +in OpenVINO IR using NNCF: .. tab-set:: @@ -108,13 +115,16 @@ memory reduction, speed gain, and accuracy loss. -The INT4 method has several parameters that can provide different performance-accuracy trade-offs after optimization: +The INT4 method has several parameters that can provide different performance-accuracy +trade-offs after optimization: * ``mode`` - there are two optimization modes: symmetric and asymmetric. **Symmetric Compression** - ``INT4_SYM`` - INT4 Symmetric mode involves quantizing weights to an unsigned 4-bit integer symmetrically with a fixed zero point of 8. This mode is faster than the INT8, making it ideal for situations where **speed and size reduction are prioritized over accuracy**. + INT4 Symmetric mode involves quantizing weights to an unsigned 4-bit integer + symmetrically with a fixed zero point of 8. This mode is faster than the INT8, making + it ideal for situations where **speed and size reduction are prioritized over accuracy**. .. code-block:: python @@ -125,7 +135,10 @@ The INT4 method has several parameters that can provide different performance-ac **Asymmetric Compression** - ``INT4_ASYM`` - INT4 Asymmetric mode also uses an unsigned 4-bit integer but quantizes weights asymmetrically with a non-fixed zero point. This mode slightly compromises speed in favor of better accuracy compared to the symmetric mode. This mode is useful when **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. + INT4 Asymmetric mode also uses an unsigned 4-bit integer but quantizes weights + asymmetrically with a non-fixed zero point. This mode slightly compromises speed in + favor of better accuracy compared to the symmetric mode. This mode is useful when + **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. .. code-block:: python @@ -134,19 +147,31 @@ The INT4 method has several parameters that can provide different performance-ac compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM) -* ``group_size`` controls the size of the group of weights that share the same quantization parameters. Shared quantization parameters help to speed up the calculation of activation values as they are dequantized and quantized between layers. However, they can reduce accuracy. The following group sizes are recommended: ``128``, ``64``, ``32`` (``128`` is default value). +* ``group_size`` controls the size of the group of weights that share the same + quantization parameters. Shared quantization parameters help to speed up the + calculation of activation values as they are dequantized and quantized between + layers. However, they can reduce accuracy. The following group sizes are + recommended: ``128``, ``64``, ``32`` (``128`` is default value). - `Smaller Group Size`: Leads to a more accurate model but increases the model's footprint and reduces inference speed. + `Smaller Group Size`: Leads to a more accurate model but increases the model's + footprint and reduces inference speed. - `Larger Group Size`: Results in faster inference and a smaller model, but might compromise accuracy. + `Larger Group Size`: Results in faster inference and a smaller model, but might + compromise accuracy. -* ``ratio`` controls the ratio between INT4 and INT8 compressed layers in the model. Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be compressed to INT4, while the rest will be compressed to INT8 precision. The default value for ratio is 1. +* ``ratio`` controls the ratio between INT4 and INT8 compressed layers in the model. + Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be + compressed to INT4, while the rest will be compressed to INT8 precision. The default + value for ratio is 1. - `Higher Ratio (more INT4)`: Reduces the model size and increase inference speed but might lead to higher accuracy degradation. + `Higher Ratio (more INT4)`: Reduces the model size and increase inference speed but + might lead to higher accuracy degradation. - `Lower Ratio (more INT8)`: Maintains better accuracy but results in a larger model size and potentially slower inference. + `Lower Ratio (more INT8)`: Maintains better accuracy but results in a larger model size + and potentially slower inference. - In this example, 90% of the model's layers are quantized to INT4 asymmetrically with a group size of 64: + In this example, 90% of the model's layers are quantized to INT4 asymmetrically with + a group size of 64: .. code-block:: python @@ -160,25 +185,43 @@ The INT4 method has several parameters that can provide different performance-ac ratio=0.9, ) -* ``dataset`` - calibration dataset for data-aware weight compression. It is required for some compression options, for example, some types ``sensitivity_metric`` can use data for precision selection. +* ``dataset`` - calibration dataset for data-aware weight compression. It is required + for some compression options, for example, some types ``sensitivity_metric`` can use + data for precision selection. -* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing layers in the bit-width selection algorithm. Some of the metrics require dataset to be provided. The following types are supported: +* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing + layers in the bit-width selection algorithm. Some of the metrics require dataset to be + provided. The following types are supported: - * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - data-free metric computed as the inverted 8-bit quantization noise. Weights with highest value of this metric can be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more accurate than per-channel, accuracy should not degrade. + * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - data-free metric computed as + the inverted 8-bit quantization noise. Weights with highest value of this metric can + be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in + 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more + accurate than per-channel, accuracy should not degrade. - * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires dataset. The average Hessian trace of weights with respect to the layer-wise quantization error multiplied by L2 norm of 8-bit quantization noise. + * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires dataset. The average + Hessian trace of weights with respect to the layer-wise quantization error multiplied + by L2 norm of 8-bit quantization noise. - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires dataset. The mean variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires dataset. The mean + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. - * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires dataset. The maximum variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires dataset. The maximum + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires dataset. The mean magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires dataset. The mean + magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. -* ``all_layers`` - boolean parameter that enables INT4 weight quantization of all Fully-Connected and Embedding layers, including the first and last layers in the model. +* ``all_layers`` - boolean parameter that enables INT4 weight quantization of all + Fully-Connected and Embedding layers, including the first and last layers in the model. -* ``awq`` - boolean parameter that enables the AWQ method for more accurate INT4 weight quantization. Especially helpful when the weights of all the layers are quantized to 4 bits. The method can sometimes result in reduced accuracy when used with Dynamic Quantization of activations. Requires dataset. +* ``awq`` - boolean parameter that enables the AWQ method for more accurate INT4 weight + quantization. Especially helpful when the weights of all the layers are quantized to + 4 bits. The method can sometimes result in reduced accuracy when used with + Dynamic Quantization of activations. Requires dataset. -For data-aware weight compression refer to the following `example `__. +For data-aware weight compression refer to the following +`example `__. The example below shows data-free 4-bit weight quantization applied on top of OpenVINO IR. Before trying the example, make sure Optimum Intel @@ -191,11 +234,11 @@ is installed in your environment by running the following command: The first example loads a pre-trained Hugging Face model using the Optimum Intel API, compresses it to INT4 using NNCF, and then executes inference with a text phrase. -If the model comes from Hugging Face and is supported by Optimum, it can -be easier to use the Optimum Intel API to perform weight compression. The compression -type is specified when the model is loaded using the ``load_in_8bit=True`` or ``load_in_4bit=True`` parameter. -The second example uses the Weight Compression API from Optimum Intel instead of NNCF -to compress the model to INT8. +If the model comes from `Hugging Face `__ and is supported +by Optimum, it may be easier to use the Optimum Intel API to perform weight compression. +The compression type is specified when the model is loaded using the ``load_in_8bit=True`` +or ``load_in_4bit=True`` parameter. The second example uses the Weight Compression API +from Optimum Intel instead of NNCF to compress the model to INT8. .. tab-set:: @@ -291,14 +334,17 @@ Make sure to install GPTQ dependencies by running the following command: results = pipe(phrase) print(results) -An `example of a model `__ that has been optimized using GPTQ. +An `example of a model `__ +that has been optimized using GPTQ. Compression Metrics Examples ######################################## -The table below shows examples of text-generation Language Models with different optimization settings in a data-free setup, where no dataset is used at the optimization step. -The Perplexity metric is a measurement of response accuracy, where a higher complexity score indicates a lower accuracy. -It is measured on the `Lambada OpenAI dataset `__. +The table below shows examples of text-generation Language Models with different +optimization settings in a data-free setup, where no dataset is used at the optimization step. +The Perplexity metric is a measurement of response accuracy, where a higher complexity +score indicates a lower accuracy. It is measured on the +`Lambada OpenAI dataset `__. .. list-table:: :widths: 40 55 25 25 @@ -370,7 +416,8 @@ It is measured on the `Lambada OpenAI dataset `__. +The following table shows accuracy metric in a data-aware 4-bit weight quantization +setup measured on the `Wikitext dataset `__. .. list-table:: :widths: 40 55 25 25 @@ -406,21 +453,28 @@ The following table shows accuracy metric in a data-aware 4-bit weight quantizat - 2.6 -\*Perplexity metric in both tables was measured without the Dynamic Quantization feature enabled in the OpenVINO runtime. +\*Perplexity metric in both tables was measured without the Dynamic Quantization feature +enabled in the OpenVINO runtime. Auto-tuning of Weight Compression Parameters ############################################ -To find the optimal weight compression parameters for a particular model, refer to the `example `__ , where weight compression parameters are being searched from the subset of values. To speed up the search, a self-designed -validation pipeline called `WhoWhatBench `__ is used. -The pipeline can quickly evaluate the changes in the accuracy of the optimized model compared to the baseline. +To find the optimal weight compression parameters for a particular model, refer to the +`example `__ , +where weight compression parameters are being searched from the subset of values. +To speed up the search, a self-designed validation pipeline called +`WhoWhatBench `__ +is used. The pipeline can quickly evaluate the changes in the accuracy of the optimized +model compared to the baseline. Additional Resources #################### - `Data-aware Weight Compression Example `__ - `Tune Weight Compression Parameters Example `__ -- `OpenVINO GenAI Repo `__: Repository containing example pipelines that implement image and text generation tasks. It also provides a tool to benchmark LLMs. +- `OpenVINO GenAI Repo `__ + : Repository containing example pipelines that implement image and text generation + tasks. It also provides a tool to benchmark LLMs. - `WhoWhatBench `__ - `NNCF GitHub `__ - :doc:`Post-training Quantization ` diff --git a/docs/articles_en/openvino-workflow/model-optimization.rst b/docs/articles_en/openvino-workflow/model-optimization.rst index 2cf08990d6b1d6..17903c760e1c4c 100644 --- a/docs/articles_en/openvino-workflow/model-optimization.rst +++ b/docs/articles_en/openvino-workflow/model-optimization.rst @@ -13,23 +13,48 @@ Model Optimization Guide model-optimization-guide/weight-compression -Model optimization is an optional offline step of improving the final model performance and reducing the model size by applying special optimization methods, such as 8-bit quantization, pruning, etc. OpenVINO offers three optimization paths implemented in `Neural Network Compression Framework (NNCF) `__: +Model optimization is an optional offline step of improving the final model performance +and reducing the model size by applying special optimization methods, such as 8-bit +quantization, pruning, etc. OpenVINO offers three optimization paths implemented in +`Neural Network Compression Framework (NNCF) `__: -- :doc:`Post-training Quantization ` is designed to optimize the inference of deep learning models by applying the post-training 8-bit integer quantization that does not require model retraining or fine-tuning. +- :doc:`Post-training Quantization ` + is designed to optimize the inference of deep learning models by applying the + post-training 8-bit integer quantization that does not require model retraining + or fine-tuning. + +- :doc:`Training-time Optimization ` + , a suite of advanced methods for training-time model optimization within the DL + framework, such as PyTorch and TensorFlow 2.x. It supports methods like + Quantization-aware Training, Structured and Unstructured Pruning, etc. -- :doc:`Training-time Optimization `, a suite of advanced methods for training-time model optimization within the DL framework, such as PyTorch and TensorFlow 2.x. It supports methods like Quantization-aware Training, Structured and Unstructured Pruning, etc. +- :doc:`Weight Compression `, + an easy-to-use method for Large Language Models footprint reduction and inference acceleration. -- :doc:`Weight Compression `, an easy-to-use method for Large Language Models footprint reduction and inference acceleration. +.. note:: -.. note:: OpenVINO also supports optimized models (for example, quantized) from source frameworks such as PyTorch, TensorFlow, and ONNX (in Q/DQ; Quantize/DeQuantize format). No special steps are required in this case and optimized models can be converted to the OpenVINO Intermediate Representation format (IR) right away. + OpenVINO also supports optimized models (for example, quantized) from source + frameworks such as PyTorch, TensorFlow, and ONNX (in Q/DQ; Quantize/DeQuantize format). + No special steps are required in this case and optimized models can be converted to + the OpenVINO Intermediate Representation format (IR) right away. -Post-training Quantization is the fastest way to optimize an arbitrary DL model and should be applied first, but it is limited in terms of achievable accuracy-performance trade-off. The recommended approach to obtain OpenVINO quantized model is to convert a model from original framework to ``ov.Model`` and ensure that the model works correctly in OpenVINO, for example, by calculating the model metrics. Then, ``ov.Model`` can be used as input for the ``nncf.quantize()`` method to get the quantized model or as input for the ``nncf.compress_weights()`` method to compress weights of Large Language Models (see the diagram below). +Post-training Quantization is the fastest way to optimize an arbitrary DL model and should +be applied first, but it is limited in terms of achievable accuracy-performance trade-off. +The recommended approach to obtain OpenVINO quantized model is to convert +`a model from original framework `__ to ``ov.Model`` +and ensure that the model works correctly in OpenVINO, for example, by calculating the +model metrics. Then, ``ov.Model`` can be used as input for the ``nncf.quantize()`` +method to get the quantized model or as input for the ``nncf.compress_weights()`` +method to compress weights of Large Language Models (see the diagram below). -In case of unsatisfactory accuracy or performance after Post-training Quantization, Training-time Optimization can be used as an option. +In case of unsatisfactory accuracy or performance after Post-training Quantization, +Training-time Optimization can be used as an option. .. image:: ../assets/images/DEVELOPMENT_FLOW_V3_crunch.svg -Once the model is optimized using the aforementioned methods, it can be used for inference using the regular OpenVINO inference workflow. No changes to the inference code are required. +Once the model is optimized using the aforementioned methods, it can be used for +inference using the regular OpenVINO inference workflow. No changes to the inference +code are required. .. image:: ../assets/images/WHAT_TO_USE.svg @@ -40,5 +65,5 @@ Additional Resources - :doc:`Training-time Optimization ` - :doc:`Weight Compression ` - :doc:`Deployment optimization ` -- `HuggingFace Optimum Intel `__ +- `Hugging Face Optimum Intel `__ diff --git a/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst b/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst index bb1626559f6987..6a7023fc16afc3 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst @@ -7,9 +7,15 @@ Conversion Parameters .. meta:: :description: Model Conversion API provides several parameters to adjust model conversion. -This document describes all available parameters for ``openvino.convert_model``, ``ovc``, and ``openvino.save_model`` without focusing on a particular framework model format. Use this information for your reference as a common description of the conversion API capabilities in general. Part of the options can be not relevant to some specific frameworks. Use :doc:`Supported Model Formats <../model-preparation>` page for more dedicated framework-dependent tutorials. +This document describes all available parameters for ``openvino.convert_model``, ``ovc``, +and ``openvino.save_model`` without focusing on a particular framework model format. +Use this information for your reference as a common description of the conversion API +capabilities in general. Part of the options can be not relevant to some specific +frameworks. Use :doc:`Supported Model Formats <../model-preparation>` page for more +dedicated framework-dependent tutorials. -In most cases when it is required to convert a model the following simple syntax can be used: +You can obtain a model from `Hugging Face `__. When you +need to convert it, in most cases, you can use the following simple syntax: .. tab-set:: @@ -36,15 +42,34 @@ In most cases when it is required to convert a model the following simple syntax ovc path_to_your_model -Providing just a path to the model or model object as ``openvino.convert_model`` argument is frequently enough to make a successful conversion. However, depending on the model topology and original deep learning framework, additional parameters may be required, which are described below. - -- ``example_input`` parameter available in Python ``openvino.convert_model`` only is intended to trace the model to obtain its graph representation. This parameter is crucial for converting PyTorch models and may sometimes be required for TensorFlow models. For more details, refer to the :doc:`PyTorch Model Conversion ` or :doc:`TensorFlow Model Conversion `. - -- ``input`` parameter to set or override shapes for model inputs. It configures dynamic and static dimensions in model inputs depending on your inference requirements. For more information on this parameter, refer to the :doc:`Setting Input Shapes ` guide. - -- ``output`` parameter to select one or multiple outputs from the original model. This is useful when the model has outputs that are not required for inference in a deployment scenario. By specifying only necessary outputs, you can create a more compact model that infers faster. - -- ``compress_to_fp16`` parameter that is provided by ``ovc`` CLI tool and ``openvino.save_model`` Python function, gives controls over the compression of model weights to FP16 format when saving OpenVINO model to IR. This option is enabled by default which means all produced IRs are saved using FP16 data type for weights which saves up to 2x storage space for the model file and in most cases doesn't sacrifice model accuracy. In case it does affect accuracy, the compression can be disabled by setting this flag to ``False``: +Providing just a path to the model or model object as ``openvino.convert_model`` argument +is frequently enough to make a successful conversion. However, depending on the model +topology and original deep learning framework, additional parameters may be required, +which are described below. + +- ``example_input`` parameter available in Python ``openvino.convert_model`` only is + intended to trace the model to obtain its graph representation. This parameter is crucial + for converting PyTorch models and may sometimes be required for TensorFlow models. + For more details, refer to the :doc:`PyTorch Model Conversion ` + or :doc:`TensorFlow Model Conversion `. + +- ``input`` parameter to set or override shapes for model inputs. It configures dynamic + and static dimensions in model inputs depending on your inference requirements. For more + information on this parameter, refer to the :doc:`Setting Input Shapes ` + guide. + +- ``output`` parameter to select one or multiple outputs from the original model. + This is useful when the model has outputs that are not required for inference in a + deployment scenario. By specifying only necessary outputs, you can create a more + compact model that infers faster. + +- ``compress_to_fp16`` parameter that is provided by ``ovc`` CLI tool and + ``openvino.save_model`` Python function, gives controls over the compression of + model weights to FP16 format when saving OpenVINO model to IR. This option is enabled + by default which means all produced IRs are saved using FP16 data type for weights + which saves up to 2x storage space for the model file and in most cases doesn't + sacrifice model accuracy. In case it does affect accuracy, the compression can be + disabled by setting this flag to ``False``: .. tab-set:: @@ -75,23 +100,59 @@ For details on how plugins handle compressed ``FP16`` models, see Refer to the :doc:`Post-training optimization <../model-optimization-guide/quantizing-models-post-training>` guide for more information about that. -- ``extension`` parameter which makes possible conversion of the models consisting of operations that are not supported by OpenVINO out-of-the-box. It requires implementing of an OpenVINO extension first, please refer to :doc:`Frontend Extensions <../../documentation/openvino-extensibility/frontend-extensions>` guide. - -- ``share_weigths`` parameter with default value ``True`` allows reusing memory with original weights. For models loaded in Python and then passed to ``openvino.convert_model``, that means that OpenVINO model will share the same areas in program memory where the original weights are located. For models loaded from files by ``openvino.convert_model``, file memory mapping is used to avoid extra memory allocation. When enabled, the original model cannot be modified (Python object cannot be deallocated and original model file cannot be deleted) for the whole lifetime of OpenVINO model. Even model inference by original framework can lead to model modification. If it is not desired, set ``share_weights=False`` when calling ``openvino.convert_model``. - -.. note:: ``ovc`` does not have ``share_weights`` option and always uses sharing to reduce conversion time and consume less amount of memory during the conversion. +- ``extension`` parameter which makes possible conversion of the models consisting of + operations that are not supported by OpenVINO out-of-the-box. It requires implementing of + an OpenVINO extension first, please refer to + :doc:`Frontend Extensions <../../documentation/openvino-extensibility/frontend-extensions>` + guide. + +- ``share_weigths`` parameter with default value ``True`` allows reusing memory with + original weights. For models loaded in Python and then passed to ``openvino.convert_model``, + that means that OpenVINO model will share the same areas in program memory where the + original weights are located. For models loaded from files by ``openvino.convert_model``, + file memory mapping is used to avoid extra memory allocation. When enabled, the + original model cannot be modified (Python object cannot be deallocated and original + model file cannot be deleted) for the whole lifetime of OpenVINO model. Even model + inference by original framework can lead to model modification. If it is not desired, + set ``share_weights=False`` when calling ``openvino.convert_model``. + + .. note:: + + ``ovc`` does not have ``share_weights`` option and always uses sharing to reduce + conversion time and consume less amount of memory during the conversion. + +- ``output_model`` parameter in ``ovc`` and ``openvino.save_model`` specifies name for + output ``.xml`` file with the resulting OpenVINO IR. The accompanying ``.bin`` file + name will be generated automatically by replacing ``.xml`` extension with ``.bin`` + extension. The value of ``output_model`` must end with ``.xml`` extension. For ``ovc`` + command line tool, ``output_model`` can also contain a name of a directory. In this case, + the resulting OpenVINO IR files will be put into that directory with a base name of + ``.xml`` and ``.bin`` files matching the original model base name passed to ``ovc`` as a + parameter. For example, when calling ``ovc your_model.onnx --output_model directory_name``, + files ``directory_name/your_model.xml`` and ``directory_name/your_model.bin`` will be + created. If ``output_model`` is not used, then the current directory is used as + a destination directory. + + .. note:: + + ``openvino.save_model`` does not support a directory for ``output_model`` + parameter value because ``openvino.save_model`` gets OpenVINO model object + represented in a memory and there is no original model file name available for + output file name generation. For the same reason, ``output_model`` is a mandatory + parameter for ``openvino.save_model``. + +- ``verbose`` parameter activates extra diagnostics printed to the standard output. + Use for debugging purposes in case there is an issue with the conversion and to collect + information for better bug reporting to OpenVINO team. -- ``output_model`` parameter in ``ovc`` and ``openvino.save_model`` specifies name for output ``.xml`` file with the resulting OpenVINO IR. The accompanying ``.bin`` file name will be generated automatically by replacing ``.xml`` extension with ``.bin`` extension. The value of ``output_model`` must end with ``.xml`` extension. For ``ovc`` command line tool, ``output_model`` can also contain a name of a directory. In this case, the resulting OpenVINO IR files will be put into that directory with a base name of ``.xml`` and ``.bin`` files matching the original model base name passed to ``ovc`` as a parameter. For example, when calling ``ovc your_model.onnx --output_model directory_name``, files ``directory_name/your_model.xml`` and ``directory_name/your_model.bin`` will be created. If ``output_model`` is not used, then the current directory is used as a destination directory. - -.. note:: ``openvino.save_model`` does not support a directory for ``output_model`` parameter value because ``openvino.save_model`` gets OpenVINO model object represented in a memory and there is no original model file name available for output file name generation. For the same reason, ``output_model`` is a mandatory parameter for ``openvino.save_model``. - -- ``verbose`` parameter activates extra diagnostics printed to the standard output. Use for debugging purposes in case there is an issue with the conversion and to collect information for better bug reporting to OpenVINO team. +.. note:: -.. note:: Weights sharing does not equally work for all the supported model formats. The value of this flag is considered as a hint for the conversion API, and actual sharing is used only if it is implemented and possible for a particular model representation. + Weights sharing does not equally work for all the supported model formats. The value + of this flag is considered as a hint for the conversion API, and actual sharing is + used only if it is implemented and possible for a particular model representation. -You can always run ``ovc -h`` or ``ovc --help`` to recall all the supported parameters for ``ovc``. +You can always run ``ovc -h`` or ``ovc --help`` to recall all the supported +parameters for ``ovc``. Use ``ovc --version`` to check the version of OpenVINO package installed. - - diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst index b52552ad55730e..9c903388a5c3bd 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst @@ -11,18 +11,30 @@ Converting an ONNX Model Introduction to ONNX #################### -`ONNX `__ is a representation format for deep learning models that enables AI developers to easily transfer models between different frameworks. +`ONNX `__ is a representation format for deep learning models +that enables AI developers to easily transfer models between different frameworks. -.. note:: An ONNX model file can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without the need to prepare an OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if the model load latency is important for the inference application. +.. note:: + + An ONNX model file can be loaded by ``openvino.Core.read_model`` or + ``openvino.Core.compile_model`` methods by OpenVINO runtime API without the need to + prepare an OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if the model + load latency is important for the inference application. Converting an ONNX Model ######################## -This page provides instructions on model conversion from the ONNX format to the OpenVINO IR format. +This page provides instructions on model conversion from the ONNX format to the +OpenVINO IR format. -For model conversion, you need an ONNX model either directly downloaded from a public repository or converted from any framework that supports exporting to the ONNX format. +For model conversion, you need an ONNX model either directly downloaded from +an online database, for example `Hugging Face `__ , or +converted from any framework that supports exporting to the ONNX format. -To convert an ONNX model, run model conversion with the path to the input model ``.onnx`` file: +To convert an ONNX model, run model conversion with the path to the input +model ``.onnx`` file: .. tab-set:: diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst index c70724b4047f95..97c968bc4a7692 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst @@ -8,16 +8,31 @@ Converting a PaddlePaddle Model :description: Learn how to convert a model from the PaddlePaddle format to the OpenVINO Model. -This page provides general instructions on how to convert a model from the PaddlePaddle format to the OpenVINO IR format using OpenVINO model conversion API. The instructions are different depending on the PaddlePaddle model format. +This page provides general instructions on how to convert a model from the PaddlePaddle +format to the OpenVINO IR format using OpenVINO model conversion API. The instructions +are different depending on the PaddlePaddle model format. -.. note:: PaddlePaddle model serialized in a file can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if model load latency matters for the inference application. +.. note:: + + PaddlePaddle model serialized in a file can be loaded by ``openvino.Core.read_model`` + or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing + OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if model load + latency matters for the inference application. Converting PaddlePaddle Model Files ################################### -PaddlePaddle inference model includes ``.pdmodel`` (storing model structure) and ``.pdiparams`` (storing model weight). For details on how to export a PaddlePaddle inference model, refer to the `Exporting PaddlePaddle Inference Model `__ Chinese guide. +PaddlePaddle inference model includes ``.pdmodel`` (storing model structure) and +``.pdiparams`` (storing model weight). For details on how to export a PaddlePaddle +inference model, refer to the +`Exporting PaddlePaddle Inference Model `__ +Chinese guide. -To convert a PaddlePaddle model, use the ``ovc`` or ``openvino.convert_model`` and specify the path to the input ``.pdmodel`` model file: +You can download a PaddlePaddle model from `Hugging Face `__. +To convert the model, use the ``ovc`` or ``openvino.convert_model`` +and specify the path to the input ``.pdmodel`` model file: .. tab-set:: diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst index 7289c6cbe020c0..6ac806daf0cda0 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst @@ -7,7 +7,9 @@ Converting a PyTorch Model PyTorch format to the OpenVINO Model. -To convert a PyTorch model, use the ``openvino.convert_model`` function. +You can download a PyTorch model from `PyTorch Hub `__ or +`Hugging Face `__. +To convert the model, use the ``openvino.convert_model`` function. Here is the simplest example of PyTorch model conversion using a model from ``torchvision``: diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst index ecf08f9ff55575..93738f48f1d39f 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst @@ -9,7 +9,10 @@ Converting a TensorFlow Lite Model TensorFlow Lite format to the OpenVINO Model. -To convert an TensorFlow Lite model, run model conversion with the path to the ``.tflite`` model file: +You can download a TensorFlow Lite model from +`Kaggle `__ +or `Hugging Face `__. +To convert the model, run model conversion with the path to the ``.tflite`` model file: .. tab-set:: @@ -28,16 +31,27 @@ To convert an TensorFlow Lite model, run model conversion with the path to the ` ovc your_model_file.tflite -.. note:: TensorFlow Lite model file can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if model load latency matters for the inference application. +.. note:: + + TensorFlow Lite model file can be loaded by ``openvino.Core.read_model`` or + ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing + OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if model + load latency matters for the inference application. Supported TensorFlow Lite Layers ################################### -For the list of supported standard layers, refer to the :doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` page. +For the list of supported standard layers, refer to the +:doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` +page. Supported TensorFlow Lite Models ################################### -More than eighty percent of public TensorFlow Lite models are supported from open sources `TensorFlow Hub `__ and `MediaPipe `__. +More than eighty percent of public TensorFlow Lite models are supported from open +sources `Kaggle `__ +and `MediaPipe `__. Unsupported models usually have custom TensorFlow Lite operations. diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst index 84aa153cfb82be..e66a5a87c48cd3 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst @@ -8,13 +8,39 @@ Converting a TensorFlow Model :description: Learn how to convert a model from a TensorFlow format to the OpenVINO Model. -This page provides general instructions on how to run model conversion from a TensorFlow format to the OpenVINO IR format. The instructions are different depending on whether your model was created with TensorFlow v1.X or TensorFlow v2.X. +This page provides general instructions on how to run model conversion from a TensorFlow +format to the OpenVINO IR format. The instructions are different depending on whether +your model was created with TensorFlow v1.X or TensorFlow v2.X. -.. note:: TensorFlow models can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if model load latency matters for the inference application. +TensorFlow models can be obtained from +`Kaggle `__ +or `Hugging Face `__. -.. note:: ``openvino.convert_model`` uses sharing of model weights by default. That means that OpenVINO model will share the same areas in program memory where the original weights are located, for this reason the original model cannot be modified (Python object cannot be deallocated and original model file cannot be deleted) for the whole lifetime of OpenVINO model. Model inference for TensorFlow models can lead to model modification, so original TF model should not be inferred during the lifetime of OpenVINO model. If it is not desired, set ``share_weights=False`` when calling ``openvino.convert_model``. +.. note:: + + TensorFlow models can be loaded by ``openvino.Core.read_model`` or + ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing + OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if model load + latency matters for the inference application. + +.. note:: + + ``openvino.convert_model`` uses sharing of model weights by default. That means that + OpenVINO model will share the same areas in program memory where the original weights + are located, for this reason the original model cannot be modified (Python object + cannot be deallocated and original model file cannot be deleted) for the whole + lifetime of OpenVINO model. Model inference for TensorFlow models can lead to model + modification, so original TF model should not be inferred during the lifetime of + OpenVINO model. If it is not desired, set ``share_weights=False`` when calling + ``openvino.convert_model``. + +.. note:: -.. note:: Examples below that convert TensorFlow models from a file, do not require any version of TensorFlow to be installed on the system, except in cases when the ``tensorflow`` module is imported explicitly. + Examples below that convert TensorFlow models from a file, do not require any version + of TensorFlow to be installed on the system, except in cases when the ``tensorflow`` + module is imported explicitly. Converting TensorFlow 2 Models ############################## @@ -25,7 +51,8 @@ Below are the instructions on how to convert each of them. SavedModel Format +++++++++++++++++ -A model in the SavedModel format consists of a directory with a ``saved_model.pb`` file and two subfolders: ``variables`` and ``assets`` inside. +A model in the SavedModel format consists of a directory with a ``saved_model.pb`` +file and two subfolders: ``variables`` and ``assets`` inside. To convert a model, run conversion with the directory as the model argument: .. tab-set:: @@ -59,8 +86,9 @@ SavedModel format. Here is an example of how to do it: model = tf.keras.models.load_model('model.h5') tf.saved_model.save(model,'model') -Converting a Keras H5 model with a custom layer to the SavedModel format requires special considerations. -For example, the model with a custom layer ``CustomLayer`` from ``custom_layer.py`` is converted as follows: +Converting a Keras H5 model with a custom layer to the SavedModel format requires special +considerations. For example, the model with a custom layer ``CustomLayer`` from +``custom_layer.py`` is converted as follows: .. code-block:: py :force: @@ -82,7 +110,8 @@ Converting TensorFlow 1 Models Converting Frozen Model Format +++++++++++++++++++++++++++++++ -To convert a TensorFlow model, run model conversion with the path to the input model ``*.pb*`` file: +To convert a TensorFlow model, run model conversion with the path to the input +model ``*.pb*`` file: .. tab-set:: @@ -107,9 +136,12 @@ Converting Non-Frozen Model Formats There are three ways to store non-frozen TensorFlow models. -1. **SavedModel format**. In this case, a model consists of a special directory with a ``.pb`` file -and several subfolders: ``variables``, ``assets``, and ``assets.extra``. For more information about the SavedModel directory, refer to the `README `__ file in the TensorFlow repository. -To convert such TensorFlow model, run the conversion similarly to other model formats and pass a path to the directory as a model argument: +1. **SavedModel format**. In this case, a model consists of a special directory with a + ``.pb`` file and several subfolders: ``variables``, ``assets``, and ``assets.extra``. + For more information about the SavedModel directory, refer to the + `README `__ + file in the TensorFlow repository. To convert such TensorFlow model, run the conversion + similarly to other model formats and pass a path to the directory as a model argument: .. tab-set:: @@ -128,9 +160,12 @@ To convert such TensorFlow model, run the conversion similarly to other model fo ovc path_to_saved_model_dir -2. **Checkpoint**. In this case, a model consists of two files: ``inference_graph.pb`` (or ``inference_graph.pbtxt``) and ``checkpoint_file.ckpt``. -If you do not have an inference graph file, refer to the `Freezing Custom Models in Python <#Freezing-Custom-Models-in-Python>`__ section. -To convert the model with the inference graph in ``.pb`` format, provide paths to both files as an argument for ``ovc`` or ``openvino.convert_model``: +2. **Checkpoint**. In this case, a model consists of two files: ``inference_graph.pb`` + (or ``inference_graph.pbtxt``) and ``checkpoint_file.ckpt``. + If you do not have an inference graph file, refer to the + `Freezing Custom Models in Python <#freezing-custom-models-in-python>`__ section. + To convert the model with the inference graph in ``.pb`` format, provide paths to both + files as an argument for ``ovc`` or ``openvino.convert_model``: .. tab-set:: @@ -149,11 +184,16 @@ To convert the model with the inference graph in ``.pb`` format, provide paths t ovc path_to_inference_graph.pb path_to_checkpoint_file.ckpt -To convert the model with the inference graph in the ``.pbtxt`` format, specify the path to ``.pbtxt`` file instead of the ``.pb`` file. The conversion API automatically detects the format of the provided file, there is no need to specify the model file format explicitly when calling ``ovc`` or ``openvino.convert_model`` in all examples in this document. +To convert the model with the inference graph in the ``.pbtxt`` format, specify the path +to ``.pbtxt`` file instead of the ``.pb`` file. The conversion API automatically detects +the format of the provided file, there is no need to specify the model file format +explicitly when calling ``ovc`` or ``openvino.convert_model`` in all examples in this document. -3. **MetaGraph**. In this case, a model consists of three or four files stored in the same directory: ``model_name.meta``, ``model_name.index``, -``model_name.data-00000-of-00001`` (the numbers may vary), and ``checkpoint`` (optional). -To convert such a TensorFlow model, run the conversion providing a path to `.meta` file as an argument: +3. **MetaGraph**. In this case, a model consists of three or four files stored in the same + directory: ``model_name.meta``, ``model_name.index``, + ``model_name.data-00000-of-00001`` (the numbers may vary), and ``checkpoint`` (optional). + To convert such a TensorFlow model, run the conversion providing a path to ``.meta`` + file as an argument: .. tab-set:: @@ -176,9 +216,11 @@ To convert such a TensorFlow model, run the conversion providing a path to `.met Freezing Custom Models in Python ++++++++++++++++++++++++++++++++ -When a model is defined in Python code, you must create an inference graph file. Graphs are usually built in a form -that allows model training. That means all trainable parameters are represented as variables in the graph. -To be able to use such a graph with the model conversion API, it should be frozen first before passing to the ``openvino.convert_model`` function: +When a model is defined in Python code, you must create an inference graph file. Graphs are +usually built in a form that allows model training. That means all trainable parameters are +represented as variables in the graph. To be able to use such a graph with the model +conversion API, it should be frozen first before passing to the +``openvino.convert_model`` function: .. code-block:: py :force: @@ -192,22 +234,28 @@ To be able to use such a graph with the model conversion API, it should be froze Where: -* ``sess`` is the instance of the TensorFlow Session object where the network topology is defined. -* ``["name_of_the_output_node"]`` is the list of output node names in the graph; ``frozen`` graph will include only those nodes from the original ``sess.graph_def`` that are directly or indirectly used to compute given output nodes. The ``'name_of_the_output_node'`` is an example of a possible output node name. You should derive the names based on your own graph. +* ``sess`` is the instance of the TensorFlow Session object where the network topology + is defined. +* ``["name_of_the_output_node"]`` is the list of output node names in the graph; + ``frozen`` graph will include only those nodes from the original ``sess.graph_def`` + that are directly or indirectly used to compute given output nodes. The + ``'name_of_the_output_node'`` is an example of a possible output node name. + You should derive the names based on your own graph. Converting TensorFlow Models from Memory Using Python API ############################################################ Model conversion API supports passing TensorFlow/TensorFlow2 models directly from memory. -* ``Trackable``. The object returned by ``hub.load()`` can be converted to ``ov.Model`` with ``convert_model()``. +* ``Trackable``. The object returned by ``hub.load()`` can be converted to + ``ov.Model`` with ``convert_model()``. .. code-block:: py :force: import tensorflow_hub as hub import openvino as ov - + model = hub.load("https://tfhub.dev/google/movenet/singlepose/lightning/4") ov_model = ov.convert_model(model) @@ -234,7 +282,10 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro model = tf.keras.applications.ResNet50(weights="imagenet") ov_model = ov.convert_model(model) -* ``tf.keras.layers.Layer``. The ``ov.Model`` converted from ``tf.keras.layers.Layer`` does not contain original input and output names. So it is recommended to convert the model to ``tf.keras.Model`` before conversion or use ``hub.load()`` for TensorFlow Hub models. +* ``tf.keras.layers.Layer``. The ``ov.Model`` converted from ``tf.keras.layers.Layer`` + does not contain original input and output names. So it is recommended to convert the + model to ``tf.keras.Model`` before conversion or use ``hub.load()`` for + TensorFlow Hub models. .. code-block:: py :force: @@ -265,7 +316,12 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro model = MyModule(name="simple_module") ov_model = ov.convert_model(model, input=[-1]) -.. note:: There is a known bug in ``openvino.convert_model`` on using ``tf.Variable`` nodes in the model graph. The results of the conversion of such models are unpredictable. It is recommended to save a model with ``tf.Variable`` into TensorFlow Saved Model format and load it with ``openvino.convert_model``. +.. note:: + + There is a known bug in ``openvino.convert_model`` on using ``tf.Variable`` nodes in + the model graph. The results of the conversion of such models are unpredictable. It + is recommended to save a model with ``tf.Variable`` into TensorFlow Saved Model format + and load it with ``openvino.convert_model``. * ``tf.compat.v1.Graph`` @@ -328,7 +384,9 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro Supported TensorFlow and TensorFlow 2 Keras Layers ################################################## -For the list of supported standard layers, refer to the :doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` page. +For the list of supported standard layers, refer to the +:doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` +page. Summary ####### @@ -339,5 +397,3 @@ In this document, you learned: * Which TensorFlow models are supported. * How to freeze a TensorFlow model. - - diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst index ac008631e49fff..be67f581173309 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst @@ -18,7 +18,8 @@ Convert to OpenVINO IR Convert from PaddlePaddle -:doc:`IR (Intermediate Representation) <../../documentation/openvino-ir-format>` is OpenVINO own format consisting of ``.xml`` and ``.bin`` files. +:doc:`IR (Intermediate Representation) <../../documentation/openvino-ir-format>` is +OpenVINO own format consisting of ``.xml`` and ``.bin`` files. Convert the model into OpenVINO IR for `better performance <#ir-conversion-benefits>`__. Convert Models @@ -616,4 +617,5 @@ Additional Resources #################### * :doc:`Transition guide from the legacy to new conversion API <../../documentation/legacy-features/transition-legacy-conversion-api>` +* `Download models from Hugging Face `__. diff --git a/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst b/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst index 7ab05fed2ec56a..9e17fdaad9999c 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst @@ -20,7 +20,8 @@ performance and memory consumption. For more information on changing input shapes in runtime, refer to the :doc:`Changing input shapes <../running-inference/changing-input-shape>` guide. To learn more about dynamic shapes in runtime, refer to the -:doc:`Dynamic Shapes <../running-inference/dynamic-shapes>` guide. +:doc:`Dynamic Shapes <../running-inference/dynamic-shapes>` guide. To download models, +you can visit `Hugging Face `__. The OpenVINO Runtime API may present certain limitations in inferring models with undefined dimensions on some hardware. See the :doc:`Feature support matrix <../../about-openvino/compatibility-and-support/supported-devices>` @@ -49,9 +50,13 @@ and specify the input shape of ``[2,300,300,3]``: ovc MobileNet.pb --input [2,300,300,3] -If a model has multiple inputs, the input shape should be specified in ``input`` parameter as a list. In ``ovc``, this is a command separate list, and in ``openvino.convert_model`` this is a Python list or tuple with number of elements matching the number of inputs in the model. Use input names from the original model to define the mapping between inputs and shapes specified. -The following example demonstrates the conversion of the ONNX OCR model with a pair of inputs ``data`` and ``seq_len`` -and specifies shapes ``[3,150,200,1]`` and ``[3]`` for them respectively: +If a model has multiple inputs, the input shape should be specified in ``input`` parameter +as a list. In ``ovc``, this is a command separate list, and in ``openvino.convert_model`` +this is a Python list or tuple with number of elements matching the number of inputs in +the model. Use input names from the original model to define the mapping between inputs +and shapes specified. The following example demonstrates the conversion of the ONNX OCR +model with a pair of inputs ``data`` and ``seq_len`` and specifies shapes ``[3,150,200,1]`` +and ``[3]`` for them respectively: .. tab-set:: @@ -71,7 +76,9 @@ and specifies shapes ``[3,150,200,1]`` and ``[3]`` for them respectively: ovc ocr.onnx --input data[3,150,200,1],seq_len[3] -If the order of inputs is defined in the input model and the order is known for the user, names could be omitted. In this case, it is important to specify shapes in the same order of input model inputs: +If the order of inputs is defined in the input model and the order is known for the user, +names could be omitted. In this case, it is important to specify shapes in the +same order of input model inputs: .. tab-set:: @@ -91,12 +98,19 @@ If the order of inputs is defined in the input model and the order is known for ovc ocr.onnx --input [3,150,200,1],[3] -Whether the model has a specified order of inputs depends on the original framework. Usually, it is convenient to set shapes without specifying the names of the parameters in the case of PyTorch model conversion because a PyTorch model is considered as a callable that usually accepts positional parameters. On the other hand, names of inputs are convenient when converting models from model files, because naming of inputs is a good practice for many frameworks that serialize models to files. +Whether the model has a specified order of inputs depends on the original framework. +Usually, it is convenient to set shapes without specifying the names of the parameters +in the case of PyTorch model conversion because a PyTorch model is considered as +a callable that usually accepts positional parameters. On the other hand, names of inputs +are convenient when converting models from model files, because naming of inputs is +a good practice for many frameworks that serialize models to files. -The ``input`` parameter allows overriding original input shapes if it is supported by the model topology. -Shapes with dynamic dimensions in the original model can be replaced with static shapes for the converted model, and vice versa. -The dynamic dimension can be marked in model conversion API parameter as ``-1`` or ``?`` when using ``ovc``. -For example, launch model conversion for the ONNX OCR model and specify dynamic batch dimension for inputs: +The ``input`` parameter allows overriding original input shapes if it is supported by +the model topology. Shapes with dynamic dimensions in the original model can be replaced +with static shapes for the converted model, and vice versa. The dynamic dimension can be +marked in model conversion API parameter as ``-1`` or ``?`` when using ``ovc``. +For example, launch model conversion for the ONNX OCR model and specify dynamic batch +dimension for inputs: .. tab-set:: @@ -116,9 +130,13 @@ For example, launch model conversion for the ONNX OCR model and specify dynamic ovc ocr.onnx --input "data[?,150,200,1],seq_len[?]" -To optimize memory consumption for models with undefined dimensions in run-time, model conversion API provides the capability to define boundaries of dimensions. -The boundaries of undefined dimension can be specified with ellipsis in the command line or with ``openvino.Dimension`` class in Python. -For example, launch model conversion for the ONNX OCR model and specify a boundary for the batch dimension 1..3, which means that the input tensor will have batch dimension minimum 1 and maximum 3 in inference: +To optimize memory consumption for models with undefined dimensions in run-time, +model conversion API provides the capability to define boundaries of dimensions. +The boundaries of undefined dimension can be specified with ellipsis in the command +line or with ``openvino.Dimension`` class in Python. +For example, launch model conversion for the ONNX OCR model and specify a boundary for +the batch dimension 1..3, which means that the input tensor will have batch dimension +minimum 1 and maximum 3 in inference: .. tab-set:: @@ -139,5 +157,9 @@ For example, launch model conversion for the ONNX OCR model and specify a bounda ovc ocr.onnx --input data[1..3,150,200,1],seq_len[1..3] -In practice, not every model is designed in a way that allows change of input shapes. An attempt to change the shape for such models may lead to an exception during model conversion, later in model inference, or even to wrong results of inference without explicit exception raised. A knowledge about model topology is required to set shapes appropriately. +In practice, not every model is designed in a way that allows change of input shapes. +An attempt to change the shape for such models may lead to an exception during model +conversion, later in model inference, or even to wrong results of inference without +explicit exception raised. A knowledge about model topology is required to set +shapes appropriately. diff --git a/docs/articles_en/openvino-workflow/running-inference.rst b/docs/articles_en/openvino-workflow/running-inference.rst index 3ccd9f3ff7cc2e..95c6bc66e902b8 100644 --- a/docs/articles_en/openvino-workflow/running-inference.rst +++ b/docs/articles_en/openvino-workflow/running-inference.rst @@ -39,9 +39,9 @@ OpenVINO IR provides by far the best first-inference latency scores. For more detailed information on how to convert, read, and compile supported model formats see the :doc:`Model Preparation article `. - Note that TensorFlow models can be run using the + Note that PyTorch models can be run using the :doc:`torch.compile feature `, as well as the standard ways of - :doc:`converting TensorFlow ` + :doc:`converting Pytorch ` or running its inference. OpenVINO Runtime uses a plugin architecture. Its plugins are software components that contain complete implementation for inference on a particular Intel® hardware device: CPU, GPU, etc. Each plugin implements the unified API and provides additional hardware-specific APIs for configuring devices or API interoperability between OpenVINO Runtime and underlying plugin backend. diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst index 19328ec7a9c1f9..c11696ce07bc18 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst @@ -83,4 +83,3 @@ Accordingly, the code that loops over all available devices of the "GPU" type on :language: cpp :fragment: [part3] - diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst index d087e369ff117d..69bdf26eb0cf79 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst @@ -61,15 +61,14 @@ How AUTO Works ############## To put it simply, when loading the model to the first device on the list fails, AUTO will try to load it to the next device in line, until one of them succeeds. -What is important, **AUTO starts inference with the CPU of the system by default**, as it provides very low latency and can start inference with no additional delays. +What is important, **AUTO starts inference with the CPU of the system by default unless there is model cached for the best suited device**, as it provides very low latency and can start inference with no additional delays. While the CPU is performing inference, AUTO continues to load the model to the device best suited for the purpose and transfers the task to it when ready. This way, the devices which are much slower in compiling models, GPU being the best example, do not impact inference at its initial stages. For example, if you use a CPU and a GPU, the first-inference latency of AUTO will be better than that of using GPU alone. Note that if you choose to exclude CPU from the priority list or disable the initial CPU acceleration feature via ``ov::intel_auto::enable_startup_fallback``, it will be -unable to support the initial model compilation stage. The models with dynamic -input/output or :doc:`stateful operations <../stateful-models>` +unable to support the initial model compilation stage. The models with :doc:`stateful operations <../stateful-models>` will be loaded to the CPU if it is in the candidate list. Otherwise, these models will follow the normal flow and be loaded to the device based on priority. @@ -510,8 +509,9 @@ For more information, refer to the :doc:`Benchmark Tool <../../../learn-openvino Additional Resources #################### -- :doc:`Debugging AUTO ` -- :doc:`Running on Multiple Devices Simultaneously ` -- :doc:`Inference Devices and Modes <../inference-devices-and-modes>` +* `Automatic Device Selection with OpenVINO™ Notebook `__ +* :doc:`Debugging AUTO ` +* :doc:`Running on Multiple Devices Simultaneously ` +* :doc:`Inference Devices and Modes <../inference-devices-and-modes>` diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst index 5abd4dbcec6742..dc158fe9352042 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst @@ -7,8 +7,8 @@ Performance Hints and Thread Scheduling on performance hints automatically. To simplify the configuration of hardware devices, it is recommended to use the -:doc:` ov::hint::PerformanceMode::LATENCY and ov::hint::PerformanceMode::THROUGHPUT <../../../../optimize-inference/high-level-performance-hints>` -high-level performance hints. Both performance hints ensure optimal portability +ov::hint::PerformanceMode::LATENCY and ov::hint::PerformanceMode::THROUGHPUT +:doc:`high-level performance hints <../../optimize-inference/high-level-performance-hints>`. Both performance hints ensure optimal portability and scalability of applications across various platforms and models. - ``ov::inference_num_threads`` limits the number of logical processors used for CPU inference. @@ -44,7 +44,8 @@ the model precision and the ratio of P-cores and E-cores. P-cores is short for Performance-cores and E-cores stands for Efficient-cores. These types of cores are available starting with the 12th Gen Intel® Core™ processors. -.. _Core Type Table of Latency Hint: +.. _core_type_latency: + +----------------------------+---------------------+---------------------+ | | INT8 Model | FP32 Model | +============================+=====================+=====================+ @@ -69,7 +70,7 @@ Then the default settings for low-level performance properties on Windows and Li +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ | ``ov::inference_num_threads`` | is equal to the number of P-cores or P-cores+E-cores on one socket | is equal to the number of P-cores or P-cores+E-cores on one socket | +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| ``ov::hint::scheduling_core_type`` | `Core Type Table of Latency Hint`_ | `Core Type Table of Latency Hint`_ | +| ``ov::hint::scheduling_core_type`` | :ref:`Core Type Table of Latency Hint ` | :ref:`Core Type Table of Latency Hint ` | +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ | ``ov::hint::enable_hyper_threading`` | No | No | +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst index 0e3506484b42ae..adad5dcf1e594d 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst @@ -476,6 +476,7 @@ Since OpenVINO relies on the OpenCL kernels for the GPU implementation, many gen Additional Resources ####################################### +* `Working with GPUs in OpenVINO™ Notebook `__ * :doc:`Inference Devices and Modes <../inference-devices-and-modes>`. * :doc:`Optimization guide <../optimize-inference>`. * `GPU plugin developer documentation `__ diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst index 0ba7d26436ee9e..24f820755555b0 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst @@ -22,7 +22,7 @@ Execution via the heterogeneous mode can be divided into two independent steps: These two steps are not interconnected and affinities can be set in one of two ways, used separately or in combination (as described below): in the ``manual`` or the ``automatic`` mode. Defining and Configuring the Hetero Device -++++++++++++++++++++++++++++++++++++++++++ +########################################## Following the OpenVINO™ naming convention, the Hetero execution plugin is assigned the label of ``"HETERO".`` It may be defined with no additional parameters, resulting in defaults being used, or configured further with the following setup options: @@ -37,10 +37,10 @@ Following the OpenVINO™ naming convention, the Hetero execution plugin is assi Manual and Automatic modes for assigning affinities -+++++++++++++++++++++++++++++++++++++++++++++++++++ +################################################### The Manual Mode --------------------- ++++++++++++++++++++++ It assumes setting affinities explicitly for all operations in the model using `ov::Node::get_rt_info `__ with the ``"affinity"`` key. @@ -66,7 +66,7 @@ Randomly selecting operations and setting affinities may lead to decrease in mod The Automatic Mode --------------------- ++++++++++++++++++++++ It decides automatically which operation is assigned to which device according to the support from dedicated devices (``GPU``, ``CPU``, etc.) and query model step is called implicitly by Hetero device during model compilation. @@ -92,7 +92,7 @@ It does not take into account device peculiarities such as the inability to infe Using Manual and Automatic Modes in Combination ------------------------------------------------ ++++++++++++++++++++++++++++++++++++++++++++++++ In some cases you may need to consider manually adjusting affinities which were set automatically. It usually serves minimizing the number of total subgraphs to optimize memory transfers. To do it, you need to "fix" the automatically assigned affinities like so: @@ -121,7 +121,7 @@ Importantly, the automatic mode will not work if any operation in a model has it `ov::Core::query_model `__ does not depend on affinities set by a user. Instead, it queries for an operation support based on device capabilities. Configure fallback devices -++++++++++++++++++++++++++ +########################## If you want different devices in Hetero execution to have different device-specific configuration options, you can use the special helper property `ov::device::properties `__: @@ -146,7 +146,7 @@ If you want different devices in Hetero execution to have different device-speci In the example above, the ``GPU`` device is configured to enable profiling data and uses the default execution precision, while ``CPU`` has the configuration property to perform inference in ``fp32``. Handling of Difficult Topologies -++++++++++++++++++++++++++++++++ +################################ Some topologies are not friendly to heterogeneous execution on some devices, even to the point of being unable to execute. For example, models having activation operations that are not supported on the primary device are split by Hetero into multiple sets of subgraphs which leads to suboptimal execution. @@ -154,7 +154,7 @@ If transmitting data from one subgraph to another part of the model in the heter In such cases, you can define the heaviest part manually and set the affinity to avoid sending data back and forth many times during one inference. Analyzing Performance of Heterogeneous Execution -++++++++++++++++++++++++++++++++++++++++++++++++ +################################################ After enabling the ``OPENVINO_HETERO_VISUALIZE`` environment variable, you can dump GraphViz ``.dot`` files with annotations of operations per devices. @@ -186,7 +186,7 @@ Here is an example of the output for Googlenet v1 running on HDDL (device no lon Sample Usage -++++++++++++++++++++ +##################### OpenVINO™ sample programs can use the Heterogeneous execution used with the ``-d`` option: diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst index ea39001b4f3fd4..5a21f2f7ccc0f8 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst @@ -20,8 +20,9 @@ NPU Plugin is now available through all relevant OpenVINO distribution channels. NPU Plugin needs an NPU Driver to be installed on the system for both compiling and executing a model. Follow the instructions below to install the latest NPU drivers: -* Windows driver: https://www.intel.com/content/www/us/en/download/794734/intel-npu-driver-windows.html -* Linux driver: https://github.com/intel/linux-npu-driver/releases + +* `Windows driver `__ +* `Linux driver `__ The plugin uses the graph extension API exposed by the driver to convert the OpenVINO specific representation @@ -151,7 +152,8 @@ guaranteed. Additional Resources ############################# -* `Vision colorization Notebook `__ +* `Working with NPUs in OpenVINO™ Notebook `__ +* `Vision colorization Notebook <./../../../notebooks/vision-image-colorization-with-output.html>`__ * `Classification Benchmark C++ Demo `__ * `3D Human Pose Estimation Python Demo `__ * `Object Detection C++ Demo `__ diff --git a/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst b/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst index ce5e6fd20722a1..222c8760d0a880 100644 --- a/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst +++ b/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst @@ -432,7 +432,8 @@ To build your project using CMake with the default build tools currently availab Additional Resources #################### -* See the :doc:`OpenVINO Samples <../../learn-openvino/openvino-samples>` page or the `Open Model Zoo Demos `__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. +* See the :doc:`OpenVINO Samples <../../learn-openvino/openvino-samples>` page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. +* Models in the OpenVINO IR format on `Hugging Face `__. * :doc:`OpenVINO™ Runtime Preprocessing ` * :doc:`String Tensors ` * :doc:`Using Encrypted Models with OpenVINO <../../documentation/openvino-security/openvino-encrypted-models>` diff --git a/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst b/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst index 438a2a5d9ae259..22223649144d9f 100644 --- a/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst +++ b/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst @@ -10,11 +10,11 @@ String Tensors OpenVINO tensors can hold not only numerical data, like floating-point or integer numbers, but also textual information, represented as one or multiple strings. Such a tensor is called a string tensor and can be passed as input or retrieved as output of a text-processing model, such as -`tokenizers and detokenizers `__. +`tokenizers and detokenizers `__. While this section describes basic API to handle string tensors, more practical examples that leverage both string tensors and OpenVINO tokenizer can be found in -`GenAI Samples `__. +`GenAI Samples `__. Representation diff --git a/docs/articles_en/openvino-workflow/torch-compile.rst b/docs/articles_en/openvino-workflow/torch-compile.rst index 280b7c01ca190e..759ca125d6dcd5 100644 --- a/docs/articles_en/openvino-workflow/torch-compile.rst +++ b/docs/articles_en/openvino-workflow/torch-compile.rst @@ -48,7 +48,7 @@ For OpenVINO installed via channels other than pip, such as conda, and versions .. image:: ../assets/images/torch_compile_backend_openvino.svg - :alt: torch.compile execution diagram + :alt: torch.compile execution diagram :width: 992px :height: 720px :scale: 60% @@ -196,7 +196,7 @@ The ``torch.compile`` feature is part of PyTorch 2.0, and is based on: (PEP 523) to dynamically modify Python bytecode right before it is executed (PyTorch operators that cannot be extracted to FX graph are executed in the native Python environment). It maintains the eager-mode capabilities using - `Guards `__ to ensure the + `Guards `__ to ensure the generated graphs are valid. * **AOTAutograd** - generates the backward graph corresponding to the forward graph captured by TorchDynamo. diff --git a/docs/dev/build_linux.md b/docs/dev/build_linux.md index 814deaa3deec3c..d3e1319f9d2c1a 100644 --- a/docs/dev/build_linux.md +++ b/docs/dev/build_linux.md @@ -12,13 +12,7 @@ The software was validated on: - [CMake](https://cmake.org/download/) 3.13 or higher - GCC 7.5 or higher to build OpenVINO Runtime - Python 3.8 - 3.11 for OpenVINO Runtime Python API -- (Optional) Install Intel® Graphics Compute Runtime for OpenCL™ Driver package to enable inference on Intel integrated GPUs. Select a driver package from the table below depending on what version of Ubuntu you are installing on. - - | Ubuntu | Driver package | - | --- | ----------- | - | 22.04 | [23.13.26032.30](https://github.com/intel/compute-runtime/releases/tag/23.13.26032.30) | - | 20.04 | [22.24.23453](https://github.com/intel/compute-runtime/releases/tag/22.24.23453) | - | 18.04 | [21.38.21026](https://github.com/intel/compute-runtime/releases/tag/21.38.21026) | +- (Optional) Install Intel® Graphics Compute Runtime for OpenCL™ Driver package to enable inference on Intel integrated GPUs. ## How to build @@ -45,6 +39,8 @@ The software was validated on: mkdir build && cd build ``` +> **NOTE**: It is recommended to disable the oneAPI environment before compiling OpenVINO from source on Linux, as it may cause build failures. + 4. OpenVINO Runtime uses a CMake-based build system. In the created `build` directory, run `cmake` to fetch project dependencies and create Unix makefiles, then run `make` to build the project: ```sh cmake -DCMAKE_BUILD_TYPE=Release .. diff --git a/docs/dev/build_windows.md b/docs/dev/build_windows.md index eaf695f898ba18..2515000e5ec4f0 100644 --- a/docs/dev/build_windows.md +++ b/docs/dev/build_windows.md @@ -6,20 +6,20 @@ Supported configurations: - Windows 10 x86 64-bit or higher with Visual Studio 2019 or higher build for X64 architecture. - Windows on ARM (shortly WoA) to build for ARM64 architecture. OpenVINO was validated on [Windows DevKit 2023](https://developer.qualcomm.com/hardware/windows-on-snapdragon/windows-dev-kit-2023) -## Software requirements +## Software requirements - [CMake](https://cmake.org/download/) 3.13 or higher - Microsoft Visual Studio 2019 or higher, version 16.3 or later - > **NOTE**: Native Microsoft Visual Studio for WoA is available since 2022. + > **NOTE**: Native Microsoft Visual Studio for WoA has been available since version 3.11. - Python 3.8 - 3.11 for OpenVINO Runtime Python API - > **NOTE**: Python for ARM64 is available since [3.11](https://www.python.org/downloads/windows/) version. + > **NOTE**: Python for ARM64 is available since [3.11](https://www.python.org/downloads/windows/) version. - [Git for Windows*] - (Windows on ARM only) [LLVM for Windows on ARM (WoA)](https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/LLVM-15.0.6-woa64.exe) - > **NOTE**: After installation, make sure `clang-cl` compiler is available from `PATH`. + > **NOTE**: After installation, make sure `clang-cl` compiler is available from `PATH`. ## How to build -> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to download and install the Intel® Graphics Driver for Windows (26.20) [driver package](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Driver. +> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to download and install the [Intel® Graphics Driver for Windows](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Driver. 1. Clone submodules: ```sh @@ -37,12 +37,12 @@ Supported configurations: ```sh cmake -G "Visual Studio 17 2022" ``` - + > **HINT**: **Generating PDB Files and Debugging Your Build**
> If you intend to generate PDB files and debug your build, it is essential to set the CMake build type appropriately. > You should utilize one of the following CMake build type options:
>* `-DCMAKE_BUILD_TYPE=RelWithDebInfo`: This option generates PDB files with release information, making it suitable for debugging optimized builds.
- >* `-DCMAKE_BUILD_TYPE=Debug`: This option generates PDB files optimized for debugging, providing comprehensive debugging information. + >* `-DCMAKE_BUILD_TYPE=Debug`: This option generates PDB files optimized for debugging, providing comprehensive debugging information. 4. Build generated solution in Visual Studio or run `cmake --build . --config Release --verbose -j` to build from the command line. View the number of available processing units with `WMIC cpu get numberofLogicalProcessors`. Be aware that this process may take some time. @@ -62,7 +62,7 @@ Supported configurations: 3. After the build process finishes, export the newly built Python libraries to the user environment variables: ``` set PYTHONPATH=/bin//Release/python;%PYTHONPATH% - set OPENVINO_LIB_PATHS=/bin//Release;%OPENVINO_LIB_PATH% + set OPENVINO_LIB_PATHS=/bin//Release;/temp/tbb/bin ``` or install the wheel with pip: ``` diff --git a/docs/dev/cmake_options_for_custom_compilation.md b/docs/dev/cmake_options_for_custom_compilation.md index a5184dbc460979..d370285f99c8b4 100644 --- a/docs/dev/cmake_options_for_custom_compilation.md +++ b/docs/dev/cmake_options_for_custom_compilation.md @@ -42,6 +42,8 @@ This document provides description and default values for CMake options that can * `ON` is default. * `ENABLE_OV_PYTORCH_FRONTEND` enables [PyTorch] frontend plugin for OpenVINO Runtime: * `ON` is default. + * `ENABLE_OV_JAX_FRONTEND` enables [JAX] frontend plugin for OpenVINO Runtime: + * `ON` is default. * `ENABLE_OV_IR_FRONTEND` enables OpenVINO Intermediate Representation frontend plugin for OpenVINO Runtime: * `ON` is default. * `OPENVINO_EXTRA_MODULES` specifies path to add extra OpenVINO modules to the build. diff --git a/docs/dev/ov_dependencies_24_2.txt b/docs/dev/ov_dependencies_24_2.txt new file mode 100644 index 00000000000000..0f24303ec47d0c --- /dev/null +++ b/docs/dev/ov_dependencies_24_2.txt @@ -0,0 +1,374 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +#This file provides a comprehensive list of all dependencies of OpenVINO 2024.2 +#The file is part of the automation pipeline for posting OpenVINO IR models on the HuggingFace Hub, including OneBOM dependency checks. + + +absl-py +accuracy-checker +addict +alabaster +appnope +argparse +astor +astpretty +astroid +astunparse +atomicwrites +attrs +babel +backcall +backports-functools-lru-cache +bandit +beautifulsoup4 +black +Boost +Bootstrap +BrainMaGe +cached-property +cachetools +certifi +chardet +charset-normalizer +chosen-js +clang +click +CNPY +colorama +ComputeLibrary +coverage +cycler +cython +dataclasses +decorator +Deformable-ConvNets +defusedxml +Detectron2 +distlib +distro +docopt +docutils +Doxygen +Vovkos doxyrest +enum34 +eradicate +execnet +fastjsonschema +Ffmpeg +filelock +flake8 +flake8_builtins +flake8_coding +flake8_commas +flake8_pep3101 +flake8_quotes +flake8-annotations-complexity +flake8-broken-line +flake8-bugbear +flake8-class-attributes-order +flake8-comprehensions +flake8-debugger +flake8-docstrings +flake8-eradicate +flake8-executable +flake8-expression-complexity +flake8-plugin-utils +flake8-print +flake8-pytest-style +flake8-rst-docstrings +flake8-string-format +flake8-variables-names +flatbuffers +fonttools +freetype +funcsigs +functools32 +future +futures +gast +generator +gflags +gitdb +gitpython +google-auth +google-auth-oauthlib +google cpplint +google fonts +google-java-format +google-pasta +Google gtest-parallel +googletest +graphviz +grpcio +Gstreamer +h5py +hamcrest-core +Hyperopt +idna +idna +imagesize +importlib-metadata +importlib-resources +import-order +iniconfig +INTEL/CAFFE +Intel Integrated Performance Primitives - Intel IPP +intel/IntelSEAPI +intel/libva +ipython +isort +itt API +jedi +jinja2 +json +json-c +jsonschema +pboettch json-schema-validator +linjackson78 jstyleson +John Hauser Berkeley SoftFloat +jquery - jquery/jquery +Junit +keras-nightly +keras-preprocessing +kiwisolver +lazy-object-proxy +libjpeg-turbo +Libpng +Llohse libnpy +llvm +lxml +mando +facebookresearch/maskrcnn-benchmark +markdown +markupsafe +mathjax mathjax +matplotlib +mccabe +mistune +mongo-python-driver +more-itertools +mxnet +mypy +mypy-extensions +networkx +nibabel +Nithinn ncc +numpy +oauthlib +OneDNN +OneTBB +onnx +onnxruntime +Open Model Zoo +omz-tools +OpenCL +OpenCL headers +OpenCL ICD Loader +OpenCL-hpp headers +OpenCV +opencv-python +OpenJDK +OpenVINO OSS +openvino-telemetry +opt-einsum +packaging +paddlepaddle +PapaParse +pandas +parso +pathlib2 +pathspec +pbr +pep8-naming +pexpect +pickleshare +pillow +platformdirs +pluggy +pluggy +progress +prompt-toolkit +protobuf +ptyprocess +pugixml +py +pyasn1 +pyasn1-modules +pycodestyle +py-cpuinfo +py-leveldb +Pybind11 +pydata-sphinx-theme +pydocstyle +pyenchant +pyflakes +pygments +pylint +pymongo +pyparsing +pyrsistent +pytest +pytest-dependency +pytest-forked +pytest-html +pytest-metadata +pytest-metadata +pytest-timeout +pytest-xdist +Pytorch +Python +python-attrs +python-dateutil +python-fastjsonschema +python pandas +Python-pillow +pytz +pyyaml +radon +Rapidjson +requests +requests-oauthlib +restructuredtext-lint +retrying +Roboto Fonts +rsa +ruGPT3Medium +safestringlib +scikit-build +scipy +setuptools +six +smmap +snowballstemmer +soupsieve +sphinx +sphinxcontrib-applehelp +sphinxcontrib-devhelp +sphinxcontrib-htmlhelp +sphinxcontrib-jsmath +sphinxcontrib-qthelp +sphinxcontrib-serializinghtml +sphinx-copybutton +sphinx-inline-tabs +sphinx-panels +sphinx-sitemap +stevedore +tensorboard +tensorboard-data-server +tensorboard-plugin-wit +tensorflow +tensorflow +tensorflow-estimator +termcolor +test-generator +threading_building_blocks +foutaise/texttable/ +toml +tomli +Torch7 +tox +traitlets +Transformers +typed-ast +types-pkg_resources +typing +typing-extensions +urllib3 +Viewer +virtualenv +Torchvision +wcwidth +werkzeug +wheel +wrapt +xbyak +zipp +Zlib +backports +contextlib2 +dill +keras +singledispatch +Fluid +libusb (for MYRIAD) +libGNA (for GNA) +Benchmark +GitPython +huggingface/transformers +Hamcrest JavaHamcrest +Openvinotoolkit model-server +Openvinotoolkit nncf +pytorch/vision +Sphinx documentation builder +FlatBuffers +POCO C++ Libraries +Google Mock +Google Snappy +Microsoft onnxruntime +ML commons/inference +take-action +bitsandbytes +diffusers +einops +optimum-intel +tiktoken +timm +tokenizers +torch +optimum +psutil +vscode-extension-samples +fast_tokenizer +re2 +icu4c +sentencepiece +auto-gptq +transformers-stream-generator +eigen3 +gtk3-devel +gstreamer1-devel +gstreamer1-plugins-base-devel +ffmpeg-devel +libmfx-devel +cmake +git +python38-devel +python38-pip +gcc-c++ +gcc +python3-dev +python3-pip +ninja-build +libgtk-3-dev +libpng-dev +libjpeg-dev +libwebp-dev +libtiff5-dev +libopenexr-dev +libopenblas-dev +libx11-dev +libavutil-dev +libavcodec-dev +libavformat-dev +libswscale-dev +libavresample-dev +libtbb2 +libssl-dev +libva-dev +libmfx-dev +libgstreamer1.0-dev +libgstreamer-plugins-base1.0-dev +epel-release-latest-8.noarch.rpm +rpmfusion-free-release-8.noarch.rpm +pytest-repeat 0.9.3 +pip +make +httpx +ultralytics +llama.cpp +lixsmm + + + + + diff --git a/docs/dev/static_libaries.md b/docs/dev/static_libaries.md index 3570b4532d9157..0f694cda73f07d 100644 --- a/docs/dev/static_libaries.md +++ b/docs/dev/static_libaries.md @@ -47,6 +47,7 @@ cmake -DENABLE_INTEL_GPU=OFF \ -DENABLE_OV_TF_FRONTEND=OFF \ -DENABLE_OV_TF_LITE_FRONTEND=OFF \ -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -DENABLE_OV_JAX_FRONTEND=OFF \ -DENABLE_INTEL_CPU=ON \ -DENABLE_OV_IR_FRONTEND=ON ``` diff --git a/docs/nbdoc/consts.py b/docs/nbdoc/consts.py index ded7c9637e0999..83c035c7766160 100644 --- a/docs/nbdoc/consts.py +++ b/docs/nbdoc/consts.py @@ -6,7 +6,7 @@ repo_owner = "openvinotoolkit" repo_name = "openvino_notebooks" repo_branch = "tree/main" -artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20240605220807/dist/rst_files/" +artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20240701220806/dist/rst_files/" blacklisted_extensions = ['.xml', '.bin'] notebooks_repo = "https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/" notebooks_binder = "https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=" diff --git a/docs/notebooks/3D-pose-estimation-with-output.rst b/docs/notebooks/3D-pose-estimation-with-output.rst index 8d52a5d3589f29..ba06f88f2bd864 100644 --- a/docs/notebooks/3D-pose-estimation-with-output.rst +++ b/docs/notebooks/3D-pose-estimation-with-output.rst @@ -12,13 +12,13 @@ out the algorithms. **Make sure you have properly installed the**\ `Jupyter extension `__\ **and been using JupyterLab to run the demo as suggested in the -``README.md``** +"README.md"** **NOTE**: *To use a webcam, you must run this Jupyter notebook on a computer with a webcam. If you run on a remote server, the webcam will not work. However, you can still do inference on a video file in the final step. This demo utilizes the Python interface in - ``Three.js`` integrated with WebGL to process data from the model + "Three.js" integrated with WebGL to process data from the model inference. These results are processed and displayed in the notebook.* @@ -26,8 +26,8 @@ been using JupyterLab to run the demo as suggested in the recommended browser on one of the following operating systems:* *Ubuntu, Windows: Chrome* *macOS: Safari* -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Imports <#imports>`__ @@ -52,7 +52,7 @@ Prerequisites -**The ``pythreejs`` extension may not display properly when using a +**The "pythreejs" extension may not display properly when using a Jupyter Notebook release. Therefore, it is recommended to use Jupyter Lab instead.** @@ -67,84 +67,84 @@ Lab instead.** Collecting pythreejs Using cached pythreejs-2.4.2-py3-none-any.whl.metadata (5.4 kB) Collecting openvino-dev>=2024.0.0 - Using cached openvino_dev-2024.1.0-15008-py3-none-any.whl.metadata (16 kB) + Using cached openvino_dev-2024.2.0-15519-py3-none-any.whl.metadata (16 kB) Collecting opencv-python - Using cached opencv_python-4.10.0.82-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB) + Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB) Collecting torch Using cached https://download.pytorch.org/whl/cpu/torch-2.3.1%2Bcpu-cp38-cp38-linux_x86_64.whl (190.4 MB) Collecting onnx Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB) - Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.3) + Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.3) Collecting ipydatawidgets>=1.1.1 (from pythreejs) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl.metadata (1.4 kB) Collecting numpy (from pythreejs) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB) - Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) - Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (0.7.1) + Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) + Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (0.7.1) Collecting networkx<=3.1.0 (from openvino-dev>=2024.0.0) Using cached networkx-3.1-py3-none-any.whl.metadata (5.3 kB) Collecting openvino-telemetry>=2023.2.1 (from openvino-dev>=2024.0.0) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl.metadata (2.3 kB) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (24.0) - Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (6.0.1) - Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (2.32.0) - Collecting openvino==2024.1.0 (from openvino-dev>=2024.0.0) - Using cached openvino-2024.1.0-15008-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.8 kB) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (24.1) + Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (6.0.1) + Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (2.32.0) + Collecting openvino==2024.2.0 (from openvino-dev>=2024.0.0) + Using cached openvino-2024.2.0-15519-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.9 kB) Collecting filelock (from torch) - Using cached filelock-3.14.0-py3-none-any.whl.metadata (2.8 kB) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.1) + Using cached filelock-3.15.4-py3-none-any.whl.metadata (2.9 kB) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) Collecting sympy (from torch) Using cached sympy-1.12.1-py3-none-any.whl.metadata (12 kB) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) Collecting fsspec (from torch) - Using cached fsspec-2024.6.0-py3-none-any.whl.metadata (11 kB) + Using cached fsspec-2024.6.1-py3-none-any.whl.metadata (11 kB) Collecting protobuf>=3.20.2 (from onnx) - Using cached protobuf-5.27.0-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) + Using cached protobuf-5.27.2-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) Collecting traittypes>=0.2.0 (from ipydatawidgets>=1.1.1->pythreejs) Using cached traittypes-0.2.1-py2.py3-none-any.whl.metadata (1.0 kB) - Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) - Requirement already satisfied: widgetsnbextension~=4.0.11 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.11) - Requirement already satisfied: jupyterlab-widgets~=3.0.11 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.11) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.3.2) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.7) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2.2.1) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2024.6.2) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) + Requirement already satisfied: widgetsnbextension~=4.0.11 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.11) + Requirement already satisfied: jupyterlab-widgets~=3.0.11 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.11) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.3.2) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.7) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2.2.2) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2024.6.2) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) Collecting mpmath<1.4.0,>=1.1.0 (from sympy->torch) Using cached https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB) - Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) - Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) - Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) - Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) - Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) - Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.46) - Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) - Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) - Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) - Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) - Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) - Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) - Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.0.1) - Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) - Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) + Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) + Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) + Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) + Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) + Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) + Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.47) + Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) + Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) + Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) + Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) + Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) + Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) + Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.0.1) + Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) + Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) Using cached pythreejs-2.4.2-py3-none-any.whl (3.4 MB) - Using cached openvino_dev-2024.1.0-15008-py3-none-any.whl (4.7 MB) - Using cached openvino-2024.1.0-15008-cp38-cp38-manylinux2014_x86_64.whl (38.7 MB) - Using cached opencv_python-4.10.0.82-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.5 MB) + Using cached openvino_dev-2024.2.0-15519-py3-none-any.whl (4.7 MB) + Using cached openvino-2024.2.0-15519-cp38-cp38-manylinux2014_x86_64.whl (38.7 MB) + Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.5 MB) Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl (271 kB) Using cached networkx-3.1-py3-none-any.whl (2.1 MB) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl (23 kB) - Using cached protobuf-5.27.0-cp38-abi3-manylinux2014_x86_64.whl (309 kB) - Using cached filelock-3.14.0-py3-none-any.whl (12 kB) - Using cached fsspec-2024.6.0-py3-none-any.whl (176 kB) + Using cached protobuf-5.27.2-cp38-abi3-manylinux2014_x86_64.whl (309 kB) + Using cached filelock-3.15.4-py3-none-any.whl (16 kB) + Using cached fsspec-2024.6.1-py3-none-any.whl (177 kB) Using cached sympy-1.12.1-py3-none-any.whl (5.7 MB) Using cached traittypes-0.2.1-py2.py3-none-any.whl (8.6 kB) Installing collected packages: openvino-telemetry, mpmath, traittypes, sympy, protobuf, numpy, networkx, fsspec, filelock, torch, openvino, opencv-python, onnx, openvino-dev, ipydatawidgets, pythreejs - Successfully installed filelock-3.14.0 fsspec-2024.6.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 onnx-1.16.1 opencv-python-4.10.0.82 openvino-2024.1.0 openvino-dev-2024.1.0 openvino-telemetry-2024.1.0 protobuf-5.27.0 pythreejs-2.4.2 sympy-1.12.1 torch-2.3.1+cpu traittypes-0.2.1 + Successfully installed filelock-3.15.4 fsspec-2024.6.1 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 onnx-1.16.1 opencv-python-4.10.0.84 openvino-2024.2.0 openvino-dev-2024.2.0 openvino-telemetry-2024.1.0 protobuf-5.27.2 pythreejs-2.4.2 sympy-1.12.1 torch-2.3.1+cpu traittypes-0.2.1 Note: you may need to restart the kernel to use updated packages. @@ -158,28 +158,28 @@ Imports import collections import time from pathlib import Path - + import cv2 import ipywidgets as widgets import numpy as np from IPython.display import clear_output, display import openvino as ov - + # Fetch `notebook_utils` module import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) with open("notebook_utils.py", "w") as f: f.write(r.text) - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/engine3js.py", ) with open("engine3js.py", "w") as f: f.write(r.text) - + import notebook_utils as utils import engine3js as engine @@ -201,19 +201,19 @@ directory structure and downloads the selected model. # directory where model will be downloaded base_model_dir = "model" - + # model name as named in Open Model Zoo model_name = "human-pose-estimation-3d-0001" # selected precision (FP32, FP16) precision = "FP32" - + BASE_MODEL_NAME = f"{base_model_dir}/public/{model_name}/{model_name}" model_path = Path(BASE_MODEL_NAME).with_suffix(".pth") onnx_path = Path(BASE_MODEL_NAME).with_suffix(".onnx") - + ir_model_path = f"model/public/{model_name}/{precision}/{model_name}.xml" model_weights_path = f"model/public/{model_name}/{precision}/{model_name}.bin" - + if not model_path.exists(): download_command = f"omz_downloader " f"--name {model_name} " f"--output_dir {base_model_dir}" ! $download_command @@ -222,12 +222,12 @@ directory structure and downloads the selected model. .. parsed-literal:: ################|| Downloading human-pose-estimation-3d-0001 ||################ - + ========== Downloading model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - - + + ========== Unpacking model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - + Convert Model to OpenVINO IR format @@ -252,19 +252,20 @@ IR format. .. parsed-literal:: ========== Converting human-pose-estimation-3d-0001 to ONNX - Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/internal_scripts/pytorch_to_onnx.py --model-path=model/public/human-pose-estimation-3d-0001 --model-name=PoseEstimationWithMobileNet --model-param=is_convertible_by_mo=True --import-module=model --weights=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.pth --input-shape=1,3,256,448 --input-names=data --output-names=features,heatmaps,pafs --output-file=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx - + Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/internal_scripts/pytorch_to_onnx.py --model-path=model/public/human-pose-estimation-3d-0001 --model-name=PoseEstimationWithMobileNet --model-param=is_convertible_by_mo=True --import-module=model --weights=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.pth --input-shape=1,3,256,448 --input-names=data --output-names=features,heatmaps,pafs --output-file=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx + ONNX check passed successfully. - + ========== Converting human-pose-estimation-3d-0001 to IR (FP32) - Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/human-pose-estimation-3d-0001/FP32 --model_name=human-pose-estimation-3d-0001 --input=data '--mean_values=data[128.0,128.0,128.0]' '--scale_values=data[255.0,255.0,255.0]' --output=features,heatmaps,pafs --input_model=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 256, 448]' --compress_to_fp16=False - - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/human-pose-estimation-3d-0001/FP32 --model_name=human-pose-estimation-3d-0001 --input=data '--mean_values=data[128.0,128.0,128.0]' '--scale_values=data[255.0,255.0,255.0]' --output=features,heatmaps,pafs --input_model=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 256, 448]' --compress_to_fp16=False + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html [ SUCCESS ] Generated IR version 11 model. - [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.xml - [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.bin - + [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.xml + [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.bin + Select inference device @@ -277,14 +278,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 core = ov.Core() - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value="AUTO", description="Device:", disabled=False, ) - + device @@ -319,7 +320,7 @@ created to infer the compiled model. compiled_model = core.compile_model(model=model, device_name=device.value) infer_request = compiled_model.create_infer_request() input_tensor_name = model.inputs[0].get_any_name() - + # get input and output names of nodes input_layer = compiled_model.input(0) output_layers = list(compiled_model.outputs) @@ -359,25 +360,25 @@ input for the 3D model. This is how you obtain the output heat maps, PAF def model_infer(scaled_img, stride): """ Run model inference on the input image - + Parameters: scaled_img: resized image according to the input size of the model stride: int, the stride of the window """ - + # Remove excess space from the picture img = scaled_img[ 0 : scaled_img.shape[0] - (scaled_img.shape[0] % stride), 0 : scaled_img.shape[1] - (scaled_img.shape[1] % stride), ] - + img = np.transpose(img, (2, 0, 1))[None,] infer_request.infer({input_tensor_name: img}) # A set of three inference results is obtained results = {name: infer_request.get_tensor(name).data[:] for name in {"features", "heatmaps", "pafs"}} # Get the results results = (results["features"][0], results["heatmaps"][0], results["pafs"][0]) - + return results Draw 2D Pose Overlays @@ -417,8 +418,8 @@ from Open Model Zoo. [13, 14], # neck - r_hip - r_knee - r_ankle ] ) - - + + body_edges_2d = np.array( [ [0, 1], # neck - nose @@ -440,25 +441,25 @@ from Open Model Zoo. [13, 14], # neck - r_hip - r_knee - r_ankle ] ) - - + + def draw_poses(frame, poses_2d, scaled_img, use_popup): """ Draw 2D pose overlays on the image to visualize estimated poses. Joints are drawn as circles and limbs are drawn as lines. - + :param frame: the input image :param poses_2d: array of human joint pairs """ for pose in poses_2d: pose = np.array(pose[0:-1]).reshape((-1, 3)).transpose() was_found = pose[2] > 0 - + pose[0], pose[1] = ( pose[0] * frame.shape[1] / scaled_img.shape[1], pose[1] * frame.shape[0] / scaled_img.shape[0], ) - + # Draw joints. for edge in body_edges_2d: if was_found[edge[0]] and was_found[edge[1]]: @@ -481,7 +482,7 @@ from Open Model Zoo. -1, cv2.LINE_AA, ) - + return frame Main Processing Function @@ -498,18 +499,18 @@ webcam feed or a video file. """ 2D image as input, using OpenVINO as inference backend, get joints 3D coordinates, and draw 3D human skeleton in the scene - + :param source: The webcam number to feed the video stream with primary webcam set to "0", or the video path. :param flip: To be used by VideoPlayer function for flipping capture image. :param use_popup: False for showing encoded frames over this notebook, True for creating a popup window. :param skip_frames: Number of frames to skip at the beginning of the video. """ - + focal_length = -1 # default stride = 8 player = None skeleton_set = None - + try: # create video player to play with target fps video_path # get the frame from camera @@ -517,16 +518,16 @@ webcam feed or a video file. player = utils.VideoPlayer(source, flip=flip, fps=30, skip_first_frames=skip_frames) # start capturing player.start() - + input_image = player.next() # set the window size resize_scale = 450 / input_image.shape[1] windows_width = int(input_image.shape[1] * resize_scale) windows_height = int(input_image.shape[0] * resize_scale) - + # use visualization library engine3D = engine.Engine3js(grid=True, axis=True, view_width=windows_width, view_height=windows_height) - + if use_popup: # display the 3D human pose in this notebook, and origin frame in popup window display(engine3D.renderer) @@ -536,43 +537,43 @@ webcam feed or a video file. # set the 2D image box, show both human pose and image in the notebook imgbox = widgets.Image(format="jpg", height=windows_height, width=windows_width) display(widgets.HBox([engine3D.renderer, imgbox])) - + skeleton = engine.Skeleton(body_edges=body_edges) - + processing_times = collections.deque() - + while True: # grab the frame frame = player.next() if frame is None: print("Source ended") break - + # resize image and change dims to fit neural network input # (see https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/human-pose-estimation-3d-0001) scaled_img = cv2.resize(frame, dsize=(model.inputs[0].shape[3], model.inputs[0].shape[2])) - + if focal_length < 0: # Focal length is unknown focal_length = np.float32(0.8 * scaled_img.shape[1]) - + # inference start start_time = time.time() # get results inference_result = model_infer(scaled_img, stride) - + # inference stop stop_time = time.time() processing_times.append(stop_time - start_time) # Process the point to point coordinates of the data poses_3d, poses_2d = engine.parse_poses(inference_result, 1, stride, focal_length, True) - + # use processing times from last 200 frames if len(processing_times) > 200: processing_times.popleft() - + processing_time = np.mean(processing_times) * 1000 fps = 1000 / processing_time - + if len(poses_3d) > 0: # From here, you can rotate the 3D point positions using the function "draw_poses", # or you can directly make the correct mapping below to properly display the object image on the screen @@ -585,28 +586,28 @@ webcam feed or a video file. -y + np.ones(poses_3d[:, 2::4].shape) * 100, -x, ) - + poses_3d = poses_3d.reshape(poses_3d.shape[0], 19, -1)[:, :, 0:3] people = skeleton(poses_3d=poses_3d) - + try: engine3D.scene_remove(skeleton_set) except Exception: pass - + engine3D.scene_add(people) skeleton_set = people - + # draw 2D frame = draw_poses(frame, poses_2d, scaled_img, use_popup) - + else: try: engine3D.scene_remove(skeleton_set) skeleton_set = None except Exception: pass - + cv2.putText( frame, f"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)", @@ -617,7 +618,7 @@ webcam feed or a video file. 1, cv2.LINE_AA, ) - + if use_popup: cv2.imshow(title, frame) key = cv2.waitKey(1) @@ -631,9 +632,9 @@ webcam feed or a video file. frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90], )[1].tobytes() - + engine3D.renderer.render(engine3D.scene, engine3D.cam) - + except KeyboardInterrupt: print("Interrupted") except RuntimeError as e: @@ -680,10 +681,10 @@ picture on the left to interact. .. code:: ipython3 USE_WEBCAM = False - + cam_id = 0 video_path = "https://github.com/intel-iot-devkit/sample-videos/raw/master/face-demographics-walking.mp4" - + source = cam_id if USE_WEBCAM else video_path - + run_pose_estimation(source=source, flip=isinstance(source, int), use_popup=False) diff --git a/docs/notebooks/3D-segmentation-point-clouds-with-output.rst b/docs/notebooks/3D-segmentation-point-clouds-with-output.rst index 8e4954bc00a95f..c73718778a29a6 100644 --- a/docs/notebooks/3D-segmentation-point-clouds-with-output.rst +++ b/docs/notebooks/3D-segmentation-point-clouds-with-output.rst @@ -22,8 +22,8 @@ segmentation, to scene semantic parsing. It is highly efficient and effective, showing strong performance on par or even better than state of the art. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Imports <#imports>`__ - `Prepare the Model <#prepare-the-model>`__ @@ -216,7 +216,7 @@ chair for example. .. parsed-literal:: - /tmp/ipykernel_3063563/2434168836.py:12: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored + /tmp/ipykernel_2758943/2434168836.py:12: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored ax.scatter3D(X, Y, Z, s=5, cmap="jet", marker="o", label="chair") @@ -317,7 +317,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - /tmp/ipykernel_3063563/2804603389.py:23: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored + /tmp/ipykernel_2758943/2804603389.py:23: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored ax.scatter(XCur, YCur, ZCur, s=5, cmap="jet", marker="o", label=classes[i]) diff --git a/docs/notebooks/action-recognition-webcam-with-output.rst b/docs/notebooks/action-recognition-webcam-with-output.rst index 3b3455ba9b487a..38cbcff848b171 100644 --- a/docs/notebooks/action-recognition-webcam-with-output.rst +++ b/docs/notebooks/action-recognition-webcam-with-output.rst @@ -35,8 +35,8 @@ Transformer and `ResNet34 `__. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Imports <#imports>`__ - `The models <#the-models>`__ diff --git a/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png b/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png index acae41e0b1ada0..e398fc093af7c9 100644 --- a/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png +++ b/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f9519b2a12072147ebf54e1d7a840ccde81b965fa1844f42f79e66c6513d844a -size 68147 +oid sha256:339e8a574ee9bb1160f222834ed747e724e89bf7148ebd01b15ab89c702cf16e +size 68902 diff --git a/docs/notebooks/all_notebooks_paths.txt b/docs/notebooks/all_notebooks_paths.txt index 481450364a1eca..636415ae99c7be 100644 --- a/docs/notebooks/all_notebooks_paths.txt +++ b/docs/notebooks/all_notebooks_paths.txt @@ -12,12 +12,12 @@ notebooks/clip-language-saliency-map/clip-language-saliency-map.ipynb notebooks/clip-zero-shot-image-classification/clip-zero-shot-classification.ipynb notebooks/controlnet-stable-diffusion/controlnet-stable-diffusion.ipynb notebooks/convert-to-openvino/convert-to-openvino.ipynb -notebooks/convert-to-openvino/legacy-mo-convert-to-openvino.ipynb notebooks/cross-lingual-books-alignment/cross-lingual-books-alignment.ipynb notebooks/ct-segmentation-quantize/ct-segmentation-quantize-nncf.ipynb notebooks/ddcolor-image-colorization/ddcolor-image-colorization.ipynb notebooks/decidiffusion-image-generation/decidiffusion-image-generation.ipynb notebooks/depth-anything/depth-anything.ipynb +notebooks/depth-anything/depth-anything-v2.ipynb notebooks/detectron2-to-openvino/detectron2-to-openvino.ipynb notebooks/distilbert-sequence-classification/distilbert-sequence-classification.ipynb notebooks/distil-whisper-asr/distil-whisper-asr.ipynb @@ -37,9 +37,9 @@ notebooks/hello-npu/hello-npu.ipynb notebooks/hello-segmentation/hello-segmentation.ipynb notebooks/hello-world/hello-world.ipynb notebooks/hugging-face-hub/hugging-face-hub.ipynb +notebooks/hunyuan-dit-image-generation/hunyuan-dit-image-generation.ipynb notebooks/image-bind/image-bind.ipynb notebooks/image-classification-quantization/image-classification-quantization.ipynb -notebooks/image-inpainting/image-inpainting.ipynb notebooks/instant-id/instant-id.ipynb notebooks/instruct-pix2pix-image-editing/instruct-pix2pix-image-editing.ipynb notebooks/knowledge-graphs-conve/knowledge-graphs-conve.ipynb @@ -51,17 +51,18 @@ notebooks/latent-consistency-models-image-generation/lcm-lora-controlnet.ipynb notebooks/llava-multimodal-chatbot/llava-multimodal-chatbot.ipynb notebooks/llava-multimodal-chatbot/videollava-multimodal-chatbot.ipynb notebooks/llava-next-multimodal-chatbot/llava-next-multimodal-chatbot.ipynb -notebooks/llm-agent-langchain/llm-agent-langchain.ipynb +notebooks/llm-agent-react/llm-agent-react-langchain.ipynb +notebooks/llm-chatbot/llm-chatbot-generate-api.ipynb notebooks/llm-chatbot/llm-chatbot.ipynb notebooks/llm-question-answering/llm-question-answering.ipynb notebooks/llm-rag-langchain/llm-rag-langchain.ipynb +notebooks/llm-rag-llamaindex/llm-rag-llamaindex.ipynb notebooks/magika-content-type-recognition/magika-content-type-recognition.ipynb notebooks/meter-reader/meter-reader.ipynb notebooks/mms-massively-multilingual-speech/mms-massively-multilingual-speech.ipynb notebooks/mobileclip-video-search/mobileclip-video-search.ipynb notebooks/mobilevlm-language-assistant/mobilevlm-language-assistant.ipynb notebooks/model-server/model-server.ipynb -notebooks/model-tools/model-tools.ipynb notebooks/music-generation/music-generation.ipynb notebooks/named-entity-recognition/named-entity-recognition.ipynb notebooks/nano-llava-multimodal-chatbot/nano-llava-multimodal-chatbot.ipynb @@ -75,12 +76,13 @@ notebooks/optimize-preprocessing/optimize-preprocessing.ipynb notebooks/paddle-ocr-webcam/paddle-ocr-webcam.ipynb notebooks/paddle-to-openvino/paddle-to-openvino-classification.ipynb notebooks/paint-by-example/paint-by-example.ipynb +notebooks/parler-tts-text-to-speech/parler-tts-text-to-speech.ipynb notebooks/person-counting-webcam/person-counting.ipynb notebooks/person-tracking-webcam/person-tracking.ipynb notebooks/photo-maker/photo-maker.ipynb notebooks/pix2struct-docvqa/pix2struct-docvqa.ipynb +notebooks/pixart/pixart.ipynb notebooks/pose-estimation-webcam/pose-estimation.ipynb -notebooks/pyannote-speaker-diarization/pyannote-speaker-diarization.ipynb notebooks/pytorch-post-training-quantization-nncf/pytorch-post-training-quantization-nncf.ipynb notebooks/pytorch-quantization-aware-training/pytorch-quantization-aware-training.ipynb notebooks/pytorch-quantization-sparsity-aware-training/pytorch-quantization-sparsity-aware-training.ipynb @@ -100,9 +102,8 @@ notebooks/softvc-voice-conversion/softvc-voice-conversion.ipynb notebooks/sound-generation-audioldm2/sound-generation-audioldm2.ipynb notebooks/sparsity-optimization/sparsity-optimization.ipynb notebooks/speculative-sampling/speculative-sampling.ipynb -notebooks/speech-recognition-quantization/speech-recognition-quantization-data2vec.ipynb +notebooks/speechbrain-emotion-recognition/speechbrain-emotion-recognition.ipynb notebooks/speech-recognition-quantization/speech-recognition-quantization-wav2vec2.ipynb -notebooks/speech-to-text/speech-to-text.ipynb notebooks/stable-cascade-image-generation/stable-cascade-image-generation.ipynb notebooks/stable-diffusion-ip-adapter/stable-diffusion-ip-adapter.ipynb notebooks/stable-diffusion-keras-cv/stable-diffusion-keras-cv.ipynb @@ -112,6 +113,7 @@ notebooks/stable-diffusion-v2/stable-diffusion-v2-infinite-zoom.ipynb notebooks/stable-diffusion-v2/stable-diffusion-v2-optimum-demo.ipynb notebooks/stable-diffusion-v2/stable-diffusion-v2-text-to-image-demo.ipynb notebooks/stable-diffusion-v2/stable-diffusion-v2-text-to-image.ipynb +notebooks/stable-diffusion-v3/stable-diffusion-v3.ipynb notebooks/stable-diffusion-xl/segmind-vegart.ipynb notebooks/stable-diffusion-xl/stable-diffusion-xl.ipynb notebooks/stable-video-diffusion/stable-video-diffusion.ipynb @@ -124,7 +126,6 @@ notebooks/tensorflow-hub/tensorflow-hub.ipynb notebooks/tensorflow-object-detection-to-openvino/tensorflow-instance-segmentation-to-openvino.ipynb notebooks/tensorflow-object-detection-to-openvino/tensorflow-object-detection-to-openvino.ipynb notebooks/tensorflow-quantization-aware-training/tensorflow-quantization-aware-training.ipynb -notebooks/text-prediction/text-prediction.ipynb notebooks/tflite-selfie-segmentation/tflite-selfie-segmentation.ipynb notebooks/tflite-to-openvino/tflite-to-openvino.ipynb notebooks/tiny-sd-image-generation/tiny-sd-image-generation.ipynb @@ -133,12 +134,10 @@ notebooks/triposr-3d-reconstruction/triposr-3d-reconstruction.ipynb notebooks/typo-detector/typo-detector.ipynb notebooks/vehicle-detection-and-recognition/vehicle-detection-and-recognition.ipynb notebooks/vision-background-removal/vision-background-removal.ipynb -notebooks/vision-image-colorization/vision-image-colorization.ipynb notebooks/vision-monodepth/vision-monodepth.ipynb notebooks/vision-paddlegan-anime/vision-paddlegan-anime.ipynb notebooks/vision-paddlegan-superresolution/vision-paddlegan-superresolution.ipynb -notebooks/whisper-subtitles-generation/whisper-convert.ipynb -notebooks/whisper-subtitles-generation/whisper-nncf-quantize.ipynb +notebooks/whisper-subtitles-generation/whisper-subtitles-generation.ipynb notebooks/wuerstchen-image-generation/wuerstchen-image-generation.ipynb notebooks/yolov10-optimization/yolov10-optimization.ipynb notebooks/yolov7-optimization/yolov7-optimization.ipynb diff --git a/docs/notebooks/amused-lightweight-text-to-image-with-output.rst b/docs/notebooks/amused-lightweight-text-to-image-with-output.rst index 4619212090a46f..50e3731ee0663b 100644 --- a/docs/notebooks/amused-lightweight-text-to-image-with-output.rst +++ b/docs/notebooks/amused-lightweight-text-to-image-with-output.rst @@ -14,8 +14,8 @@ small parameter count and few forward pass generation process, amused can generate many images quickly. This benefit is seen particularly at larger batch sizes. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Load and run the original @@ -76,12 +76,6 @@ Load and run the original pipeline image.save("text2image_256.png") -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/transformers/transformer_2d.py:34: FutureWarning: `Transformer2DModelOutput` is deprecated and will be removed in version 1.0.0. Importing `Transformer2DModelOutput` from `diffusers.models.transformer_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.modeling_outputs import Transformer2DModelOutput`, instead. - deprecate("Transformer2DModelOutput", "1.0.0", deprecation_message) - - .. parsed-literal:: @@ -202,29 +196,29 @@ Convert the Text Encoder .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4565: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:622: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:621: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! encoder_states = () if output_hidden_states else None - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:627: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:626: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:276: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:275: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:284: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:283: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:316: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:315: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:650: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:649: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:653: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:652: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:745: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:744: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1230: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1231: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: @@ -275,7 +269,7 @@ Convert the U-ViT transformer Convert VQ-GAN decoder (VQVAE) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Function ``get_latents`` is +Function ``get_latents`` is needed to return real latents for the conversion. Due to the VQVAE implementation autogenerated tensor of the required shape is not suitable. This function repeats part of ``AmusedPipeline``. @@ -333,13 +327,13 @@ suitable. This function repeats part of ``AmusedPipeline``. .. parsed-literal:: - /tmp/ipykernel_3064357/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /tmp/ipykernel_2636725/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! shape=shape.tolist(), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/autoencoders/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not force_not_quantize: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -477,7 +471,7 @@ And insert wrappers instances in the pipeline: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -571,7 +565,7 @@ Prepare calibration dataset We use a portion of -`conceptual_captions `__ +`conceptual_captions `__ dataset from Hugging Face as calibration data. To collect intermediate model inputs for calibration we customize ``CompiledModel``. @@ -612,7 +606,7 @@ model inputs for calibration we customize ``CompiledModel``. pipe.transformer.transformer = CompiledModelDecorator(ov_transformer_model, calibration_data, keep_prob=1.0) disable_progress_bar(pipe) - dataset = datasets.load_dataset("conceptual_captions", split="train").shuffle(seed=42) + dataset = datasets.load_dataset("google-research-datasets/conceptual_captions", split="train", trust_remote_code=True).shuffle(seed=42) # Run inference for data collection pbar = tqdm(total=calibration_dataset_size) @@ -686,14 +680,6 @@ model. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/datasets/load.py:1491: FutureWarning: The repository for conceptual_captions contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/conceptual_captions - You can avoid this message in future by passing the argument `trust_remote_code=True`. - Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`. - warnings.warn( - - .. parsed-literal:: @@ -702,7 +688,7 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -713,17 +699,17 @@ model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -734,17 +720,17 @@ model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -752,9 +738,9 @@ model. INFO:nncf:3 ignored nodes were found by types in the NNCFGraph INFO:nncf:182 ignored nodes were found by name in the NNCFGraph - INFO:nncf:Not adding activation input quantizer for operation: 37 __module.transformer.embed.conv/aten::_convolution/Convolution - INFO:nncf:Not adding activation input quantizer for operation: 2883 __module.transformer.mlm_layer.conv1/aten::_convolution/Convolution - INFO:nncf:Not adding activation input quantizer for operation: 3243 __module.transformer.mlm_layer.conv2/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 120 __module.transformer.embed.conv/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 2154 __module.transformer.mlm_layer.conv1/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 2993 __module.transformer.mlm_layer.conv2/aten::_convolution/Convolution @@ -764,41 +750,41 @@ model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) -.. raw:: html -
-    
+ + + @@ -822,7 +808,7 @@ Demo generation with quantized pipeline .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -866,7 +852,7 @@ a rough estimate of generation quality. pipe.transformer.transformer = core.compile_model(ov_transformer_model_path, device.value) disable_progress_bar(pipe) - dataset = datasets.load_dataset("conceptual_captions", "unlabeled", split="validation").shuffle(seed=42) + dataset = datasets.load_dataset("google-research-datasets/conceptual_captions", "unlabeled", split="validation", trust_remote_code=True).shuffle(seed=42) dataset = islice(dataset, validation_set_size) inception_score = InceptionScore(normalize=True, splits=1) @@ -906,11 +892,7 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/datasets/load.py:1491: FutureWarning: The repository for conceptual_captions contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/conceptual_captions - You can avoid this message in future by passing the argument `trust_remote_code=True`. - Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`. - warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. warnings.warn(\*args, \*\*kwargs) # noqa: B028 @@ -922,9 +904,9 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:176: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1807.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-716/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:176: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1807.) return kl.mean(), kl.std() @@ -941,8 +923,8 @@ a rough estimate of generation quality. .. parsed-literal:: - Quantized pipeline Inception Score: 9.630992889404297 - Quantization speed-up: 2.10x + Quantized pipeline Inception Score: 9.63099479675293 + Quantization speed-up: 2.09x Interactive inference diff --git a/docs/notebooks/animate-anyone-with-output.rst b/docs/notebooks/animate-anyone-with-output.rst index 9312ff261b123c..00b78cb83331f9 100644 --- a/docs/notebooks/animate-anyone-with-output.rst +++ b/docs/notebooks/animate-anyone-with-output.rst @@ -34,17 +34,14 @@ Learn more in `GitHub repo `__ and `paper `__. -.. container:: alert alert-warning - :: +.. warning:: -

! WARNING !

-

- This tutorial requires at least 96 GB of RAM for model conversion and 40 GB for inference. Changing the values of HEIGHT, WIDTH and VIDEO_LENGTH variables will change the memory consumption but will also affect accuracy. -

+ This tutorial requires at least **96 GB** of RAM for model conversion and **40 GB** for inference. Changing the values of ``HEIGHT``, ``WIDTH`` and ``VIDEO_LENGTH`` variables will change the memory consumption but will also affect accuracy. + + +**Table of contents:** -Table of contents: -^^^^^^^^^^^^^^^^^^ - `Prerequisites <#prerequisites>`__ - `Prepare base model <#prepare-base-model>`__ @@ -67,6 +64,11 @@ Table of contents: .. |image0| image:: https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/animate-anyone/animate-anyone.gif + + + + + Prerequisites ------------- @@ -154,11 +156,11 @@ Note that we clone a fork of original repo with tweaked forward methods. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( @@ -279,7 +281,7 @@ Download weights .. parsed-literal:: - config.json: 0%| | 0.00/547 [00:00`__. Weight @@ -503,7 +511,6 @@ of the pipeline, it will be better to convert them to separate models. .. parsed-literal:: - WARNING:nncf:NNCF provides best results with torch==2.2.*, while current torch version is 2.3.1+cpu. If you encounter issues, consider switching to torch==2.2.* INFO:nncf:Statistics of the bitwidth distribution: ┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │ @@ -519,17 +526,17 @@ of the pipeline, it will be better to convert them to separate models. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -572,17 +579,17 @@ of the pipeline, it will be better to convert them to separate models. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -646,17 +653,17 @@ step. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -747,17 +754,17 @@ step. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -801,17 +808,17 @@ efficiently integrate pose control signals into the denoising process. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -840,7 +847,7 @@ required for both reference and denoising UNets. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4565: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( @@ -861,17 +868,17 @@ required for both reference and denoising UNets. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -1211,7 +1218,7 @@ Video post-processing .. raw:: html diff --git a/docs/notebooks/async-api-with-output.rst b/docs/notebooks/async-api-with-output.rst index 73435822bbac2e..a498c9b34a73ce 100644 --- a/docs/notebooks/async-api-with-output.rst +++ b/docs/notebooks/async-api-with-output.rst @@ -11,8 +11,8 @@ device is busy with inference, the application can perform other tasks in parallel (for example, populating inputs or scheduling other requests) rather than wait for the current inference to complete first. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Imports <#imports>`__ - `Prepare model and data @@ -352,7 +352,7 @@ Test performance in Sync Mode .. parsed-literal:: Source ended - average throuput in sync mode: 60.54 fps + average throuput in sync mode: 60.97 fps Async Mode @@ -491,7 +491,7 @@ Test the performance in Async Mode .. parsed-literal:: Source ended - average throuput in async mode: 103.70 fps + average throuput in async mode: 105.34 fps Compare the performance @@ -634,5 +634,5 @@ Test the performance with ``AsyncInferQueue`` .. parsed-literal:: - average throughput in async mode with async infer queue: 148.11 fps + average throughput in async mode with async infer queue: 149.73 fps diff --git a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png index 589ab94948728b..4c12110dc64fec 100644 --- a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png +++ b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d82d618fb2b2ef25ecd8ad941de1d1173b3e21a3340314cc584dca9b32d6c55 -size 29416 +oid sha256:f244959cfb281521cd783e5942c77a18a614a8f6bfd4cb96b6f6d9c14a5cab17 +size 29396 diff --git a/docs/notebooks/auto-device-with-output.rst b/docs/notebooks/auto-device-with-output.rst index d05af42d51629e..4abf3cc4f87076 100644 --- a/docs/notebooks/auto-device-with-output.rst +++ b/docs/notebooks/auto-device-with-output.rst @@ -30,13 +30,13 @@ first inference. auto -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Import modules and create Core <#import-modules-and-create-core>`__ - `Convert the model to OpenVINO IR format <#convert-the-model-to-openvino-ir-format>`__ -- `(1) Simplify selection logic <#1-simplify-selection-logic>`__ +- `(1) Simplify selection logic <#simplify-selection-logic>`__ - `Default behavior of Core::compile_model API without device_name <#default-behavior-of-corecompile_model-api-without-device_name>`__ @@ -69,10 +69,10 @@ Import modules and create Core .. code:: ipython3 import platform - + # Install required packages %pip install -q "openvino>=2023.1.0" Pillow torch torchvision tqdm --extra-index-url https://download.pytorch.org/whl/cpu - + if platform.system() != "Windows": %pip install -q "matplotlib>=3.4" else: @@ -89,13 +89,13 @@ Import modules and create Core import time import sys - + import openvino as ov - + from IPython.display import Markdown, display - + core = ov.Core() - + if not any("GPU" in device for device in core.available_devices): display( Markdown( @@ -139,11 +139,11 @@ For more information about model conversion API, see this import torchvision from pathlib import Path - + base_model_dir = Path("./model") base_model_dir.mkdir(exist_ok=True) model_path = base_model_dir / "resnet50.xml" - + if not model_path.exists(): pt_model = torchvision.models.resnet50(weights="DEFAULT") ov_model = ov.convert_model(pt_model, input=[[1, 3, 224, 224]]) @@ -176,25 +176,26 @@ By default, ``compile_model`` API will select **AUTO** as # Set LOG_LEVEL to LOG_INFO. core.set_property("AUTO", {"LOG_LEVEL": "LOG_INFO"}) - + # Load the model onto the target device. compiled_model = core.compile_model(ov_model) - + if isinstance(compiled_model, ov.CompiledModel): print("Successfully compiled model without a device_name.") .. parsed-literal:: - [23:27:27.6972]I[plugin.cpp:418][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO - [23:27:27.6973]I[plugin.cpp:418][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY - [23:27:27.6973]I[plugin.cpp:418][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 - [23:27:27.6973]I[plugin.cpp:418][AUTO] device:CPU, config:PERF_COUNT=NO - [23:27:27.6973]I[plugin.cpp:423][AUTO] device:CPU, priority:0 - [23:27:27.6973]I[schedule.cpp:17][AUTO] scheduler starting - [23:27:27.6973]I[auto_schedule.cpp:131][AUTO] select device:CPU - [23:27:27.8462]I[auto_schedule.cpp:109][AUTO] device:CPU compiling model finished - [23:27:27.8464]I[plugin.cpp:451][AUTO] underlying hardware does not support hardware context + [23:25:26.9165]I[plugin.cpp:421][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO + [23:25:26.9166]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY + [23:25:26.9166]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 + [23:25:26.9166]I[plugin.cpp:421][AUTO] device:CPU, config:PERF_COUNT=NO + [23:25:26.9166]I[plugin.cpp:426][AUTO] device:CPU, priority:0 + [23:25:26.9166]I[schedule.cpp:17][AUTO] scheduler starting + [23:25:26.9166]I[auto_schedule.cpp:134][AUTO] select device:CPU + [23:25:27.0671]I[auto_schedule.cpp:336][AUTO] Device: [CPU]: Compile model took 150.493079 ms + [23:25:27.0674]I[auto_schedule.cpp:112][AUTO] device:CPU compiling model finished + [23:25:27.0675]I[plugin.cpp:454][AUTO] underlying hardware does not support hardware context Successfully compiled model without a device_name. @@ -208,7 +209,7 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: Deleted compiled_model - [23:27:27.8575]I[schedule.cpp:303][AUTO] scheduler ending + [23:25:27.0786]I[schedule.cpp:308][AUTO] scheduler ending Explicitly pass AUTO as device_name to Core::compile_model API @@ -223,9 +224,9 @@ improve readability of your code. # Set LOG_LEVEL to LOG_NONE. core.set_property("AUTO", {"LOG_LEVEL": "LOG_NONE"}) - + compiled_model = core.compile_model(model=ov_model, device_name="AUTO") - + if isinstance(compiled_model, ov.CompiledModel): print("Successfully compiled model using AUTO.") @@ -275,25 +276,25 @@ function, we will reuse it for preparing input data. # Fetch `notebook_utils` module import requests - + r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py") open("notebook_utils.py", "w").write(r.text) - + from notebook_utils import download_file .. code:: ipython3 from PIL import Image - + # Download the image from the openvino_notebooks storage image_filename = download_file( "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", directory="data", ) - + image = Image.open(str(image_filename)) input_transform = torchvision.models.ResNet50_Weights.DEFAULT.transforms() - + input_tensor = input_transform(image) input_tensor = input_tensor.unsqueeze(0).numpy() image @@ -324,10 +325,10 @@ Load the model to GPU device and perform inference # Start time. gpu_load_start_time = time.perf_counter() compiled_model = core.compile_model(model=ov_model, device_name="GPU") # load to GPU - + # Execute the first inference. results = compiled_model(input_tensor)[0] - + # Measure time to the first inference. gpu_fil_end_time = time.perf_counter() gpu_fil_span = gpu_fil_end_time - gpu_load_start_time @@ -353,11 +354,11 @@ executed on CPU until GPU is ready. # Start time. auto_load_start_time = time.perf_counter() compiled_model = core.compile_model(model=ov_model) # The device_name is AUTO by default. - + # Execute the first inference. results = compiled_model(input_tensor)[0] - - + + # Measure time to the first inference. auto_fil_end_time = time.perf_counter() auto_fil_span = auto_fil_end_time - auto_load_start_time @@ -410,7 +411,7 @@ Class and callback definition :member: latency_list: Record the latency of each inference execution over @interval seconds duration. :member: interval: The metrics will be updated every @interval seconds """ - + def __init__(self, interval): """ Create and initilize one instance of class PerformanceMetrics. @@ -420,11 +421,11 @@ Class and callback definition """ self.fps = 0 self.latency = 0 - + self.start_time = time.perf_counter() self.latency_list = [] self.interval = interval - + def update(self, infer_request: ov.InferRequest) -> bool: """ Update the metrics if current ongoing @interval seconds duration is expired. Record the latency only if it is not expired. @@ -446,8 +447,8 @@ Class and callback definition return True else: return False - - + + class InferContext: """ Inference context. Record and update peforamnce metrics via @metrics, set @feed_inference to False once @remaining_update_num <=0 @@ -455,7 +456,7 @@ Class and callback definition :member: remaining_update_num: the remaining times for peforamnce metrics updating. :member: feed_inference: if feed inference request is required or not. """ - + def __init__(self, update_interval, num): """ Create and initilize one instance of class InferContext. @@ -467,7 +468,7 @@ Class and callback definition self.metrics = PerformanceMetrics(update_interval) self.remaining_update_num = num self.feed_inference = True - + def update(self, infer_request: ov.InferRequest): """ Update the context. Set @feed_inference to False if the number of remaining performance metric updates (@remaining_update_num) reaches 0 @@ -476,13 +477,13 @@ Class and callback definition """ if self.remaining_update_num <= 0: self.feed_inference = False - + if self.metrics.update(infer_request): self.remaining_update_num = self.remaining_update_num - 1 if self.remaining_update_num <= 0: self.feed_inference = False - - + + def completion_callback(infer_request: ov.InferRequest, context) -> None: """ callback for the inference request, pass the @infer_request to @context for updating @@ -491,8 +492,8 @@ Class and callback definition :returns: None """ context.update(infer_request) - - + + # Performance metrics update interval (seconds) and number of times. metrics_update_interval = 10 metrics_update_num = 6 @@ -508,29 +509,29 @@ Loop for inference and update the FPS/Latency every .. code:: ipython3 THROUGHPUT_hint_context = InferContext(metrics_update_interval, metrics_update_num) - + print("Compiling Model for AUTO device with THROUGHPUT hint") sys.stdout.flush() - + compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT": "THROUGHPUT"}) - + infer_queue = ov.AsyncInferQueue(compiled_model, 0) # Setting to 0 will query optimal number by default. infer_queue.set_callback(completion_callback) - + print(f"Start inference, {metrics_update_num: .0f} groups of FPS/latency will be measured over {metrics_update_interval: .0f}s intervals") sys.stdout.flush() - + while THROUGHPUT_hint_context.feed_inference: infer_queue.start_async(input_tensor, THROUGHPUT_hint_context) - + infer_queue.wait_all() - + # Take the FPS and latency of the latest period. THROUGHPUT_hint_fps = THROUGHPUT_hint_context.metrics.fps THROUGHPUT_hint_latency = THROUGHPUT_hint_context.metrics.latency - + print("Done") - + del compiled_model @@ -538,12 +539,12 @@ Loop for inference and update the FPS/Latency every Compiling Model for AUTO device with THROUGHPUT hint Start inference, 6 groups of FPS/latency will be measured over 10s intervals - throughput: 177.73fps, latency: 32.02ms, time interval: 10.01s - throughput: 179.52fps, latency: 32.63ms, time interval: 10.00s - throughput: 178.56fps, latency: 32.79ms, time interval: 10.00s - throughput: 177.70fps, latency: 32.99ms, time interval: 10.01s - throughput: 178.80fps, latency: 32.69ms, time interval: 10.02s - throughput: 177.72fps, latency: 33.00ms, time interval: 10.01s + throughput: 178.13fps, latency: 31.99ms, time interval: 10.00s + throughput: 180.15fps, latency: 32.47ms, time interval: 10.00s + throughput: 179.79fps, latency: 32.55ms, time interval: 10.01s + throughput: 178.57fps, latency: 32.85ms, time interval: 10.01s + throughput: 179.62fps, latency: 32.61ms, time interval: 10.00s + throughput: 179.43fps, latency: 32.69ms, time interval: 10.00s Done @@ -558,30 +559,30 @@ Loop for inference and update the FPS/Latency for each .. code:: ipython3 LATENCY_hint_context = InferContext(metrics_update_interval, metrics_update_num) - + print("Compiling Model for AUTO Device with LATENCY hint") sys.stdout.flush() - + compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT": "LATENCY"}) - + # Setting to 0 will query optimal number by default. infer_queue = ov.AsyncInferQueue(compiled_model, 0) infer_queue.set_callback(completion_callback) - + print(f"Start inference, {metrics_update_num: .0f} groups fps/latency will be out with {metrics_update_interval: .0f}s interval") sys.stdout.flush() - + while LATENCY_hint_context.feed_inference: infer_queue.start_async(input_tensor, LATENCY_hint_context) - + infer_queue.wait_all() - + # Take the FPS and latency of the latest period. LATENCY_hint_fps = LATENCY_hint_context.metrics.fps LATENCY_hint_latency = LATENCY_hint_context.metrics.latency - + print("Done") - + del compiled_model @@ -589,12 +590,12 @@ Loop for inference and update the FPS/Latency for each Compiling Model for AUTO Device with LATENCY hint Start inference, 6 groups fps/latency will be out with 10s interval - throughput: 135.52fps, latency: 6.87ms, time interval: 10.01s - throughput: 137.89fps, latency: 6.85ms, time interval: 10.00s - throughput: 137.71fps, latency: 6.82ms, time interval: 10.01s - throughput: 137.83fps, latency: 6.83ms, time interval: 10.01s - throughput: 137.80fps, latency: 6.83ms, time interval: 10.01s - throughput: 138.34fps, latency: 6.84ms, time interval: 10.00s + throughput: 138.97fps, latency: 6.66ms, time interval: 10.00s + throughput: 142.02fps, latency: 6.67ms, time interval: 10.01s + throughput: 141.78fps, latency: 6.62ms, time interval: 10.00s + throughput: 141.76fps, latency: 6.63ms, time interval: 10.00s + throughput: 141.93fps, latency: 6.66ms, time interval: 10.01s + throughput: 142.25fps, latency: 6.64ms, time interval: 10.00s Done @@ -606,16 +607,16 @@ Difference in FPS and latency .. code:: ipython3 import matplotlib.pyplot as plt - + TPUT = 0 LAT = 1 labels = ["THROUGHPUT hint", "LATENCY hint"] - + fig1, ax1 = plt.subplots(1, 1) fig1.patch.set_visible(False) ax1.axis("tight") ax1.axis("off") - + cell_text = [] cell_text.append( [ @@ -624,7 +625,7 @@ Difference in FPS and latency ] ) cell_text.append(["%.2f%s" % (LATENCY_hint_fps, " FPS"), "%.2f%s" % (LATENCY_hint_latency, " ms")]) - + table = ax1.table( cellText=cell_text, colLabels=["FPS (Higher is better)", "Latency (Lower is better)"], @@ -639,7 +640,7 @@ Difference in FPS and latency table.auto_set_column_width(0) table.auto_set_column_width(1) table.scale(1, 3) - + fig1.tight_layout() plt.show() @@ -653,28 +654,28 @@ Difference in FPS and latency # Output the difference. width = 0.4 fontsize = 14 - + plt.rc("font", size=fontsize) fig, ax = plt.subplots(1, 2, figsize=(10, 8)) - + rects1 = ax[0].bar([0], THROUGHPUT_hint_fps, width, label=labels[TPUT], color="#557f2d") rects2 = ax[0].bar([width], LATENCY_hint_fps, width, label=labels[LAT]) ax[0].set_ylabel("frames per second") ax[0].set_xticks([width / 2]) ax[0].set_xticklabels(["FPS"]) ax[0].set_xlabel("Higher is better") - + rects1 = ax[1].bar([0], THROUGHPUT_hint_latency, width, label=labels[TPUT], color="#557f2d") rects2 = ax[1].bar([width], LATENCY_hint_latency, width, label=labels[LAT]) ax[1].set_ylabel("milliseconds") ax[1].set_xticks([width / 2]) ax[1].set_xticklabels(["Latency (ms)"]) ax[1].set_xlabel("Lower is better") - + fig.suptitle("Performance Hints") fig.legend(labels, fontsize=fontsize) fig.tight_layout() - + plt.show() diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png index 9e2d67fe539252..54de802ad396dc 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:50f5fe192a152bfb8036a782ee93ab26543c9a30fc626e67aab5e5a7dba9e35a -size 27240 +oid sha256:807232c3d860a113c5a6bb3549f746e61548c95977692ad10c94a6ce8f17d12c +size 26625 diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png index 1f42981e0fb5b4..7447b7e75aaf75 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a4bfe1e79236b08d4848e6ff59eccf1e2c459d74cb8940b54e8cad3ef01a948 -size 40033 +oid sha256:d0304a4d2a5cd64a60a96e78eea3ee2afa0f6c8115767ee620e6509d109d8269 +size 39949 diff --git a/docs/notebooks/bark-text-to-audio-with-output.rst b/docs/notebooks/bark-text-to-audio-with-output.rst index 1c2c30cc4f5b80..c6d67c50fb30eb 100644 --- a/docs/notebooks/bark-text-to-audio-with-output.rst +++ b/docs/notebooks/bark-text-to-audio-with-output.rst @@ -54,8 +54,8 @@ tokens into audio codec tokens to generate the full waveform. To enable the community to use Bark via public code, EnCodec codec from Facebook is used to act as an audio representation. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Download and Convert models <#download-and-convert-models>`__ diff --git a/docs/notebooks/blip-visual-language-processing-with-output.rst b/docs/notebooks/blip-visual-language-processing-with-output.rst index 8a00c7cdf6dc7e..88b1c999c03b89 100644 --- a/docs/notebooks/blip-visual-language-processing-with-output.rst +++ b/docs/notebooks/blip-visual-language-processing-with-output.rst @@ -30,8 +30,8 @@ The tutorial consists of the following parts: 5. Compare original and optimized models 6. Launch interactive demo -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Background <#background>`__ @@ -270,7 +270,7 @@ text and vision modalities and postprocessing of generation results. import platform - %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "torch>=2.1.0" torchvision "transformers>=4.26.0" "gradio>=4.19" "openvino>=2023.3.0" "datasets>==2.14.6" "nncf>=2.8.1" "tqdm" + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "torch>=2.1.0" torchvision "transformers>=4.26.0" "gradio>=4.19" "openvino>=2023.3.0" "datasets>=2.14.6" "nncf>=2.8.1" "tqdm" if platform.system() != "Windows": %pip install -q "matplotlib>=3.4" else: @@ -831,7 +831,7 @@ understanding of vision, language and commonsense knowledge to answer. Prepares a vision-text dataset for quantization. """ split = f"train[:{opt_init_steps}]" if not streaming else "train" - dataset = load_dataset("HuggingFaceM4/VQAv2", split=split, streaming=streaming) + dataset = load_dataset("HuggingFaceM4/VQAv2", split=split, streaming=streaming, trust_remote_code=True) dataset = dataset.shuffle(seed=42) if streaming: dataset = dataset.take(opt_init_steps) @@ -878,17 +878,17 @@ Quantize vision model -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -899,17 +899,17 @@ Quantize vision model -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -926,17 +926,17 @@ Quantize vision model -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -947,17 +947,17 @@ Quantize vision model -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -988,17 +988,17 @@ Quantize text encoder -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -1009,17 +1009,17 @@ Quantize text encoder -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -1036,17 +1036,17 @@ Quantize text encoder -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -1057,17 +1057,17 @@ Quantize text encoder -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -1113,17 +1113,17 @@ The optimization process contains the following steps: -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + diff --git a/docs/notebooks/clip-language-saliency-map-with-output.rst b/docs/notebooks/clip-language-saliency-map-with-output.rst index 9ac43fe4b4266d..08248f3aa58168 100644 --- a/docs/notebooks/clip-language-saliency-map-with-output.rst +++ b/docs/notebooks/clip-language-saliency-map-with-output.rst @@ -79,8 +79,8 @@ used to build the saliency map. Here is how it can be done: 5. Update the corresponding region on the ``saliency map``. 6. Repeat steps 2-5 multiple times (``n_iters``). -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Initial Implementation with Transformers and Pytorch <#initial-implementation-with-transformers-and-pytorch>`__ diff --git a/docs/notebooks/clip-zero-shot-classification-with-output.rst b/docs/notebooks/clip-zero-shot-classification-with-output.rst index 40cca0e2ec9f15..f8d06e6dd91422 100644 --- a/docs/notebooks/clip-zero-shot-classification-with-output.rst +++ b/docs/notebooks/clip-zero-shot-classification-with-output.rst @@ -36,8 +36,8 @@ The notebook contains the following steps: 8. Compare performance of converted and quantized models. 9. Launch interactive demo -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Instantiate model <#instantiate-model>`__ - `Run PyTorch model inference <#run-pytorch-model-inference>`__ @@ -318,6 +318,8 @@ Quantize model to INT8 using NNCF --------------------------------- + + The goal of this part of tutorial is to demonstrate how to speed up the model by applying 8-bit post-training quantization from `NNCF `__ (Neural Network @@ -474,7 +476,7 @@ model. """ Prepares a vision-text dataset for quantization. """ - dataset = load_dataset("conceptual_captions") + dataset = load_dataset("google-research-datasets/conceptual_captions", trust_remote_code=True) train_dataset = dataset["train"].shuffle(seed=42) dataloader = torch.utils.data.DataLoader(train_dataset, collate_fn=collate_fn, batch_size=1) calibration_data = prepare_calibration_data(dataloader, opt_init_steps) @@ -563,17 +565,17 @@ Create a quantized model from the pre-trained ``FP16`` model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -584,17 +586,17 @@ Create a quantized model from the pre-trained ``FP16`` model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -605,17 +607,17 @@ Create a quantized model from the pre-trained ``FP16`` model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -626,17 +628,17 @@ Create a quantized model from the pre-trained ``FP16`` model. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -697,7 +699,7 @@ Compare File Size Compare inference time of the FP16 IR and quantized models ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - To measure the inference +To measure the inference performance of the ``FP16`` and ``INT8`` models, we use median inference time on calibration dataset. So we can approximately estimate the speed up of the dynamic quantized models. @@ -740,6 +742,7 @@ up of the dynamic quantized models. Interactive demo ---------------- + Now, it is your turn! You can provide your own image and comma-separated list of labels for zero-shot classification. diff --git a/docs/notebooks/controlnet-stable-diffusion-with-output.rst b/docs/notebooks/controlnet-stable-diffusion-with-output.rst index 1e9957a95b8646..fd4eb02ad019f6 100644 --- a/docs/notebooks/controlnet-stable-diffusion-with-output.rst +++ b/docs/notebooks/controlnet-stable-diffusion-with-output.rst @@ -141,8 +141,8 @@ of the target in the image: This tutorial focuses mainly on conditioning by pose. However, the discussed steps are also applicable to other annotation modes. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Instantiating Generation diff --git a/docs/notebooks/convert-to-openvino-with-output.rst b/docs/notebooks/convert-to-openvino-with-output.rst index c3a12ea70bd6cd..cd272f1436edd3 100644 --- a/docs/notebooks/convert-to-openvino-with-output.rst +++ b/docs/notebooks/convert-to-openvino-with-output.rst @@ -4,8 +4,8 @@ OpenVINO™ Model conversion This notebook shows how to convert a model from original framework format to OpenVINO Intermediate Representation (IR). -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `OpenVINO IR format <#openvino-ir-format>`__ - `Fetching example models <#fetching-example-models>`__ @@ -35,7 +35,7 @@ Table of contents: .. parsed-literal:: - Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.0) + Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.1.1) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -181,11 +181,11 @@ NLP model from Hugging Face and export it in ONNX format: .. parsed-literal:: - 2024-06-05 23:48:38.001731: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-06-05 23:48:38.036985: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-07-01 23:46:23.388228: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-07-01 23:46:23.422415: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-06-05 23:48:38.551703: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:231: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + 2024-07-01 23:46:24.062925: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:230: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mask, torch.tensor(torch.finfo(scores.dtype).min) @@ -662,12 +662,12 @@ frameworks conversion guides. .. parsed-literal:: - 2024-06-05 23:48:58.596260: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW - 2024-06-05 23:48:58.596295: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07 - 2024-06-05 23:48:58.596299: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07 - 2024-06-05 23:48:58.596508: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2 - 2024-06-05 23:48:58.596524: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3 - 2024-06-05 23:48:58.596528: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration + 2024-07-01 23:46:43.443603: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW + 2024-07-01 23:46:43.443639: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07 + 2024-07-01 23:46:43.443643: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07 + 2024-07-01 23:46:43.443849: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2 + 2024-07-01 23:46:43.443867: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3 + 2024-07-01 23:46:43.443871: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration Migration from Legacy conversion API @@ -725,6 +725,13 @@ Resnet50 model that was exported to the ONNX format: ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, layout="nchw") +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + .. parsed-literal:: huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... @@ -768,6 +775,17 @@ and the layout of an original model: # alternatively use source_layout and target_layout parameters ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, source_layout="nchw", target_layout="nhwc") + +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + Specifying Mean and Scale Values ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -806,6 +824,14 @@ more examples. scale_values=[255 * x for x in [0.229, 0.224, 0.225]], ) + +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + Reversing Input Channels ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -836,6 +862,14 @@ the color channels before inference. ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, reverse_input_channels=True) + +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + Cutting Off Parts of a Model ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/notebooks/convnext-classification-with-output.rst b/docs/notebooks/convnext-classification-with-output.rst index 6e7c73162fa6c7..4e175694fc9ff0 100644 --- a/docs/notebooks/convnext-classification-with-output.rst +++ b/docs/notebooks/convnext-classification-with-output.rst @@ -22,8 +22,8 @@ maintaining the simplicity and efficiency of standard ConvNets. The several pretrained ConvNeXt model. In this tutorial we will use ConvNeXt Tiny model. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Get a test image <#get-a-test-image>`__ @@ -50,29 +50,27 @@ Prerequisites .. parsed-literal:: - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. Get a test image ---------------- - First of all lets get a test +First of all lets get a test image from an open dataset. .. code:: ipython3 import requests - + from torchvision.io import read_image import torchvision.transforms as transforms - - + + img_path = "cats_image.jpeg" r = requests.get("https://huggingface.co/datasets/huggingface/cats-image/resolve/main/cats_image.jpeg") - + with open(img_path, "wb") as f: f.write(r.content) image = read_image(img_path) @@ -86,19 +84,19 @@ image from an open dataset. Get a pretrained model ---------------------- - Torchvision provides a +Torchvision provides a mechanism of `listing and retrieving available models `__. .. code:: ipython3 import torchvision.models as models - + # List available models all_models = models.list_models() # List of models by type. Classification models are in the parent module. classification_models = models.list_models(module=models) - + print(classification_models) @@ -126,7 +124,7 @@ initialize pre-trained models Define a preprocessing and prepare an input data ------------------------------------------------ - You can use +You can use ``torchvision.transforms`` to make a preprocessing or use\ `preprocessing transforms from the model wight `__. @@ -134,10 +132,10 @@ wight `__ - `Clean Text <#clean-text>`__ diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst index 602a5753f5c417..59623292d4026a 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst @@ -53,8 +53,8 @@ demonstration purposes, this tutorial will download one converted CT scan and use that scan for quantization and inference. For production purposes, use a representative dataset for quantizing the model. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Imports <#imports>`__ - `Settings <#settings>`__ @@ -152,10 +152,10 @@ Imports .. parsed-literal:: - 2024-06-05 23:50:19.239580: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-06-05 23:50:19.274774: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-07-01 23:47:08.962451: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-07-01 23:47:08.997285: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-06-05 23:50:19.860831: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-07-01 23:47:09.575374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -435,7 +435,7 @@ this notebook. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: @@ -486,17 +486,17 @@ steps: -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -507,17 +507,17 @@ steps: -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -534,18 +534,18 @@ Convert quantized model to OpenVINO IR model and save it. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:337: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:339: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_low.item() - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:345: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:347: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_high.item() - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 249914 / 262144 (95.3%) - Greatest absolute difference: 4.319500803947449 at index (0, 0, 125, 295) (up to 1e-05 allowed) - Greatest relative difference: 10952.699411314505 at index (0, 0, 220, 387) (up to 1e-05 allowed) + Mismatched elements: 247635 / 262144 (94.5%) + Greatest absolute difference: 4.091559410095215 at index (0, 0, 441, 89) (up to 1e-05 allowed) + Greatest relative difference: 6430.725591012176 at index (0, 0, 48, 134) (up to 1e-05 allowed) _check_trace( @@ -668,18 +668,18 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 8.75 ms + [ INFO ] Read model took 8.87 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,?,?,?] @@ -693,7 +693,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [?,1,16..,16..] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 150.71 ms + [ INFO ] Compile model took 192.84 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -728,9 +728,9 @@ be run in the notebook with ``! benchmark_app`` or [Step 9/11] Creating infer requests and preparing input tensors [ ERROR ] Input x is dynamic. Provide data shapes! Traceback (most recent call last): - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 486, in main + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 486, in main data_queue = get_input_data(paths_to_input, app_inputs_info) - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data raise Exception(f"Input {info.name} is dynamic. Provide data shapes!") Exception: Input x is dynamic. Provide data shapes! @@ -747,18 +747,18 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 10.69 ms + [ INFO ] Read model took 10.93 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,1,512,512] @@ -772,7 +772,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 275.30 ms + [ INFO ] Compile model took 227.54 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model49 @@ -809,17 +809,17 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 29.48 ms + [ INFO ] First inference took 31.23 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 969 iterations - [ INFO ] Duration: 15001.31 ms + [ INFO ] Count: 970 iterations + [ INFO ] Duration: 15001.74 ms [ INFO ] Latency: - [ INFO ] Median: 15.23 ms - [ INFO ] Average: 15.28 ms - [ INFO ] Min: 14.97 ms - [ INFO ] Max: 17.10 ms - [ INFO ] Throughput: 64.59 FPS + [ INFO ] Median: 15.20 ms + [ INFO ] Average: 15.26 ms + [ INFO ] Min: 14.94 ms + [ INFO ] Max: 19.19 ms + [ INFO ] Throughput: 64.66 FPS Visually Compare Inference Results @@ -904,7 +904,7 @@ seed is displayed to enable reproducing specific runs of this cell. .. parsed-literal:: - Visualizing results with seed 1717624288 + Visualizing results with seed 1719870497 @@ -987,8 +987,8 @@ performs inference, and displays the results on the frames loaded in .. parsed-literal:: - Loaded model to AUTO in 0.23 seconds. - Total time for 68 frames: 2.33 seconds, fps:29.56 + Loaded model to AUTO in 0.19 seconds. + Total time for 68 frames: 2.41 seconds, fps:28.61 References diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png index 60f97677de531c..66c0fdf7363487 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1a4a6ad8cf666b4ce12cb07fa53b22a2ba0257697d926d9846ee4c18752fc553 -size 380300 +oid sha256:f98b4bde92e898a1005c5a07234464687d828ba03e25ac44551a12f75504c878 +size 379807 diff --git a/docs/notebooks/ddcolor-image-colorization-with-output.rst b/docs/notebooks/ddcolor-image-colorization-with-output.rst index 24f1c30e773fa3..42e12cca2092f5 100644 --- a/docs/notebooks/ddcolor-image-colorization-with-output.rst +++ b/docs/notebooks/ddcolor-image-colorization-with-output.rst @@ -25,10 +25,8 @@ In this tutorial we consider how to convert and run DDColor using OpenVINO. Additionally, we will demonstrate how to optimize this model using `NNCF `__. -🪄 Let’s start to explore magic of image colorization! - -Table of contents: -^^^^^^^^^^^^^^^^^^ +🪄 Let’s start to explore magic of image colorization! #### Table of +contents: - `Prerequisites <#prerequisites>`__ - `Load PyTorch model <#load-pytorch-model>`__ @@ -59,14 +57,10 @@ Prerequisites .. code:: ipython3 import platform - import os - - os.environ["GIT_CLONE_PROTECTION_ACTIVE"] = "false" - - %pip install -q timm "torch>=2.1" "torchvision" "opencv_python" "pillow" "PyYAML" "scipy" "scikit-image" "datasets" "gradio>=4.19" --extra-index-url https://download.pytorch.org/whl/cpu + + %pip install -q "nncf>=2.11.0" "torch>=2.1" "torchvision" "timm" "opencv_python" "pillow" "PyYAML" "scipy" "scikit-image" "datasets" "gradio>=4.19" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -Uq --pre "openvino" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - %pip install -q "git+https://github.com/openvinotoolkit/nncf.git" - + if platform.python_version_tuple()[1] in ["8", "9"]: %pip install -q "gradio-imageslider<=0.0.17" "typing-extensions>=4.9.0" else: @@ -77,8 +71,7 @@ Prerequisites Note: you may need to restart the kernel to use updated packages. ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - openvino-dev 2024.1.0 requires openvino==2024.1.0, but you have openvino 2024.3.0.dev20240605 which is incompatible. - Note: you may need to restart the kernel to use updated packages. + openvino-dev 2024.2.0 requires openvino==2024.2.0, but you have openvino 2024.3.0.dev20240627 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -87,24 +80,24 @@ Prerequisites import sys from pathlib import Path - + repo_dir = Path("DDColor") - + if not repo_dir.exists(): !git clone https://github.com/piddnad/DDColor.git - + sys.path.append(str(repo_dir)) .. parsed-literal:: Cloning into 'DDColor'... - remote: Enumerating objects: 223, done. - remote: Counting objects: 100% (69/69), done. - remote: Compressing objects: 100% (35/35), done. - remote: Total 223 (delta 51), reused 36 (delta 33), pack-reused 154 - Receiving objects: 100% (223/223), 13.34 MiB | 22.50 MiB/s, done. - Resolving deltas: 100% (72/72), done. + remote: Enumerating objects: 230, done. + remote: Counting objects: 100% (76/76), done. + remote: Compressing objects: 100% (39/39), done. + remote: Total 230 (delta 54), reused 40 (delta 36), pack-reused 154 + Receiving objects: 100% (230/230), 13.34 MiB | 21.61 MiB/s, done. + Resolving deltas: 100% (75/75), done. .. code:: ipython3 @@ -128,30 +121,17 @@ models from DDColor family. .. code:: ipython3 import torch - + model_name = "ddcolor_paper_tiny" - + ddcolor_model = DDColorHF.from_pretrained(f"piddnad/{model_name}") - - + + colorizer = ImageColorizationPipelineHF(model=ddcolor_model, input_size=512) - + ddcolor_model.to("cpu") colorizer.device = torch.device("cpu") - - -.. parsed-literal:: - - config.json: 0%| | 0.00/258 [00:00 lab -> get grey -> rgb img = cv2.resize(img, (512, 512)) img_l = cv2.cvtColor(img, cv2.COLOR_BGR2Lab)[:, :, :1] img_gray_lab = np.concatenate((img_l, np.zeros_like(img_l), np.zeros_like(img_l)), axis=-1) img_gray_rgb = cv2.cvtColor(img_gray_lab, cv2.COLOR_LAB2RGB) - + # Transpose HWC -> CHW and add batch dimension tensor_gray_rgb = torch.from_numpy(img_gray_rgb.transpose((2, 0, 1))).float().unsqueeze(0) - + # Run model inference output_ab = compiled_model(tensor_gray_rgb)[0] - + # Postprocess result # resize ab -> concat original l -> rgb output_ab_resize = F.interpolate(torch.from_numpy(output_ab), size=(height, width))[0].float().numpy().transpose(1, 2, 0) output_lab = np.concatenate((orig_l, output_ab_resize), axis=-1) output_bgr = cv2.cvtColor(output_lab, cv2.COLOR_LAB2BGR) - + output_img = (output_bgr * 255.0).round().astype(np.uint8) - + return output_img .. code:: ipython3 @@ -325,7 +305,7 @@ improve model inference speed. description="Quantization", disabled=False, ) - + to_quantize @@ -340,15 +320,15 @@ improve model inference speed. .. code:: ipython3 import requests - + OV_INT8_COLORIZER_PATH = Path("ddcolor_int8.xml") compiled_int8_model = None - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/skip_kernel_extension.py", ) open("skip_kernel_extension.py", "w").write(r.text) - + %load_ext skip_kernel_extension Collect quantization dataset @@ -363,14 +343,14 @@ dataset from Hugging Face as calibration data. .. code:: ipython3 %%skip not $to_quantize.value - + from datasets import load_dataset - + subset_size = 300 calibration_data = [] - + if not OV_INT8_COLORIZER_PATH.exists(): - dataset = load_dataset("ummagumm-a/colorization_dataset", split="train").shuffle(seed=42) + dataset = load_dataset("ummagumm-a/colorization_dataset", split="train", streaming=True).shuffle(seed=42).take(subset_size) for idx, batch in enumerate(dataset): if idx >= subset_size: break @@ -380,29 +360,10 @@ dataset from Hugging Face as calibration data. img_l = cv2.cvtColor(np.stack([img, img, img], axis=2), cv2.COLOR_BGR2Lab)[:, :, :1] img_gray_lab = np.concatenate((img_l, np.zeros_like(img_l), np.zeros_like(img_l)), axis=-1) img_gray_rgb = cv2.cvtColor(img_gray_lab, cv2.COLOR_LAB2RGB) - + image = np.expand_dims(img_gray_rgb.transpose((2, 0, 1)).astype(np.float32), axis=0) calibration_data.append(image) - - -.. parsed-literal:: - - Downloading readme: 0%| | 0.00/574 [00:00 -.. raw:: html -
-    
+ + + + + @@ -465,17 +426,17 @@ Perform model quantization -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -487,7 +448,7 @@ Run INT8 model inference .. code:: ipython3 from IPython.display import display - + if OV_INT8_COLORIZER_PATH.exists(): compiled_int8_model = core.compile_model(OV_INT8_COLORIZER_PATH, device.value) img = cv2.imread("DDColor/assets/test_images/Ansel Adams _ Moore Photography.jpeg") @@ -507,9 +468,9 @@ Compare FP16 and INT8 model size .. code:: ipython3 fp16_ir_model_size = OV_COLORIZER_PATH.with_suffix(".bin").stat().st_size / 2**20 - + print(f"FP16 model size: {fp16_ir_model_size:.2f} MB") - + if OV_INT8_COLORIZER_PATH.exists(): quantized_model_size = OV_INT8_COLORIZER_PATH.with_suffix(".bin").stat().st_size / 2**20 print(f"INT8 model size: {quantized_model_size:.2f} MB") @@ -547,18 +508,18 @@ Tool `__ - `Prepare DeciDiffusion models for OpenVINO format @@ -89,7 +89,7 @@ install required packages .. code:: ipython3 - %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "diffusers" "transformers" "torch>=2.1" "pillow" "openvino>=2023.1.0" "gradio>=4.19" "datasets>=2.14.6" "huggingface-hub>=0.19.4" "nncf>=2.7.0" "peft==0.6.2" "opencv-python" + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "diffusers<0.29.0" "transformers" "torch>=2.1" "pillow" "openvino>=2023.1.0" "gradio>=4.19" "datasets>=2.14.6" "huggingface-hub>=0.19.4" "nncf>=2.7.0" "peft==0.6.2" "opencv-python" Prepare DeciDiffusion models for OpenVINO format conversion ----------------------------------------------------------- @@ -1210,7 +1210,7 @@ Prepare calibration dataset We use a portion of -`conceptual_captions `__ +`conceptual_captions `__ dataset from Hugging Face as calibration data. To collect intermediate model inputs for calibration we should customize ``CompiledModel``. @@ -1245,7 +1245,7 @@ model inputs for calibration we should customize ``CompiledModel``. pipeline.unet = CompiledModelDecorator(original_unet, prob=0.3) pipeline.set_progress_bar_config(disable=True) - dataset = datasets.load_dataset("conceptual_captions", split="train", streaming=True).shuffle(seed=42) + dataset = datasets.load_dataset("google-research-datasets/conceptual_captions", split="train", streaming=True, trust_remote_code=True).shuffle(seed=42) pbar = tqdm(total=subset_size) for batch in dataset: @@ -1440,7 +1440,7 @@ pipelines, we use median inference time on calibration subset. import time validation_size = 10 - calibration_dataset = datasets.load_dataset("conceptual_captions", split="train", streaming=True) + calibration_dataset = datasets.load_dataset("conceptual_captions", split="train", streaming=True, trust_remote_code=True) validation_data = [] for idx, batch in enumerate(calibration_dataset): if idx >= validation_size: diff --git a/docs/notebooks/depth-anything-v2-with-output.rst b/docs/notebooks/depth-anything-v2-with-output.rst new file mode 100644 index 00000000000000..731488f78c7007 --- /dev/null +++ b/docs/notebooks/depth-anything-v2-with-output.rst @@ -0,0 +1,1176 @@ +Depth estimation with DepthAnythingV2 and OpenVINO +================================================== + +`Depth Anything V2 `__ is a +solution for robust relative depth estimation. Without pursuing fancy +techniques, this project aim to reveal crucial findings to pave the way +towards building a powerful monocular depth estimation model. This model +is an improvement of the `Depth Anything +V1 `__. Notably, compared with V1, +this version produces much finer and more robust depth predictions +through three key practices: replacing all labeled real images with +synthetic images, scaling up the capacity of our teacher model and +teaching student models via the bridge of large-scale pseudo-labeled +real images. + +The pipeline of training of Depth Anything V2 is shown below. It +consists of three steps: train a reliable teacher model purely on +high-quality synthetic image; produce precise pseudo depth on +large-scale unlabeled real images; train final student models on +pseudo-labeled real images for robust generalization. + +.. figure:: https://depth-anything-v2.github.io/static/images/pipeline.png + :alt: image.png + + image.png + +More details about model can be found in `project web +page `__, +`paper `__ and official +`repository `__ + +In this tutorial we will explore how to convert and run DepthAnythingV2 +using OpenVINO. An additional part demonstrates how to run quantization +with `NNCF `__ to speed up the +model. + +**Table of contents:** + + +- `Prerequisites <#prerequisites>`__ +- `Load and run PyTorch model <#load-and-run-pytorch-model>`__ + + - `Prepare input data <#prepare-input-data>`__ + - `Run model inference <#run-model-inference>`__ + +- `Convert Model to OpenVINO IR + format <#convert-model-to-openvino-ir-format>`__ +- `Run OpenVINO model inference <#run-openvino-model-inference>`__ + + - `Select inference device <#select-inference-device>`__ + - `Run inference on image <#run-inference-on-image>`__ + - `Run inference on video <#run-inference-on-video>`__ + +- `Quantization <#quantization>`__ + + - `Prepare calibration dataset <#prepare-calibration-dataset>`__ + - `Run quantization <#run-quantization>`__ + + - `Compare model file size <#compare-model-file-size>`__ + + - `Compare inference time of the FP16 and INT8 + models <#compare-inference-time-of-the-fp16-and-int8-models>`__ + +- `Interactive demo <#interactive-demo>`__ + +Prerequisites +------------- + + + +.. code:: ipython3 + + from pathlib import Path + + + repo_dir = Path("Depth-Anything-V2") + + if not repo_dir.exists(): + !git clone https://huggingface.co/spaces/depth-anything/Depth-Anything-V2 + %cd Depth-Anything-V2 + + +.. parsed-literal:: + + Cloning into 'Depth-Anything-V2'... + remote: Enumerating objects: 154, done. + remote: Counting objects: 100% (150/150), done. + remote: Compressing objects: 100% (147/147), done. + remote: Total 154 (delta 43), reused 0 (delta 0), pack-reused 4 (from 1) + Receiving objects: 100% (154/154), 7.76 MiB | 13.62 MiB/s, done. + Resolving deltas: 100% (43/43), done. + Filtering content: 100% (5/5), 28.66 MiB | 24.37 MiB/s, done. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2 + + +.. code:: ipython3 + + import platform + + %pip install -q "openvino>=2024.2.0" "datasets>=2.14.6" "nncf>=2.11.0" "tqdm" + %pip install -q "typing-extensions>=4.9.0" eval-type-backport "gradio>=4.19" + %pip install -q -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu + + if platform.python_version_tuple()[1] in ["8", "9"]: + %pip install -q "gradio-imageslider<=0.0.17" "typing-extensions>=4.9.0" + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + + +Load and run PyTorch model +-------------------------- + + + +To be able run PyTorch model on CPU, we should disable xformers +attention optimizations first. + +.. code:: ipython3 + + attention_file_path = Path("./depth_anything_v2/dinov2_layers/attention.py") + orig_attention_path = attention_file_path.parent / ("orig_" + attention_file_path.name) + + if not orig_attention_path.exists(): + attention_file_path.rename(orig_attention_path) + + with orig_attention_path.open("r") as f: + data = f.read() + data = data.replace("XFORMERS_AVAILABLE = True", "XFORMERS_AVAILABLE = False") + with attention_file_path.open("w") as out_f: + out_f.write(data) + +Prepare input data +~~~~~~~~~~~~~~~~~~ + + + +.. code:: ipython3 + + from PIL import Image + + import requests + + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", + ) + + open("notebook_utils.py", "w").write(r.text) + from notebook_utils import download_file + + download_file( + "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/3f779fc1-c1b2-4dec-915a-64dae510a2bb", + "furseal.png", + ) + + Image.open("furseal.png").resize((600, 400)) + + + +.. parsed-literal:: + + furseal.png: 0%| | 0.00/2.55M [00:00 + + + + +.. image:: depth-anything-v2-with-output_files/depth-anything-v2-with-output_14_1.png + + +Convert Model to OpenVINO IR format +----------------------------------- + + + +OpenVINO supports PyTorch models via conversion to OpenVINO Intermediate +Representation (IR). OpenVINO model conversion API should be used for +these purposes. ``ov.convert_model`` function accepts original PyTorch +model instance and example input for tracing and returns ``ov.Model`` +representing this model in OpenVINO framework. Converted model can be +used for saving on disk using ``ov.save_model`` function or directly +loading on device using ``core.complie_model``. + +.. code:: ipython3 + + import openvino as ov + + OV_DEPTH_ANYTHING_PATH = Path(f"{model_id}.xml") + + if not OV_DEPTH_ANYTHING_PATH.exists(): + ov_model = ov.convert_model(model, example_input=torch.rand(1, 3, 518, 518), input=[1, 3, 518, 518]) + ov.save_model(ov_model, OV_DEPTH_ANYTHING_PATH) + + +.. parsed-literal:: + + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if npatch == N and w == h: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dpt.py:147: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) + + +Run OpenVINO model inference +---------------------------- + + + +Now, we are ready to run OpenVINO model + +Select inference device +~~~~~~~~~~~~~~~~~~~~~~~ + + + +For starting work, please select inference device from dropdown list. + +.. code:: ipython3 + + import ipywidgets as widgets + + core = ov.Core() + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value="AUTO", + description="Device:", + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +.. code:: ipython3 + + compiled_model = core.compile_model(OV_DEPTH_ANYTHING_PATH, device.value) + +Run inference on image +~~~~~~~~~~~~~~~~~~~~~~ + + + +For simplicity of usage, model authors provide helper functions for +preprocessing input image. The main conditions are that image size +should be divisible on 14 (size of vit patch) and normalized in [0, 1] +range. + +.. code:: ipython3 + + from depth_anything_v2.util.transform import Resize, NormalizeImage, PrepareForNet + from torchvision.transforms import Compose + + transform = Compose( + [ + Resize( + width=518, + height=518, + resize_target=False, + ensure_multiple_of=14, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + ] + ) + +.. code:: ipython3 + + h, w = raw_img.shape[:-1] + + image = cv2.cvtColor(raw_img, cv2.COLOR_BGR2RGB) / 255.0 + image = transform({"image": image})["image"] + image = torch.from_numpy(image).unsqueeze(0) + + res = compiled_model(image)[0] + +.. code:: ipython3 + + depth_color = get_depth_map(res[0], w, h) + +.. code:: ipython3 + + plt.imshow(depth_color[:, :, ::-1]) + + + + +.. parsed-literal:: + + + + + + +.. image:: depth-anything-v2-with-output_files/depth-anything-v2-with-output_24_1.png + + +Run inference on video +~~~~~~~~~~~~~~~~~~~~~~ + + + +.. code:: ipython3 + + download_file( + "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/Coco%20Walking%20in%20Berkeley.mp4", + "./Coco Walking in Berkeley.mp4", + ) + + VIDEO_FILE = "./Coco Walking in Berkeley.mp4" + # Number of seconds of input video to process. Set `NUM_SECONDS` to 0 to process + # the full video. + NUM_SECONDS = 4 + # Set `ADVANCE_FRAMES` to 1 to process every frame from the input video + # Set `ADVANCE_FRAMES` to 2 to process every second frame. This reduces + # the time it takes to process the video. + ADVANCE_FRAMES = 2 + # Set `SCALE_OUTPUT` to reduce the size of the result video + # If `SCALE_OUTPUT` is 0.5, the width and height of the result video + # will be half the width and height of the input video. + SCALE_OUTPUT = 0.5 + # The format to use for video encoding. The 'vp09` is slow, + # but it works on most systems. + # Try the `THEO` encoding if you have FFMPEG installed. + # FOURCC = cv2.VideoWriter_fourcc(*"THEO") + FOURCC = cv2.VideoWriter_fourcc(*"vp09") + + # Create Path objects for the input video and the result video. + output_directory = Path("output") + output_directory.mkdir(exist_ok=True) + result_video_path = output_directory / f"{Path(VIDEO_FILE).stem}_depth_anything.mp4" + + + +.. parsed-literal:: + + Coco Walking in Berkeley.mp4: 0%| | 0.00/877k [00:00 np.ndarray: + """ + Convert image_data from BGR to RGB + """ + return cv2.cvtColor(image_data, cv2.COLOR_BGR2RGB) + +.. code:: ipython3 + + import time + from IPython.display import ( + HTML, + FileLink, + Pretty, + ProgressBar, + Video, + clear_output, + display, + ) + + + def process_video(compiled_model, video_file, result_video_path): + # Initialize variables. + input_video_frame_nr = 0 + start_time = time.perf_counter() + total_inference_duration = 0 + + # Open the input video + cap = cv2.VideoCapture(str(video_file)) + + # Create a result video. + out_video = cv2.VideoWriter( + str(result_video_path), + FOURCC, + target_fps, + (target_frame_width * 2, target_frame_height), + ) + + num_frames = int(NUM_SECONDS * input_fps) + total_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) if num_frames == 0 else num_frames + progress_bar = ProgressBar(total=total_frames) + progress_bar.display() + + try: + while cap.isOpened(): + ret, image = cap.read() + if not ret: + cap.release() + break + + if input_video_frame_nr >= total_frames: + break + + h, w = image.shape[:-1] + input_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) / 255.0 + input_image = transform({"image": input_image})["image"] + # Reshape the image to network input shape NCHW. + input_image = np.expand_dims(input_image, 0) + + # Do inference. + inference_start_time = time.perf_counter() + result = compiled_model(input_image)[0] + inference_stop_time = time.perf_counter() + inference_duration = inference_stop_time - inference_start_time + total_inference_duration += inference_duration + + if input_video_frame_nr % (10 * ADVANCE_FRAMES) == 0: + clear_output(wait=True) + progress_bar.display() + # input_video_frame_nr // ADVANCE_FRAMES gives the number of + # Frames that have been processed by the network. + display( + Pretty( + f"Processed frame {input_video_frame_nr // ADVANCE_FRAMES}" + f"/{total_frames // ADVANCE_FRAMES}. " + f"Inference time per frame: {inference_duration:.2f} seconds " + f"({1/inference_duration:.2f} FPS)" + ) + ) + + # Transform the network result to a RGB image. + result_frame = to_rgb(convert_result_to_image(result)) + # Resize the image and the result to a target frame shape. + result_frame = cv2.resize(result_frame, (target_frame_width, target_frame_height)) + image = cv2.resize(image, (target_frame_width, target_frame_height)) + # Put the image and the result side by side. + stacked_frame = np.hstack((image, result_frame)) + # Save a frame to the video. + out_video.write(stacked_frame) + + input_video_frame_nr = input_video_frame_nr + ADVANCE_FRAMES + cap.set(1, input_video_frame_nr) + + progress_bar.progress = input_video_frame_nr + progress_bar.update() + + except KeyboardInterrupt: + print("Processing interrupted.") + finally: + clear_output() + processed_frames = num_frames // ADVANCE_FRAMES + out_video.release() + cap.release() + end_time = time.perf_counter() + duration = end_time - start_time + + print( + f"Processed {processed_frames} frames in {duration:.2f} seconds. " + f"Total FPS (including video processing): {processed_frames/duration:.2f}." + f"Inference FPS: {processed_frames/total_inference_duration:.2f} " + ) + print(f"Video saved to '{str(result_video_path)}'.") + return stacked_frame + +.. code:: ipython3 + + stacked_frame = process_video(compiled_model, VIDEO_FILE, result_video_path) + + +.. parsed-literal:: + + Processed 60 frames in 13.35 seconds. Total FPS (including video processing): 4.49.Inference FPS: 10.45 + Video saved to 'output/Coco Walking in Berkeley_depth_anything.mp4'. + + +.. code:: ipython3 + + def display_video(stacked_frame): + video = Video(result_video_path, width=800, embed=True) + if not result_video_path.exists(): + plt.imshow(stacked_frame) + raise ValueError("OpenCV was unable to write the video file. Showing one video frame.") + else: + print(f"Showing video saved at\n{result_video_path.resolve()}") + print("If you cannot see the video in your browser, please click on the " "following link to download the video ") + video_link = FileLink(result_video_path) + video_link.html_link_str = "%s" + display(HTML(video_link._repr_html_())) + display(video) + +.. code:: ipython3 + + display_video(stacked_frame) + + +.. parsed-literal:: + + Showing video saved at + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 + If you cannot see the video in your browser, please click on the following link to download the video + + + +.. raw:: html + + output/Coco Walking in Berkeley_depth_anything.mp4
+ + + +.. raw:: html + + + + +Quantization +------------ + + + +`NNCF `__ enables +post-training quantization by adding quantization layers into model +graph and then using a subset of the training dataset to initialize the +parameters of these additional quantization layers. Quantized operations +are executed in ``INT8`` instead of ``FP32``/``FP16`` making model +inference faster. + +The optimization process contains the following steps: + +1. Create a calibration dataset for quantization. +2. Run ``nncf.quantize()`` to obtain quantized model. +3. Save the ``INT8`` model using ``openvino.save_model()`` function. + +Please select below whether you would like to run quantization to +improve model inference speed. + +.. code:: ipython3 + + to_quantize = widgets.Checkbox( + value=True, + description="Quantization", + disabled=False, + ) + + to_quantize + + + + +.. parsed-literal:: + + Checkbox(value=True, description='Quantization') + + + +.. code:: ipython3 + + # Fetch `skip_kernel_extension` module + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/skip_kernel_extension.py", + ) + open("skip_kernel_extension.py", "w").write(r.text) + + OV_DEPTH_ANYTHING_INT8_PATH = Path(f"{model_id}_int8.xml") + + %load_ext skip_kernel_extension + +Prepare calibration dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +We use a portion of +`Nahrawy/VIDIT-Depth-ControlNet `__ +dataset from Hugging Face as calibration data. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + import datasets + + if not OV_DEPTH_ANYTHING_INT8_PATH.exists(): + subset_size = 300 + calibration_data = [] + dataset = datasets.load_dataset("Nahrawy/VIDIT-Depth-ControlNet", split="train", streaming=True).shuffle(seed=42).take(subset_size) + for batch in dataset: + image = np.array(batch["image"])[...,:3] + image = image / 255.0 + image = transform({'image': image})['image'] + image = np.expand_dims(image, 0) + calibration_data.append(image) + + + +.. parsed-literal:: + + Resolving data files: 0%| | 0/42 [00:00 **NOTE**: Quantization is time and memory consuming operation. Running +quantization code below may take some time. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + import nncf + + if not OV_DEPTH_ANYTHING_INT8_PATH.exists(): + model = core.read_model(OV_DEPTH_ANYTHING_PATH) + quantized_model = nncf.quantize( + model=model, + subset_size=subset_size, + model_type=nncf.ModelType.TRANSFORMER, + calibration_dataset=nncf.Dataset(calibration_data), + ) + ov.save_model(quantized_model, OV_DEPTH_ANYTHING_INT8_PATH) + + +.. parsed-literal:: + + INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + + +.. parsed-literal:: + + 2024-07-01 23:58:14.192557: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-07-01 23:58:14.226081: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-07-01 23:58:14.785892: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + + +.. parsed-literal:: + + Output() + + + + + + + + + + + + + + + + + + +.. parsed-literal:: + + Output() + + + + + + + + + + + + + + + + + +.. parsed-literal:: + + INFO:nncf:36 ignored nodes were found by name in the NNCFGraph + INFO:nncf:48 ignored nodes were found by name in the NNCFGraph + + + +.. parsed-literal:: + + Output() + + + + + + + + + + + + + + + + + + +.. parsed-literal:: + + Output() + + + + + + + + + + + + + + + + + +Let us check predictions with the quantized model using the same input +data. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + def visualize_results(orig_img:Image.Image, optimized_img:Image.Image): + """ + Helper function for results visualization + + Parameters: + orig_img (Image.Image): generated image using FP16 model + optimized_img (Image.Image): generated image using quantized model + Returns: + fig (matplotlib.pyplot.Figure): matplotlib generated figure contains drawing result + """ + orig_title = "FP16 model" + control_title = "INT8 model" + figsize = (20, 20) + fig, axs = plt.subplots(1, 2, figsize=figsize, sharex='all', sharey='all') + list_axes = list(axs.flat) + for a in list_axes: + a.set_xticklabels([]) + a.set_yticklabels([]) + a.get_xaxis().set_visible(False) + a.get_yaxis().set_visible(False) + a.grid(False) + list_axes[0].imshow(np.array(orig_img)) + list_axes[1].imshow(np.array(optimized_img)) + list_axes[0].set_title(orig_title, fontsize=15) + list_axes[1].set_title(control_title, fontsize=15) + + fig.subplots_adjust(wspace=0.01, hspace=0.01) + fig.tight_layout() + return fig + +.. code:: ipython3 + + %%skip not $to_quantize.value + + image = cv2.cvtColor(cv2.imread('furseal.png'), cv2.COLOR_BGR2RGB) / 255.0 + image = transform({'image': image})['image'] + image = torch.from_numpy(image).unsqueeze(0) + + int8_compiled_model = core.compile_model(OV_DEPTH_ANYTHING_INT8_PATH, device.value) + int8_res = int8_compiled_model(image)[0] + int8_depth_color = get_depth_map(int8_res[0], w, h) + +.. code:: ipython3 + + %%skip not $to_quantize.value + + visualize_results(depth_color[:, :, ::-1], int8_depth_color[:, :, ::-1]) + + + +.. image:: depth-anything-v2-with-output_files/depth-anything-v2-with-output_43_0.png + + +.. code:: ipython3 + + %%skip not $to_quantize.value + + int8_result_video_path = output_directory / f"{Path(VIDEO_FILE).stem}_depth_anything_int8.mp4" + stacked_frame = process_video(int8_compiled_model, VIDEO_FILE, int8_result_video_path) + display_video(stacked_frame) + + +.. parsed-literal:: + + Processed 60 frames in 12.73 seconds. Total FPS (including video processing): 4.71.Inference FPS: 12.94 + Video saved to 'output/Coco Walking in Berkeley_depth_anything_int8.mp4'. + Showing video saved at + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 + If you cannot see the video in your browser, please click on the following link to download the video + + + +.. raw:: html + + output/Coco Walking in Berkeley_depth_anything.mp4
+ + + +.. raw:: html + + + + +Compare model file size +^^^^^^^^^^^^^^^^^^^^^^^ + + + +.. code:: ipython3 + + %%skip not $to_quantize.value + + fp16_ir_model_size = OV_DEPTH_ANYTHING_PATH.with_suffix(".bin").stat().st_size / 2**20 + quantized_model_size = OV_DEPTH_ANYTHING_INT8_PATH.with_suffix(".bin").stat().st_size / 2**20 + + print(f"FP16 model size: {fp16_ir_model_size:.2f} MB") + print(f"INT8 model size: {quantized_model_size:.2f} MB") + print(f"Model compression rate: {fp16_ir_model_size / quantized_model_size:.3f}") + + +.. parsed-literal:: + + FP16 model size: 47.11 MB + INT8 model size: 24.41 MB + Model compression rate: 1.930 + + +Compare inference time of the FP16 and INT8 models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +To measure the inference performance of OpenVINO FP16 and INT8 models, +use `Benchmark +Tool `__. + + **NOTE**: For the most accurate performance estimation, it is + recommended to run ``benchmark_app`` in a terminal/command prompt + after closing other applications. + +.. code:: ipython3 + + import re + + + def get_fps(benchmark_output: str): + parsed_output = [line for line in benchmark_output if "Throughput:" in line] + fps = re.findall(r"\d+\.\d+", parsed_output[0])[0] + return fps + + + if OV_DEPTH_ANYTHING_INT8_PATH.exists(): + benchmark_output = !benchmark_app -m $OV_DEPTH_ANYTHING_PATH -d $device.value -api async + original_fps = get_fps(benchmark_output) + print(f"FP16 Throughput: {original_fps} FPS") + + benchmark_output = !benchmark_app -m $OV_DEPTH_ANYTHING_INT8_PATH -d $device.value -api async + optimized_fps = get_fps(benchmark_output) + print(f"INT8 Throughput: {optimized_fps} FPS") + print(f"Speed-up: {float(optimized_fps) / float(original_fps):.2f}") + + +.. parsed-literal:: + + FP16 Throughput: 10.63 FPS + INT8 Throughput: 14.30 FPS + Speed-up: 1.35 + + +Interactive demo +---------------- + + + +You can apply model on own images. You can move the slider on the +resulting image to switch between the original image and the depth map +view. + +Please select below whether you would like to use the quantized model to +launch the interactive demo. + +.. code:: ipython3 + + quantized_model_present = OV_DEPTH_ANYTHING_INT8_PATH.exists() + + use_quantized_model = widgets.Checkbox( + value=True if quantized_model_present else False, + description="Use quantized model", + disabled=False, + ) + + use_quantized_model + + + + +.. parsed-literal:: + + Checkbox(value=True, description='Use quantized model') + + + +.. code:: ipython3 + + import gradio as gr + import cv2 + import numpy as np + import os + import tempfile + from gradio_imageslider import ImageSlider + + css = """ + #img-display-container { + max-height: 100vh; + } + #img-display-input { + max-height: 80vh; + } + #img-display-output { + max-height: 80vh; + } + """ + + + title = "# Depth Anything with OpenVINO" + + if use_quantized_model.value: + compiled_model = core.compile_model(OV_DEPTH_ANYTHING_INT8_PATH, device.value) + + + def predict_depth(model, image): + return model(image)[0] + + + with gr.Blocks(css=css) as demo: + gr.Markdown(title) + gr.Markdown("### Depth Prediction demo") + gr.Markdown("You can slide the output to compare the depth prediction with input image") + + with gr.Row(): + input_image = gr.Image(label="Input Image", type="numpy", elem_id="img-display-input") + depth_image_slider = ImageSlider(label="Depth Map with Slider View", elem_id="img-display-output", position=0) + raw_file = gr.File(label="16-bit raw depth (can be considered as disparity)") + submit = gr.Button("Submit") + + def on_submit(image): + original_image = image.copy() + + h, w = image.shape[:2] + + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) / 255.0 + image = transform({"image": image})["image"] + image = np.expand_dims(image, 0) + + depth = predict_depth(compiled_model, image) + depth = cv2.resize(depth[0], (w, h), interpolation=cv2.INTER_LINEAR) + + raw_depth = Image.fromarray(depth.astype("uint16")) + tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False) + raw_depth.save(tmp.name) + + depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0 + depth = depth.astype(np.uint8) + colored_depth = cv2.applyColorMap(depth, cv2.COLORMAP_INFERNO)[:, :, ::-1] + + return [(original_image, colored_depth), tmp.name] + + submit.click(on_submit, inputs=[input_image], outputs=[depth_image_slider, raw_file]) + + example_files = os.listdir("assets/examples") + example_files.sort() + example_files = [os.path.join("assets/examples", filename) for filename in example_files] + examples = gr.Examples( + examples=example_files, + inputs=[input_image], + outputs=[depth_image_slider, raw_file], + fn=on_submit, + cache_examples=False, + ) + + + if __name__ == "__main__": + try: + demo.queue().launch(debug=False) + except Exception: + demo.queue().launch(share=True, debug=False) + # if you are launching remotely, specify server_name and server_port + # demo.launch(server_name='your server name', server_port='server port in int') + # Read more in the docs: https://gradio.app/docs/ + + +.. parsed-literal:: + + Running on local URL: http://127.0.0.1:7860 + + To create a public link, set `share=True` in `launch()`. + + + + + + + diff --git a/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_14_1.png b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_14_1.png new file mode 100644 index 00000000000000..a4e01af7f912e6 --- /dev/null +++ b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_14_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4d703d3b914455b7a7d10e2b43fe3b0a9ab3c9e4aa9336490d30b3fbe24015 +size 88497 diff --git a/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_24_1.png b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_24_1.png new file mode 100644 index 00000000000000..9c25e30b532a13 --- /dev/null +++ b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_24_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:833f02eec49d4529fb1c46026ed9743fc23ff1343026bfb7c44c50bc32f19a1b +size 85083 diff --git a/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_43_0.png b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_43_0.png new file mode 100644 index 00000000000000..9f8084b50c61e5 --- /dev/null +++ b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_43_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12eb576b73adf6b36bc6a89316bf77368af7311d8262215c7dea53186393c4fe +size 629130 diff --git a/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_8_1.png b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_8_1.png new file mode 100644 index 00000000000000..096d76fe63cc45 --- /dev/null +++ b/docs/notebooks/depth-anything-v2-with-output_files/depth-anything-v2-with-output_8_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f279ac10beacc7f27d1af73f7917d060c4c33db80b09d78c85dc864bd4bb3a7 +size 424224 diff --git a/docs/notebooks/depth-anything-with-output.rst b/docs/notebooks/depth-anything-with-output.rst index f9d03cb4fcdeaf..50d946f009543c 100644 --- a/docs/notebooks/depth-anything-with-output.rst +++ b/docs/notebooks/depth-anything-with-output.rst @@ -19,8 +19,8 @@ using OpenVINO. An additional part demonstrates how to run quantization with `NNCF `__ to speed up the model. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Load and run PyTorch model <#load-and-run-pytorch-model>`__ @@ -75,13 +75,13 @@ Prerequisites .. parsed-literal:: Cloning into 'Depth-Anything'... - remote: Enumerating objects: 421, done. - remote: Counting objects: 100% (144/144), done. - remote: Compressing objects: 100% (105/105), done. - remote: Total 421 (delta 101), reused 43 (delta 39), pack-reused 277 - Receiving objects: 100% (421/421), 237.89 MiB | 26.31 MiB/s, done. - Resolving deltas: 100% (144/144), done. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything + remote: Enumerating objects: 436, done. + remote: Counting objects: 100% (156/156), done. + remote: Compressing objects: 100% (115/115), done. + remote: Total 436 (delta 113), reused 41 (delta 41), pack-reused 280 + Receiving objects: 100% (436/436), 237.89 MiB | 24.04 MiB/s, done. + Resolving deltas: 100% (156/156), done. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. WARNING: typer 0.12.3 does not provide the extra 'all' @@ -273,13 +273,13 @@ loading on device using ``core.complie_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/vision_transformer.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/vision_transformer.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if npatch == N and w == h: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/depth_anything/dpt.py:133: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/depth_anything/dpt.py:133: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) @@ -571,7 +571,7 @@ Run inference on video .. parsed-literal:: - Processed 60 frames in 13.62 seconds. Total FPS (including video processing): 4.41.Inference FPS: 10.01 + Processed 60 frames in 13.25 seconds. Total FPS (including video processing): 4.53.Inference FPS: 10.44 Video saved to 'output/Coco Walking in Berkeley_depth_anything.mp4'. @@ -598,7 +598,7 @@ Run inference on video .. parsed-literal:: Showing video saved at - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4 If you cannot see the video in your browser, please click on the following link to download the video @@ -735,10 +735,10 @@ quantization code below may take some time. .. parsed-literal:: - 2024-06-06 00:01:26.132845: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-06-06 00:01:26.165931: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-07-02 00:07:04.850517: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-07-02 00:07:04.883202: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-06-06 00:01:26.726838: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-07-02 00:07:05.467923: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -748,17 +748,17 @@ quantization code below may take some time. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -769,17 +769,17 @@ quantization code below may take some time. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -796,17 +796,17 @@ quantization code below may take some time. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -817,17 +817,17 @@ quantization code below may take some time. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -902,10 +902,10 @@ data. .. parsed-literal:: - Processed 60 frames in 12.67 seconds. Total FPS (including video processing): 4.74.Inference FPS: 12.82 + Processed 60 frames in 13.18 seconds. Total FPS (including video processing): 4.55.Inference FPS: 11.58 Video saved to 'output/Coco Walking in Berkeley_depth_anything_int8.mp4'. Showing video saved at - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4 If you cannot see the video in your browser, please click on the following link to download the video @@ -985,9 +985,9 @@ Tool `__ dataset as examples for object detection and instance segmentation respectively. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg index 009298597f3ff1..ab0c1b78ebfca1 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eb6a3c05535408d011ad8aa46258978d309afda0fe57ab8c97ea81078386469c -size 58366 +oid sha256:e8d7d90b84bd0b73815bc09ba20bc70af0fcc8a056b3d00db1b2b0914d07f8f6 +size 57923 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png index 84292f2d1d35ad..84b2911e640610 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84897b3dce254b9f229b11ab01345853e5a8397bb232c2dbf9e9062f6a3d3857 -size 509075 +oid sha256:f16ba7ee95192141747aa9eb5f7bcbde67fae8f0c0d84ecc056be1c896cd6659 +size 509091 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg index 42770b2466d915..7b80e48c89ae43 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c558219ed9c6dae7fa76cbd8791cdb30ea5ee7d88af767ff82b989067e11ff6b -size 54727 +oid sha256:10b9ef56b21a181e5a72b2ad211355d22211d9e88458bef06738ab527d64f4c1 +size 56024 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png index 946beed11b4517..f5d36c77d54d6e 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a3202c1e5f4b8914b4290f1189e147f965b87d6556b8ed8d59d12ebf7812db9a -size 457983 +oid sha256:2ba18e298e6381279761c526d16d39d57daaeb1804a72983b4c969c176cc0ed0 +size 457758 diff --git a/docs/notebooks/distil-whisper-asr-with-output.rst b/docs/notebooks/distil-whisper-asr-with-output.rst index 1e27e7ebf6a893..72c69751103cda 100644 --- a/docs/notebooks/distil-whisper-asr-with-output.rst +++ b/docs/notebooks/distil-whisper-asr-with-output.rst @@ -36,8 +36,8 @@ convert the model to OpenVINO™ IR format. To further improve OpenVINO Distil-Whisper model performance ``INT8`` post-training quantization from `NNCF `__ is applied. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Load PyTorch model <#load-pytorch-model>`__ @@ -186,7 +186,7 @@ by Hugging Face datasets implementation. return input_features - dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation", trust_remote_code=True) sample = dataset[0] input_features = extract_input_features(sample) @@ -475,7 +475,7 @@ seconds is optimal. To activate batching, pass the argument batch_size. .. code:: ipython3 - dataset = load_dataset("distil-whisper/librispeech_long", "clean", split="validation") + dataset = load_dataset("distil-whisper/librispeech_long", "clean", split="validation", trust_remote_code=True) sample_long = dataset[0] @@ -687,7 +687,7 @@ improves quantization quality. apply_caching=True) try: - calibration_dataset = load_dataset("librispeech_asr", "clean", split="validation", streaming=True) + calibration_dataset = load_dataset("openslr/librispeech_asr", "clean", split="validation", streaming=True, trust_remote_code=True) for sample in tqdm(islice(calibration_dataset, calibration_dataset_size), desc="Collecting calibration data", total=calibration_dataset_size): input_features = extract_input_features(sample) @@ -833,7 +833,7 @@ models. %%skip not $to_quantize.value dataset = load_dataset( - "hf-internal-testing/librispeech_asr_dummy", "clean", split="validation" + "hf-internal-testing/librispeech_asr_dummy", "clean", split="validation", trust_remote_code=True ) sample = dataset[0] input_features = extract_input_features(sample) @@ -944,7 +944,7 @@ decoder-with-past model forwards, and for the whole model inference too. mean_decoder_with_time_infer_time = sum(decoder_with_past_infer_times) return word_accuracy, (mean_whole_infer_time, mean_encoder_infer_time, mean_decoder_with_time_infer_time) - test_dataset = load_dataset("librispeech_asr", "clean", split="test", streaming=True) + test_dataset = load_dataset("openslr/librispeech_asr", "clean", split="test", streaming=True, trust_remote_code=True) test_dataset = test_dataset.shuffle(seed=42).take(TEST_DATASET_SIZE) test_samples = [sample for sample in test_dataset] diff --git a/docs/notebooks/distilbert-sequence-classification-with-output.rst b/docs/notebooks/distilbert-sequence-classification-with-output.rst index 545a5b3429f262..a345deeabbe402 100644 --- a/docs/notebooks/distilbert-sequence-classification-with-output.rst +++ b/docs/notebooks/distilbert-sequence-classification-with-output.rst @@ -7,8 +7,8 @@ identify, extract, quantify, and study affective states and subjective information. This notebook demonstrates how to convert and run a sequence classification model using OpenVINO. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Imports <#imports>`__ - `Initializing the Model <#initializing-the-model>`__ @@ -36,31 +36,31 @@ Imports .. parsed-literal:: Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu - Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.3.0.dev20240605) - Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.41.2) - Requirement already satisfied: torch>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.3.1+cpu) - Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.4) - Requirement already satisfied: numpy<2.0.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) - Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.0) - Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.14.0) - Requirement already satisfied: huggingface-hub<1.0,>=0.23.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.23.3) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.1) - Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2024.5.15) - Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.32.3) - Requirement already satisfied: tokenizers<0.20,>=0.19 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.19.1) - Requirement already satisfied: safetensors>=0.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.3) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (4.12.1) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (1.12.1) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1.4) - Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (2024.3.1) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1) (2.1.5) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.3.2) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.7) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.2.1) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2024.6.2) - Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1) (1.3.0) + Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.3.0.dev20240627) + Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.42.3) + Requirement already satisfied: torch>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.3.1+cpu) + Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.4) + Requirement already satisfied: numpy<2.0.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.15.4) + Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.23.4) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.1) + Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2024.5.15) + Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.32.3) + Requirement already satisfied: safetensors>=0.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.3) + Requirement already satisfied: tokenizers<0.20,>=0.19 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.19.1) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (1.12.1) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1.4) + Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (2024.5.0) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1) (2.1.5) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.3.2) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.7) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.2.2) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2024.6.2) + Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1) (1.3.0) Note: you may need to restart the kernel to use updated packages. @@ -143,9 +143,9 @@ optimal execution on end-point target devices. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4565: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:231: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:230: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mask, torch.tensor(torch.finfo(scores.dtype).min) diff --git a/docs/notebooks/dolly-2-instruction-following-with-output.rst b/docs/notebooks/dolly-2-instruction-following-with-output.rst index 4344393ba488c5..1863874479b765 100644 --- a/docs/notebooks/dolly-2-instruction-following-with-output.rst +++ b/docs/notebooks/dolly-2-instruction-following-with-output.rst @@ -81,8 +81,8 @@ dataset can be found in `Databricks blog post `__ and `repo `__ -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Convert model using Optimum-CLI @@ -124,7 +124,10 @@ documentation `__. .. code:: ipython3 - %pip install -Uq pip + import os + + os.environ["GIT_CLONE_PROTECTION_ACTIVE"] = "false" + %pip uninstall -q -y optimum optimum-intel %pip install --pre -Uq openvino openvino-tokenizers[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly %pip install -q "diffusers>=0.16.1" "transformers>=4.33.0" "torch>=2.1" "nncf>=2.10.0" onnx "gradio>=4.19" --extra-index-url https://download.pytorch.org/whl/cpu @@ -136,7 +139,7 @@ Convert model using Optimum-CLI tool `Optimum Intel `__ is -the interface between the +the interface between the `Transformers `__ and `Diffusers `__ libraries and OpenVINO to accelerate end-to-end pipelines on Intel architectures. @@ -190,12 +193,13 @@ to make it `symmetric `__ you can add ``--sym``. -For INT4 quantization you can also specify the following arguments : - -The ``--group-size`` parameter will define the group size to use for -quantization, -1 it will results in per-column quantization. - The -``--ratio`` parameter controls the ratio between 4-bit and 8-bit -quantization. If set to 0.9, it means that 90% of the layers will be -quantized to int4 while 10% will be quantized to int8. +For INT4 quantization you can also specify the following arguments : + +- The ``--group-size`` parameter will define the group size to use for + quantization, -1 it will results in per-column quantization. +- The ``--ratio`` parameter controls the ratio between 4-bit and 8-bit + quantization. If set to 0.9, it means that 90% of the layers will be + quantized to int4 while 10% will be quantized to int8. Smaller group_size and ratio values usually improve accuracy at the sacrifice of the model size and inference latency. @@ -207,7 +211,7 @@ sacrifice of the model size and inference latency. from IPython.display import Markdown, display import ipywidgets as widgets - + prepare_int4_model = widgets.Checkbox( value=True, description="Prepare INT4 model", @@ -223,7 +227,7 @@ sacrifice of the model size and inference latency. description="Prepare FP16 model", disabled=False, ) - + display(prepare_int4_model) display(prepare_int8_model) display(prepare_fp16_model) @@ -250,15 +254,15 @@ sacrifice of the model size and inference latency. .. code:: ipython3 from pathlib import Path - + model_id = "databricks/dolly-v2-3b" model_path = Path("dolly-v2-3b") - + fp16_model_dir = model_path / "FP16" int8_model_dir = model_path / "INT8_compressed_weights" int4_model_dir = model_path / "INT4_compressed_weights" - - + + def convert_to_fp16(): if (fp16_model_dir / "openvino_model.xml").exists(): return @@ -268,8 +272,8 @@ sacrifice of the model size and inference latency. display(Markdown("**Export command:**")) display(Markdown(f"`{export_command}`")) ! $export_command - - + + def convert_to_int8(): if (int8_model_dir / "openvino_model.xml").exists(): return @@ -279,8 +283,8 @@ sacrifice of the model size and inference latency. display(Markdown("**Export command:**")) display(Markdown(f"`{export_command}`")) ! $export_command - - + + def convert_to_int4(): if (int4_model_dir / "openvino_model.xml").exists(): return @@ -290,8 +294,8 @@ sacrifice of the model size and inference latency. display(Markdown("**Export command:**")) display(Markdown(f"`{export_command}`")) ! $export_command - - + + if prepare_fp16_model.value: convert_to_fp16() if prepare_int8_model.value: @@ -304,7 +308,7 @@ sacrifice of the model size and inference latency. fp16_weights = fp16_model_dir / "openvino_model.bin" int8_weights = int8_model_dir / "openvino_model.bin" int4_weights = int4_model_dir / "openvino_model.bin" - + if fp16_weights.exists(): print(f"Size of FP16 model is {fp16_weights.stat().st_size / 1024 / 1024:.2f} MB") for precision, compressed_weights in zip([8, 4], [int8_weights, int4_weights]): @@ -335,14 +339,14 @@ select device from dropdown list for running inference using OpenVINO available_models.append("INT8") if fp16_model_dir.exists(): available_models.append("FP16") - + model_to_run = widgets.Dropdown( options=available_models, value=available_models[0], description="Model to run:", disabled=False, ) - + model_to_run @@ -358,16 +362,16 @@ select device from dropdown list for running inference using OpenVINO import ipywidgets as widgets import openvino as ov - + core = ov.Core() - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value="CPU", description="Device:", disabled=False, ) - + device @@ -420,7 +424,7 @@ guide `__ from pathlib import Path from transformers import AutoTokenizer from optimum.intel.openvino import OVModelForCausalLM - + if model_to_run.value == "INT4": model_dir = int4_model_dir elif model_to_run.value == "INT8": @@ -428,13 +432,13 @@ guide `__ else: model_dir = fp16_model_dir print(f"Loading model from {model_dir}") - + tokenizer = AutoTokenizer.from_pretrained(model_dir) - + current_device = device.value - + ov_config = {"PERFORMANCE_HINT": "LATENCY", "NUM_STREAMS": "1", "CACHE_DIR": ""} - + ov_model = OVModelForCausalLM.from_pretrained(model_dir, device=current_device, ov_config=ov_config) @@ -617,14 +621,14 @@ into model with providing additional context. RESPONSE_KEY = "### Response:" END_KEY = "### End" INTRO_BLURB = "Below is an instruction that describes a task. Write a response that appropriately completes the request." - + # This is the prompt that is used for generating responses using an already trained model. It ends with the response # key, where the job of the model is to provide the completion that follows it (i.e. the response itself). PROMPT_FOR_GENERATION_FORMAT = """{intro} - + {instruction_key} {instruction} - + {response_key} """.format( intro=INTRO_BLURB, @@ -646,17 +650,17 @@ the code below find its id for using it as generation stop-criteria. def get_special_token_id(tokenizer: AutoTokenizer, key: str) -> int: """ Gets the token ID for a given string that has been added to the tokenizer as a special token. - + When training, we configure the tokenizer so that the sequences like "### Instruction:" and "### End" are treated specially and converted to a single, new token. This retrieves the token ID each of these keys map to. - + Args: tokenizer (PreTrainedTokenizer): the tokenizer key (str): the key to convert to a single token - + Raises: RuntimeError: if more than one ID was generated - + Returns: int: the token ID for the given key """ @@ -664,13 +668,13 @@ the code below find its id for using it as generation stop-criteria. if len(token_ids) > 1: raise ValueError(f"Expected only a single token for '{key}' but found {token_ids}") return token_ids[0] - - + + tokenizer_response_key = next( (token for token in tokenizer.additional_special_tokens if token.startswith(RESPONSE_KEY)), None, ) - + end_key_token_id = None if tokenizer_response_key: try: @@ -700,7 +704,7 @@ parameter and returns model response. ): """ Text generation function - + Parameters: user_text (str): User-provided instruction for a generation. top_p (float): Nucleus sampling. If set to < 1, only the smallest set of most probable tokens with probabilities that add up to top_p or higher are kept for a generation. @@ -712,13 +716,13 @@ parameter and returns model response. model_output (str) - model-generated text perf_text (str) - updated perf text filed content """ - + # Prepare input prompt according to model expected template prompt_text = PROMPT_FOR_GENERATION_FORMAT.format(instruction=user_text) - + # Tokenize the user text. model_inputs = tokenizer(prompt_text, return_tensors="pt") - + # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread. streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) @@ -734,7 +738,7 @@ parameter and returns model response. ) t = Thread(target=ov_model.generate, kwargs=generate_kwargs) t.start() - + # Pull the generated text from the streamer, and update the model output. model_output = "" per_token_time = [] @@ -768,14 +772,14 @@ elements. ): """ Helper function for performance estimation - + Parameters: current_time (float): This step time in seconds. current_perf_text (str): Current content of performance UI field. new_gen_text (str): New generated text. per_token_time (List[float]): history of performance from previous steps. num_tokens (int): Total number of generated tokens. - + Returns: update for performance text field update for a total number of tokens @@ -790,27 +794,27 @@ elements. num_tokens, ) return current_perf_text, num_tokens - - + + def reset_textbox(instruction: str, response: str, perf: str): """ Helper function for resetting content of all text fields - + Parameters: instruction (str): Content of user instruction field. response (str): Content of model response field. perf (str): Content of performance info filed - + Returns: empty string for each placeholder """ return "", "", "" - - + + def select_device(device_str: str, current_text: str = "", progress: gr.Progress = gr.Progress()): """ Helper function for uploading model on the device. - + Parameters: device_str (str): Device name. current_text (str): Current content of user instruction field (used only for backup purposes, temporally replacing it on the progress bar during model loading). @@ -821,7 +825,7 @@ elements. if device_str != ov_model._device: ov_model.request = None ov_model._device = device_str - + for i in progress.tqdm(range(1), desc=f"Model loading on {device_str}"): ov_model.compile() return current_text @@ -852,7 +856,7 @@ generation parameters: .. code:: ipython3 available_devices = ov.Core().available_devices + ["AUTO"] - + examples = [ "Give me recipe for pizza with pineapple", "Write me a tweet about new OpenVINO release", @@ -865,13 +869,13 @@ generation parameters: "Write instructions on how to become a good AI engineer", "Write a love letter to my best friend", ] - + with gr.Blocks() as demo: gr.Markdown( "# Instruction following using Databricks Dolly 2.0 and OpenVINO.\n" "Provide insturction which describes a task below or select among predefined examples and model writes response that performs requested task." ) - + with gr.Row(): with gr.Column(scale=4): user_text = gr.Textbox( @@ -918,7 +922,7 @@ generation parameters: interactive=True, label="Temperature", ) - + user_text.submit( run_generation, [user_text, top_p, temperature, top_k, max_new_tokens, performance], @@ -936,13 +940,13 @@ generation parameters: [user_text, model_output, performance], ) device.change(select_device, [device, user_text], [user_text]) - + if __name__ == "__main__": try: demo.queue().launch(debug=False, height=800) except Exception: demo.queue().launch(debug=False, share=True, height=800) - + # If you are launching remotely, specify server_name and server_port # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` # To learn more please refer to the Gradio docs: https://gradio.app/docs/ diff --git a/docs/notebooks/dynamicrafter-animating-images-with-output.rst b/docs/notebooks/dynamicrafter-animating-images-with-output.rst index 5f431841999aa8..5504281d547091 100644 --- a/docs/notebooks/dynamicrafter-animating-images-with-output.rst +++ b/docs/notebooks/dynamicrafter-animating-images-with-output.rst @@ -98,8 +98,8 @@ as higher conformity to the input image. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Load the original model <#load-the-original-model>`__ @@ -123,14 +123,16 @@ Prerequisites .. code:: ipython3 - %pip install -Uq pip - %pip install --pre -Uq openvino --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + %pip install "openvino>=2024.2.0" %pip install -q "gradio>=4.19" omegaconf decord einops pytorch_lightning kornia open_clip_torch transformers av opencv-python torch --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: - Note: you may need to restart the kernel to use updated packages. + Requirement already satisfied: openvino>=2024.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.3.0.dev20240627) + Requirement already satisfied: numpy<2.0.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2024.2.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2024.2.0) (2024.1.0) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2024.2.0) (24.1) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -139,26 +141,31 @@ Prerequisites import sys from pathlib import Path - - + + dynamicrafter_path = Path("dynamicrafter") - + if not dynamicrafter_path.exists(): dynamicrafter_path.mkdir(parents=True, exist_ok=True) !git clone https://github.com/Doubiiu/DynamiCrafter.git dynamicrafter - + %cd dynamicrafter + !git checkout 26e665cd6c174234238d2ded661e2e56f875d360 -q # to avoid breaking changes + %cd .. + sys.path.append(str(dynamicrafter_path)) .. parsed-literal:: Cloning into 'dynamicrafter'... - remote: Enumerating objects: 294, done. - remote: Counting objects: 100% (131/131), done. - remote: Compressing objects: 100% (75/75), done. - remote: Total 294 (delta 79), reused 76 (delta 56), pack-reused 163 - Receiving objects: 100% (294/294), 72.40 MiB | 26.63 MiB/s, done. - Resolving deltas: 100% (94/94), done. + remote: Enumerating objects: 320, done. + remote: Counting objects: 100% (157/157), done. + remote: Compressing objects: 100% (93/93), done. + remote: Total 320 (delta 95), reused 88 (delta 64), pack-reused 163 + Receiving objects: 100% (320/320), 72.40 MiB | 21.86 MiB/s, done. + Resolving deltas: 100% (110/110), done. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images Load and run the original pipeline @@ -173,14 +180,14 @@ We will use model for 256x256 resolution as example. Also, models for .. code:: ipython3 import os - + from huggingface_hub import hf_hub_download from omegaconf import OmegaConf - + from dynamicrafter.scripts.evaluation.funcs import load_model_checkpoint from dynamicrafter.utils.utils import instantiate_from_config - - + + def download_model(): REPO_ID = "Doubiiu/DynamiCrafter" if not os.path.exists("./checkpoints/dynamicrafter_256_v1/"): @@ -188,7 +195,7 @@ We will use model for 256x256 resolution as example. Also, models for local_file = os.path.join("./checkpoints/dynamicrafter_256_v1/model.ckpt") if not os.path.exists(local_file): hf_hub_download(repo_id=REPO_ID, filename="model.ckpt", local_dir="./checkpoints/dynamicrafter_256_v1/", local_dir_use_symlinks=False) - + ckpt_path = "checkpoints/dynamicrafter_256_v1/model.ckpt" config_file = "dynamicrafter/configs/inference_256_v1.0.yaml" config = OmegaConf.load(config_file) @@ -197,16 +204,16 @@ We will use model for 256x256 resolution as example. Also, models for model = instantiate_from_config(model_config) model = load_model_checkpoint(model, ckpt_path) model.eval() - + return model - - + + model = download_model() .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`. For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder. warnings.warn( @@ -236,10 +243,10 @@ file. .. code:: ipython3 import torch - + import openvino as ov - - + + def convert(model: torch.nn.Module, xml_path: str, example_input, input_shape=None): xml_path = Path(xml_path) if not xml_path.exists(): @@ -250,7 +257,7 @@ file. else: converted_model = ov.convert_model(model, example_input=example_input, input=input_shape) ov.save_model(converted_model, xml_path, compress_to_fp16=False) - + # cleanup memory torch._C._jit_clear_class_registry() torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() @@ -274,21 +281,21 @@ Convert CLIP text encoder .. code:: ipython3 from dynamicrafter.lvdm.modules.encoders.condition import FrozenOpenCLIPEmbedder - - + + COND_STAGE_MODEL_OV_PATH = Path("models/cond_stage_model.xml") - - + + class FrozenOpenCLIPEmbedderWrapper(FrozenOpenCLIPEmbedder): - + def forward(self, tokens): z = self.encode_with_transformer(tokens.to(self.device)) return z - - + + cond_stage_model = FrozenOpenCLIPEmbedderWrapper(device="cpu") - - + + convert( cond_stage_model, COND_STAGE_MODEL_OV_PATH, @@ -305,53 +312,53 @@ resolutions. .. code:: ipython3 EMBEDDER_OV_PATH = Path("models/embedder_ir.xml") - - + + dummy_input = torch.rand([1, 3, 767, 767], dtype=torch.float32) - + model.embedder.model.visual.input_patchnorm = None # fix error: visual model has not attribute 'input_patchnorm' convert(model.embedder, EMBEDDER_OV_PATH, example_input=dummy_input, input_shape=[1, 3, -1, -1]) .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/utils/image.py:226: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/utils/image.py:226: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input.numel() == 0: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:573: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:573: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if size == input_size: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:579: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:579: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! antialias = antialias and (max(factors) > 1) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:581: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:581: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if antialias: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:584: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:584: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! sigmas = (max((factors[0] - 1.0) / 2.0, 0.001), max((factors[1] - 1.0) / 2.0, 0.001)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. sigma = tensor([sigma], device=input.device, dtype=input.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! sigma = tensor([sigma], device=input.device, dtype=input.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/core/check.py:77: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/core/check.py:78: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_shape_to_check[i] != dim: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/kernels.py:92: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/kernels.py:92: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mean = tensor([[mean]], device=sigma.device, dtype=sigma.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:101: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:101: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if len(mean.shape) == 0 or mean.shape[0] == 1: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if len(std.shape) == 0 or std.shape[0] == 1: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:107: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:107: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if mean.shape and mean.shape[0] != 1: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if mean.shape[0] != data.shape[1] and mean.shape[:2] != data.shape[:2]: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:112: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:112: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if std.shape and std.shape[0] != 1: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if std.shape[0] != data.shape[1] and std.shape[:2] != data.shape[:2]: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:116: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:116: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mean = torch.as_tensor(mean, device=data.device, dtype=data.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:117: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:117: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. std = torch.as_tensor(std, device=data.device, dtype=data.dtype) @@ -363,10 +370,10 @@ Convert AE encoder .. code:: ipython3 ENCODER_FIRST_STAGE_OV_PATH = Path("models/encode_first_stage_ir.xml") - - + + dummy_input = torch.rand([1, 3, 256, 256], dtype=torch.float32) - + convert( model.first_stage_model.encoder, ENCODER_FIRST_STAGE_OV_PATH, @@ -376,7 +383,7 @@ Convert AE encoder .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/ae_modules.py:67: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/ae_modules.py:67: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! w_ = w_ * (int(c)**(-0.5)) @@ -388,18 +395,18 @@ Convert Diffusion U-Net model .. code:: ipython3 MODEL_OV_PATH = Path("models/model_ir.xml") - - + + class ModelWrapper(torch.nn.Module): def __init__(self, diffusion_model): super().__init__() self.diffusion_model = diffusion_model - + def forward(self, xc, t, context=None, fs=None, temporal_length=None): outputs = self.diffusion_model(xc, t, context=context, fs=fs, temporal_length=temporal_length) return outputs - - + + convert( ModelWrapper(model.model.diffusion_model), MODEL_OV_PATH, @@ -415,22 +422,22 @@ Convert Diffusion U-Net model .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:556: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:556: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if l_context == 77 + t*16: ## !!! HARD CODE here - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if batch_size: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:232: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:232: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if self.use_temporal_conv and batch_size: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:76: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:76: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:99: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:99: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[1] == self.channels Convert AE decoder ~~~~~~~~~~~~~~~~~~ - ``Decoder`` receives a +``Decoder`` receives a ``bfloat16`` tensor. numpy doesn’t support this type. To avoid problems with the conversion lets replace ``decode`` method to convert bfloat16 to float32. @@ -438,24 +445,24 @@ to float32. .. code:: ipython3 import types - - + + def decode(self, z, **kwargs): z = self.post_quant_conv(z) z = z.float() dec = self.decoder(z) return dec - - + + model.first_stage_model.decode = types.MethodType(decode, model.first_stage_model) .. code:: ipython3 DECODER_FIRST_STAGE_OV_PATH = Path("models/decoder_first_stage_ir.xml") - - + + dummy_input = torch.rand([16, 4, 32, 32], dtype=torch.float32) - + convert( model.first_stage_model.decoder, DECODER_FIRST_STAGE_OV_PATH, @@ -472,7 +479,7 @@ Select device from dropdown list for running inference using OpenVINO. .. code:: ipython3 import ipywidgets as widgets - + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -480,7 +487,7 @@ Select device from dropdown list for running inference using OpenVINO. description="Device:", disabled=False, ) - + device @@ -512,49 +519,49 @@ return ``torch.Tensor``\ s instead of ``np.array``\ s. .. code:: ipython3 import open_clip - - + + class CondStageModelWrapper(torch.nn.Module): def __init__(self, cond_stage_model): super().__init__() self.cond_stage_model = cond_stage_model - + def encode(self, tokens): if isinstance(tokens, list): tokens = open_clip.tokenize(tokens[0]) outs = self.cond_stage_model(tokens)[0] - + return torch.from_numpy(outs) - - + + class EncoderFirstStageModelWrapper(torch.nn.Module): def __init__(self, encode_first_stage): super().__init__() self.encode_first_stage = encode_first_stage - + def forward(self, x): outs = self.encode_first_stage(x)[0] - + return torch.from_numpy(outs) - - + + class EmbedderWrapper(torch.nn.Module): def __init__(self, embedder): super().__init__() self.embedder = embedder - + def forward(self, x): outs = self.embedder(x)[0] - + return torch.from_numpy(outs) - - + + class CModelWrapper(torch.nn.Module): def __init__(self, diffusion_model, out_channels): super().__init__() self.diffusion_model = diffusion_model self.out_channels = out_channels - + def forward(self, xc, t, context, fs, temporal_length): inputs = { "xc": xc, @@ -563,19 +570,19 @@ return ``torch.Tensor``\ s instead of ``np.array``\ s. "fs": fs, } outs = self.diffusion_model(inputs)[0] - + return torch.from_numpy(outs) - - + + class DecoderFirstStageModelWrapper(torch.nn.Module): def __init__(self, decoder_first_stage): super().__init__() self.decoder_first_stage = decoder_first_stage - + def forward(self, x): x.float() outs = self.decoder_first_stage(x)[0] - + return torch.from_numpy(outs) And insert wrappers instances in the pipeline: @@ -596,31 +603,31 @@ Interactive inference .. code:: ipython3 import time - + from einops import repeat from pytorch_lightning import seed_everything import torchvision.transforms as transforms - + from dynamicrafter.scripts.evaluation.funcs import save_videos, batch_ddim_sampling, get_latent_z - from dynamicrafter.lvdm.models.samplers.ddim import DDIMSampler - - + from lvdm.models.samplers.ddim import DDIMSampler + + def register_buffer(self, name, attr): if isinstance(attr, torch.Tensor): if attr.device != torch.device("cpu"): attr = attr.to(torch.device("cpu")) setattr(self, name, attr) - - + + # monkey patching to replace the original method 'register_buffer' that uses CUDA DDIMSampler.register_buffer = types.MethodType(register_buffer, DDIMSampler) - - + + def get_image(image, prompt, steps=5, cfg_scale=7.5, eta=1.0, fs=3, seed=123, model=model): result_dir = "results" if not os.path.exists(result_dir): os.mkdir(result_dir) - + seed_everything(seed) transform = transforms.Compose( [ @@ -639,31 +646,31 @@ Interactive inference frames = model.temporal_length h, w = 256 // 8, 256 // 8 noise_shape = [batch_size, channels, frames, h, w] - + # text cond with torch.no_grad(), torch.cpu.amp.autocast(): text_emb = model.get_learned_conditioning([prompt]) - + # img cond img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device) img_tensor = (img_tensor / 255.0 - 0.5) * 2 - + image_tensor_resized = transform(img_tensor) # 3,h,w videos = image_tensor_resized.unsqueeze(0) # bchw - + z = get_latent_z(model, videos.unsqueeze(2)) # bc,1,hw - + img_tensor_repeat = repeat(z, "b c t h w -> b c (repeat t) h w", repeat=frames) - + cond_images = model.embedder(img_tensor.unsqueeze(0)) # blc - + img_emb = model.image_proj_model(cond_images) - + imtext_cond = torch.cat([text_emb, img_emb], dim=1) - + fs = torch.tensor([fs], dtype=torch.long, device=model.device) cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]} - + ## inference batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale) ## b,samples,c,t,h,w @@ -672,17 +679,17 @@ Interactive inference prompt_str = prompt_str[:40] if len(prompt_str) == 0: prompt_str = "empty_prompt" - + save_videos(batch_samples, result_dir, filenames=[prompt_str], fps=8) print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds") - + return os.path.join(result_dir, f"{prompt_str}.mp4") .. code:: ipython3 import gradio as gr - - + + i2v_examples_256 = [ ["dynamicrafter/prompts/256/art.png", "man fishing in a boat at sunset", 50, 7.5, 1.0, 3, 234], ["dynamicrafter/prompts/256/boy.png", "boy walking on the street", 50, 7.5, 1.0, 3, 125], @@ -690,11 +697,11 @@ Interactive inference ["dynamicrafter/prompts/256/fire_and_beach.jpg", "a campfire on the beach and the ocean waves in the background", 50, 7.5, 1.0, 3, 111], ["dynamicrafter/prompts/256/guitar0.jpeg", "bear playing guitar happily, snowing", 50, 7.5, 1.0, 3, 122], ] - - + + def dynamicrafter_demo(): css = """#input_img {max-width: 256px !important} #output_vid {max-width: 256px; max-height: 256px}""" - + with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface: with gr.Tab(label="Image2Video_256x256"): with gr.Column(): @@ -714,7 +721,7 @@ Interactive inference i2v_end_btn = gr.Button("Generate") with gr.Row(): i2v_output_video = gr.Video(label="Generated Video", elem_id="output_vid", autoplay=True, show_share_button=True) - + gr.Examples( examples=i2v_examples_256, inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed], @@ -727,13 +734,13 @@ Interactive inference outputs=[i2v_output_video], fn=get_image, ) - + return dynamicrafter_iface - - + + demo = dynamicrafter_demo() - - + + try: demo.queue().launch(debug=False) except Exception: @@ -746,7 +753,7 @@ Interactive inference .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 - + To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/efficient-sam-with-output.rst b/docs/notebooks/efficient-sam-with-output.rst index b78a5d8df01ef0..c85d1e371471fe 100644 --- a/docs/notebooks/efficient-sam-with-output.rst +++ b/docs/notebooks/efficient-sam-with-output.rst @@ -28,8 +28,8 @@ In this tutorial we consider how to convert and run EfficientSAM using OpenVINO. We also demonstrate how to quantize model using `NNCF `__ -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Load PyTorch model <#load-pytorch-model>`__ @@ -72,12 +72,12 @@ Prerequisites .. code:: ipython3 import platform - + if platform.system() != "Windows": %pip install -q "matplotlib>=3.4" else: %pip install -q "matplotlib>=3.4,<3.7" - + %pip install -q "openvino>=2023.3.0" "nncf>=2.7.0" opencv-python "gradio>=4.13" torch torchvision tqdm --extra-index-url https://download.pytorch.org/whl/cpu @@ -90,9 +90,9 @@ Prerequisites .. code:: ipython3 from pathlib import Path - + repo_dir = Path("EfficientSAM") - + if not repo_dir.exists(): !git clone https://github.com/yformer/EfficientSAM.git %cd $repo_dir @@ -105,9 +105,9 @@ Prerequisites remote: Counting objects: 100% (85/85), done. remote: Compressing objects: 100% (33/33), done. remote: Total 424 (delta 76), reused 52 (delta 52), pack-reused 339 - Receiving objects: 100% (424/424), 262.14 MiB | 26.65 MiB/s, done. + Receiving objects: 100% (424/424), 262.14 MiB | 26.74 MiB/s, done. Resolving deltas: 100% (246/246), done. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM Load PyTorch model @@ -136,12 +136,12 @@ one of them as example. build_efficient_sam_vits, ) import zipfile - + MODELS_LIST = { "efficient-sam-vitt": build_efficient_sam_vitt, "efficient-sam-vits": build_efficient_sam_vits, } - + # Since EfficientSAM-S checkpoint file is >100MB, we store the zip file. with zipfile.ZipFile("weights/efficient_sam_vits.pt.zip", "r") as zip_ref: zip_ref.extractall("weights") @@ -151,16 +151,16 @@ Select one from supported models: .. code:: ipython3 import ipywidgets as widgets - + model_ids = list(MODELS_LIST) - + model_id = widgets.Dropdown( options=model_ids, value=model_ids[0], description="Model:", disabled=False, ) - + model_id @@ -177,13 +177,13 @@ build PyTorch model .. code:: ipython3 pt_model = MODELS_LIST[model_id.value]() - + pt_model.eval(); Run PyTorch model inference --------------------------- - Now, when we selected and +Now, when we selected and loaded PyTorch model, we can check its result Prepare input data @@ -203,9 +203,9 @@ bounding box, 3 - right-bottom point of bounding box. .. code:: ipython3 from PIL import Image - + image_path = "figs/examples/dogs.jpg" - + image = Image.open(image_path) image @@ -232,8 +232,8 @@ points. We also provided some helper function for results visualization. import torch import matplotlib.pyplot as plt import numpy as np - - + + def prepare_input(input_image, points, labels, torch_tensor=True): img_tensor = np.ascontiguousarray(input_image)[None, ...].astype(np.float32) / 255 img_tensor = np.transpose(img_tensor, (0, 3, 1, 2)) @@ -244,16 +244,16 @@ points. We also provided some helper function for results visualization. pts_sampled = torch.from_numpy(pts_sampled) pts_labels = torch.from_numpy(pts_labels) return img_tensor, pts_sampled, pts_labels - - + + def postprocess_results(predicted_iou, predicted_logits): sorted_ids = np.argsort(-predicted_iou, axis=-1) predicted_iou = np.take_along_axis(predicted_iou, sorted_ids, axis=2) predicted_logits = np.take_along_axis(predicted_logits, sorted_ids[..., None, None], axis=2) - + return predicted_logits[0, 0, 0, :, :] >= 0 - - + + def show_points(coords, labels, ax, marker_size=375): pos_points = coords[labels == 1] neg_points = coords[labels == 0] @@ -275,14 +275,14 @@ points. We also provided some helper function for results visualization. edgecolor="white", linewidth=1.25, ) - - + + def show_box(box, ax): x0, y0 = box[0], box[1] w, h = box[2] - box[0], box[3] - box[1] ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor="yellow", facecolor=(0, 0, 0, 0), lw=5)) - - + + def show_anns(mask, ax): ax.set_autoscale_on(False) img = np.ones((mask.shape[0], mask.shape[1], 4)) @@ -299,17 +299,17 @@ The complete model inference example demonstrated below input_points = [[580, 350], [650, 350]] input_labels = [1, 1] - + example_input = prepare_input(image, input_points, input_labels) - + predicted_logits, predicted_iou = pt_model(*example_input) - + predicted_mask = postprocess_results(predicted_iou.detach().numpy(), predicted_logits.detach().numpy()) .. code:: ipython3 image = Image.open(image_path) - + plt.figure(figsize=(20, 20)) plt.axis("off") plt.imshow(image) @@ -348,11 +348,11 @@ disk using ``openvino.save_model``. .. code:: ipython3 import openvino as ov - + core = ov.Core() - + ov_model_path = Path(f"{model_id.value}.xml") - + if not ov_model_path.exists(): ov_model = ov.convert_model(pt_model, example_input=example_input) ov.save_model(ov_model, ov_model_path) @@ -362,23 +362,23 @@ disk using ``openvino.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:220: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:220: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if ( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:241: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:241: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert ( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:163: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:163: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! size = int(math.sqrt(xy_num)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert size * size == xy_num - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:166: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:166: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if size != h or size != w: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:251: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:251: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[2] == num_patches - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if num_pts > self.decoder_max_num_input_points: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:92: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:92: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! elif num_pts < self.decoder_max_num_input_points: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_w > 0 and output_h > 0: @@ -400,7 +400,7 @@ Select inference device from dropdown list description="Device:", disabled=False, ) - + device @@ -432,11 +432,11 @@ Now, we can take a look on OpenVINO model prediction example_input = prepare_input(image, input_points, input_labels, torch_tensor=False) result = compiled_model(example_input) - + predicted_logits, predicted_iou = result[0], result[1] - + predicted_mask = postprocess_results(predicted_iou, predicted_logits) - + plt.figure(figsize=(20, 20)) plt.axis("off") plt.imshow(image) @@ -490,7 +490,7 @@ quantization. description="Quantization", disabled=False, ) - + to_quantize @@ -506,12 +506,12 @@ quantization. # Fetch `skip_kernel_extension` module import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/skip_kernel_extension.py", ) open("skip_kernel_extension.py", "w").write(r.text) - + %load_ext skip_kernel_extension Prepare calibration datasets @@ -529,22 +529,22 @@ creates DataLoader for preparing inputs for EfficientSAM model. .. code:: ipython3 %%skip not $to_quantize.value - + from zipfile import ZipFile - + r = requests.get( url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py', ) - + open('notebook_utils.py', 'w').write(r.text) - + from notebook_utils import download_file - + DATA_URL = "https://ultralytics.com/assets/coco128.zip" OUT_DIR = Path('.') - + download_file(DATA_URL, directory=OUT_DIR, show_progress=True) - + if not (OUT_DIR / "coco128/images/train2017").exists(): with ZipFile('coco128.zip' , "r") as zip_ref: zip_ref.extractall(OUT_DIR) @@ -559,21 +559,21 @@ creates DataLoader for preparing inputs for EfficientSAM model. .. code:: ipython3 %%skip not $to_quantize.value - + import torch.utils.data as data - + class COCOLoader(data.Dataset): def __init__(self, images_path): self.images = list(Path(images_path).iterdir()) self.labels_dir = images_path.parents[1] / 'labels' / images_path.name - + def get_points(self, image_path, image_width, image_height): file_name = image_path.name.replace('.jpg', '.txt') label_file = self.labels_dir / file_name if not label_file.exists(): x1, x2 = np.random.randint(low=0, high=image_width, size=(2, )) y1, y2 = np.random.randint(low=0, high=image_height, size=(2, )) - else: + else: with label_file.open("r") as f: box_line = f.readline() _, x1, y1, x2, y2 = box_line.split() @@ -582,7 +582,7 @@ creates DataLoader for preparing inputs for EfficientSAM model. x2 = int(float(x2) * image_width) y2 = int(float(y2) * image_height) return [[x1, y1], [x2, y2]] - + def __getitem__(self, index): image_path = self.images[index] image = Image.open(image_path) @@ -592,14 +592,14 @@ creates DataLoader for preparing inputs for EfficientSAM model. labels = [1, 1] if index % 2 == 0 else [2, 3] batched_images, batched_points, batched_point_labels = prepare_input(image, points, labels, torch_tensor=False) return {'batched_images': np.ascontiguousarray(batched_images)[0], 'batched_points': np.ascontiguousarray(batched_points)[0], 'batched_point_labels': np.ascontiguousarray(batched_point_labels)[0]} - + def __len__(self): return len(self.images) .. code:: ipython3 %%skip not $to_quantize.value - + coco_dataset = COCOLoader(OUT_DIR / 'coco128/images/train2017') calibration_loader = torch.utils.data.DataLoader(coco_dataset) @@ -622,11 +622,11 @@ architecture type, we should specify ``transformer`` in ``model_type``. .. code:: ipython3 %%skip not $to_quantize.value - + import nncf - + calibration_dataset = nncf.Dataset(calibration_loader) - + model = core.read_model(ov_model_path) quantized_model = nncf.quantize(model, calibration_dataset, @@ -642,10 +642,10 @@ architecture type, we should specify ``transformer`` in ``model_type``. .. parsed-literal:: - 2024-06-06 00:16:32.701538: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-06-06 00:16:32.734051: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-07-02 00:21:49.366604: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-07-02 00:21:49.398532: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-06-06 00:16:33.371409: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-07-02 00:21:50.014189: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -655,17 +655,17 @@ architecture type, we should specify ``transformer`` in ``model_type``. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -676,17 +676,17 @@ architecture type, we should specify ``transformer`` in ``model_type``. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -703,17 +703,17 @@ architecture type, we should specify ``transformer`` in ``model_type``. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -724,17 +724,17 @@ architecture type, we should specify ``transformer`` in ``model_type``. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -751,15 +751,15 @@ Verify quantized model inference .. code:: ipython3 %%skip not $to_quantize.value - + compiled_model = core.compile_model(quantized_model, device.value) - + result = compiled_model(example_input) - + predicted_logits, predicted_iou = result[0], result[1] - + predicted_mask = postprocess_results(predicted_iou, predicted_logits) - + plt.figure(figsize=(20, 20)) plt.axis("off") plt.imshow(image) @@ -788,7 +788,7 @@ Save quantize model on disk .. code:: ipython3 %%skip not $to_quantize.value - + quantized_model_path = Path(f"{model_id.value}_int8.xml") ov.save_model(quantized_model, quantized_model_path) @@ -800,10 +800,10 @@ Compare quantized model size .. code:: ipython3 %%skip not $to_quantize.value - + fp16_weights = ov_model_path.with_suffix('.bin') quantized_weights = quantized_model_path.with_suffix('.bin') - + print(f"Size of FP16 model is {fp16_weights.stat().st_size / 1024 / 1024:.2f} MB") print(f"Size of INT8 quantized model is {quantized_weights.stat().st_size / 1024 / 1024:.2f} MB") print(f"Compression rate for INT8 model: {fp16_weights.stat().st_size / quantized_weights.stat().st_size:.3f}") @@ -839,18 +839,18 @@ models, we use ``bencmark_app``. [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2024.3.0-15599-de4d00a5970 - [ INFO ] + [ INFO ] Build ................................. 2024.3.0-15837-76a668b0662 + [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2024.3.0-15599-de4d00a5970 - [ INFO ] - [ INFO ] + [ INFO ] Build ................................. 2024.3.0-15837-76a668b0662 + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 29.27 ms + [ INFO ] Read model took 49.00 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] batched_images (node: batched_images) : f32 / [...] / [?,?,?,?] @@ -870,7 +870,7 @@ models, we use ``bencmark_app``. [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_3) : f32 / [...] / [?,?,3,?,?] [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_2) : f32 / [...] / [?,?,3] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1448.36 ms + [ INFO ] Compile model took 1374.04 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -882,7 +882,7 @@ models, we use ``bencmark_app``. [ INFO ] AFFINITY: Affinity.CORE [ INFO ] CPU_DENORMALS_OPTIMIZATION: False [ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0 - [ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0 + [ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 32 [ INFO ] ENABLE_CPU_PINNING: True [ INFO ] ENABLE_HYPER_THREADING: True [ INFO ] EXECUTION_DEVICES: ['CPU'] @@ -906,22 +906,22 @@ models, we use ``bencmark_app``. [ WARNING ] No input files were given for input 'batched_images'!. This input will be filled with random values! [ WARNING ] No input files were given for input 'batched_points'!. This input will be filled with random values! [ WARNING ] No input files were given for input 'batched_point_labels'!. This input will be filled with random values! - [ INFO ] Fill input 'batched_images' with random values - [ INFO ] Fill input 'batched_points' with random values - [ INFO ] Fill input 'batched_point_labels' with random values + [ INFO ] Fill input 'batched_images' with random values + [ INFO ] Fill input 'batched_points' with random values + [ INFO ] Fill input 'batched_point_labels' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in full mode (inputs filling are included in measurement loop). - [ INFO ] First inference took 677.52 ms + [ INFO ] First inference took 672.94 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] [ INFO ] Count: 49 iterations - [ INFO ] Duration: 15956.35 ms + [ INFO ] Duration: 15915.46 ms [ INFO ] Latency: - [ INFO ] Median: 1926.16 ms - [ INFO ] Average: 1900.48 ms - [ INFO ] Min: 644.61 ms - [ INFO ] Max: 1989.53 ms - [ INFO ] Throughput: 3.07 FPS + [ INFO ] Median: 1916.49 ms + [ INFO ] Average: 1893.66 ms + [ INFO ] Min: 633.01 ms + [ INFO ] Max: 1977.93 ms + [ INFO ] Throughput: 3.08 FPS .. code:: ipython3 @@ -936,18 +936,18 @@ models, we use ``bencmark_app``. [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2024.3.0-15599-de4d00a5970 - [ INFO ] + [ INFO ] Build ................................. 2024.3.0-15837-76a668b0662 + [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2024.3.0-15599-de4d00a5970 - [ INFO ] - [ INFO ] + [ INFO ] Build ................................. 2024.3.0-15837-76a668b0662 + [ INFO ] + [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 42.94 ms + [ INFO ] Read model took 42.67 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] batched_images (node: batched_images) : f32 / [...] / [?,?,?,?] @@ -967,7 +967,7 @@ models, we use ``bencmark_app``. [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_3) : f32 / [...] / [?,?,3,?,?] [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_2) : f32 / [...] / [?,?,3] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1975.21 ms + [ INFO ] Compile model took 1885.12 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -979,7 +979,7 @@ models, we use ``bencmark_app``. [ INFO ] AFFINITY: Affinity.CORE [ INFO ] CPU_DENORMALS_OPTIMIZATION: False [ INFO ] CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0 - [ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 0 + [ INFO ] DYNAMIC_QUANTIZATION_GROUP_SIZE: 32 [ INFO ] ENABLE_CPU_PINNING: True [ INFO ] ENABLE_HYPER_THREADING: True [ INFO ] EXECUTION_DEVICES: ['CPU'] @@ -1003,22 +1003,22 @@ models, we use ``bencmark_app``. [ WARNING ] No input files were given for input 'batched_images'!. This input will be filled with random values! [ WARNING ] No input files were given for input 'batched_points'!. This input will be filled with random values! [ WARNING ] No input files were given for input 'batched_point_labels'!. This input will be filled with random values! - [ INFO ] Fill input 'batched_images' with random values - [ INFO ] Fill input 'batched_points' with random values - [ INFO ] Fill input 'batched_point_labels' with random values + [ INFO ] Fill input 'batched_images' with random values + [ INFO ] Fill input 'batched_points' with random values + [ INFO ] Fill input 'batched_point_labels' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in full mode (inputs filling are included in measurement loop). - [ INFO ] First inference took 609.28 ms + [ INFO ] First inference took 623.83 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] [ INFO ] Count: 55 iterations - [ INFO ] Duration: 16666.59 ms + [ INFO ] Duration: 16362.06 ms [ INFO ] Latency: - [ INFO ] Median: 1799.56 ms - [ INFO ] Average: 1780.10 ms - [ INFO ] Min: 641.70 ms - [ INFO ] Max: 1867.14 ms - [ INFO ] Throughput: 3.30 FPS + [ INFO ] Median: 1762.26 ms + [ INFO ] Average: 1740.85 ms + [ INFO ] Min: 488.03 ms + [ INFO ] Max: 1807.11 ms + [ INFO ] Throughput: 3.36 FPS Interactive segmentation demo @@ -1034,37 +1034,37 @@ Interactive segmentation demo from PIL import ImageDraw, Image import cv2 import matplotlib.pyplot as plt - + example_images = [ "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/b8083dd5-1ce7-43bf-8b09-a2ebc280c86e", "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/9a90595d-70e7-469b-bdaf-469ef4f56fa2", "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/b626c123-9fa2-4aa6-9929-30565991bf0c", ] - + examples_dir = Path("examples") examples_dir.mkdir(exist_ok=True) - + for img_id, image_url in enumerate(example_images): r = requests.get(image_url) img_path = examples_dir / f"example_{img_id}.jpg" with img_path.open("wb") as f: f.write(r.content) - - + + def sigmoid(x): return 1 / (1 + np.exp(-x)) - - + + def clear(): return None, None, [], [] - - + + def format_results(masks, scores, logits, filter=0): annotations = [] n = len(scores) for i in range(n): annotation = {} - + mask = masks[i] tmp = np.where(mask != 0) if np.sum(mask) < filter: @@ -1081,8 +1081,8 @@ Interactive segmentation demo annotation["area"] = annotation["segmentation"].sum() annotations.append(annotation) return annotations - - + + def point_prompt(masks, points, point_label, target_height, target_width): # numpy h = masks[0]["segmentation"].shape[0] w = masks[0]["segmentation"].shape[1] @@ -1102,8 +1102,8 @@ Interactive segmentation demo onemask -= mask onemask = onemask >= 1 return onemask, 0 - - + + def show_mask( annotation, ax, @@ -1120,7 +1120,7 @@ Interactive segmentation demo areas = np.sum(annotation, axis=(1, 2)) sorted_indices = np.argsort(areas)[::1] annotation = annotation[sorted_indices] - + index = (annotation != 0).argmax(axis=0) if random_color: color = np.random.random((mask_sum, 1, 1, 3)) @@ -1129,23 +1129,23 @@ Interactive segmentation demo transparency = np.ones((mask_sum, 1, 1, 1)) * 0.6 visual = np.concatenate([color, transparency], axis=-1) mask_image = np.expand_dims(annotation, -1) * visual - + mask = np.zeros((height, weight, 4)) - + h_indices, w_indices = np.meshgrid(np.arange(height), np.arange(weight), indexing="ij") indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None)) - + mask[h_indices, w_indices, :] = mask_image[indices] if bbox is not None: x1, y1, x2, y2 = bbox ax.add_patch(plt.Rectangle((x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor="b", linewidth=1)) - + if not retinamask: mask = cv2.resize(mask, (target_width, target_height), interpolation=cv2.INTER_NEAREST) - + return mask - - + + def process( annotations, image, @@ -1159,7 +1159,7 @@ Interactive segmentation demo ): if isinstance(annotations[0], dict): annotations = [annotation["segmentation"] for annotation in annotations] - + original_h = image.height original_w = image.width if better_quality: @@ -1178,10 +1178,10 @@ Interactive segmentation demo target_height=original_h, target_width=original_w, ) - + if isinstance(annotations, torch.Tensor): annotations = annotations.cpu().numpy() - + if withContours: contour_all = [] temp = np.zeros((original_h, original_w, 1)) @@ -1201,22 +1201,22 @@ Interactive segmentation demo cv2.drawContours(temp, contour_all, -1, (255, 255, 255), 2 // scale) color = np.array([0 / 255, 0 / 255, 255 / 255, 0.9]) contour_mask = temp / 255 * color.reshape(1, 1, -1) - + image = image.convert("RGBA") overlay_inner = Image.fromarray((inner_mask * 255).astype(np.uint8), "RGBA") image.paste(overlay_inner, (0, 0), overlay_inner) - + if withContours: overlay_contour = Image.fromarray((contour_mask * 255).astype(np.uint8), "RGBA") image.paste(overlay_contour, (0, 0), overlay_contour) - + return image - - + + # Description title = "
Efficient Segment Anything with OpenVINO and EfficientSAM
" - - + + description_p = """# Interactive Instance Segmentation - Point-prompt instruction
    @@ -1230,15 +1230,15 @@ Interactive segmentation demo
  1. Click the button of Segment with Box Prompt
""" - + # examples examples = [[img] for img in examples_dir.glob("*.jpg")] - + default_example = examples[0] - + css = "h1 { text-align: center } .about { text-align: justify; padding-left: 10%; padding-right: 10%; }" - - + + def segment_with_boxs( image, seg_image, @@ -1252,48 +1252,48 @@ Interactive segmentation demo ): if global_points is None or len(global_points) < 2 or global_points[0] is None: return image, global_points, global_point_label - + input_size = int(input_size) w, h = image.size scale = input_size / max(w, h) new_w = int(w * scale) new_h = int(h * scale) image = image.resize((new_w, new_h)) - + scaled_points = np.array([[int(x * scale) for x in point] for point in global_points]) scaled_points = scaled_points[:2] scaled_point_label = np.array(global_point_label)[:2] - + if scaled_points.size == 0 and scaled_point_label.size == 0: return image, global_points, global_point_label - + nd_image = np.array(image) img_tensor = nd_image.astype(np.float32) / 255 img_tensor = np.transpose(img_tensor, (2, 0, 1)) - + pts_sampled = np.reshape(scaled_points, [1, 1, -1, 2]) pts_sampled = pts_sampled[:, :, :2, :] pts_labels = np.reshape(np.array([2, 3]), [1, 1, 2]) - + results = compiled_model([img_tensor[None, ...], pts_sampled, pts_labels]) predicted_logits = results[0] predicted_iou = results[1] all_masks = sigmoid(predicted_logits[0, 0, :, :, :]) >= 0.5 predicted_iou = predicted_iou[0, 0, ...] - + max_predicted_iou = -1 selected_mask_using_predicted_iou = None selected_predicted_iou = None - + for m in range(all_masks.shape[0]): curr_predicted_iou = predicted_iou[m] if curr_predicted_iou > max_predicted_iou or selected_mask_using_predicted_iou is None: max_predicted_iou = curr_predicted_iou selected_mask_using_predicted_iou = all_masks[m : m + 1] selected_predicted_iou = predicted_iou[m : m + 1] - + results = format_results(selected_mask_using_predicted_iou, selected_predicted_iou, predicted_logits, 0) - + annotations = results[0]["segmentation"] annotations = np.array([annotations]) fig = process( @@ -1306,12 +1306,12 @@ Interactive segmentation demo bbox=scaled_points.reshape([4]), withContours=withContours, ) - + global_points = [] global_point_label = [] return fig, global_points, global_point_label - - + + def segment_with_points( image, global_points, @@ -1328,32 +1328,32 @@ Interactive segmentation demo new_w = int(w * scale) new_h = int(h * scale) image = image.resize((new_w, new_h)) - + if global_points is None or len(global_points) < 1 or global_points[0] is None: return image, global_points, global_point_label scaled_points = np.array([[int(x * scale) for x in point] for point in global_points]) scaled_point_label = np.array(global_point_label) - + if scaled_points.size == 0 and scaled_point_label.size == 0: return image, global_points, global_point_label - + nd_image = np.array(image) img_tensor = (nd_image).astype(np.float32) / 255 img_tensor = np.transpose(img_tensor, (2, 0, 1)) - + pts_sampled = np.reshape(scaled_points, [1, 1, -1, 2]) pts_labels = np.reshape(np.array(global_point_label), [1, 1, -1]) - + results = compiled_model([img_tensor[None, ...], pts_sampled, pts_labels]) predicted_logits = results[0] predicted_iou = results[1] all_masks = sigmoid(predicted_logits[0, 0, :, :, :]) >= 0.5 predicted_iou = predicted_iou[0, 0, ...] - + results = format_results(all_masks, predicted_iou, predicted_logits, 0) annotations, _ = point_prompt(results, scaled_points, scaled_point_label, new_h, new_w) annotations = np.array([annotations]) - + fig = process( annotations=annotations, image=image, @@ -1365,13 +1365,13 @@ Interactive segmentation demo use_retina=use_retina, withContours=withContours, ) - + global_points = [] global_point_label = [] # return fig, None return fig, global_points, global_point_label - - + + def get_points_with_draw(image, cond_image, global_points, global_point_label, evt: gr.SelectData): print(global_points) if len(global_points) == 0: @@ -1389,10 +1389,10 @@ Interactive segmentation demo ) global_points.append([x, y]) global_point_label.append(1 if label == "Add Mask" else 0) - + if image is not None: draw = ImageDraw.Draw(image) - + draw.ellipse( [ (x - point_radius, y - point_radius), @@ -1400,10 +1400,10 @@ Interactive segmentation demo ], fill=point_color, ) - + return image, global_points, global_point_label - - + + def get_points_with_draw_(image, cond_image, global_points, global_point_label, evt: gr.SelectData): if len(global_points) == 0: image = copy.deepcopy(cond_image) @@ -1422,7 +1422,7 @@ Interactive segmentation demo ) global_points.append([x, y]) global_point_label.append(1 if label == "Add Mask" else 0) - + if image is not None: draw = ImageDraw.Draw(image) draw.ellipse( @@ -1432,7 +1432,7 @@ Interactive segmentation demo ], fill=point_color, ) - + if len(global_points) == 2: x1, y1 = global_points[0] x2, y2 = global_points[1] @@ -1456,17 +1456,17 @@ Interactive segmentation demo global_points[0][1] = y2 global_points[1][0] = x1 global_points[1][1] = y1 - + return image, global_points, global_point_label - - + + cond_img_p = gr.Image(label="Input with Point", value=default_example[0], type="pil") cond_img_b = gr.Image(label="Input with Box", value=default_example[0], type="pil") - + segm_img_p = gr.Image(label="Segmented Image with Point-Prompt", interactive=False, type="pil") segm_img_b = gr.Image(label="Segmented Image with Box-Prompt", interactive=False, type="pil") - - + + with gr.Blocks(css=css, title="Efficient SAM") as demo: global_points = gr.State([]) global_point_label = gr.State([]) @@ -1474,16 +1474,16 @@ Interactive segmentation demo with gr.Column(scale=1): # Title gr.Markdown(title) - + with gr.Tab("Point mode"): # Images with gr.Row(variant="panel"): with gr.Column(scale=1): cond_img_p.render() - + with gr.Column(scale=1): segm_img_p.render() - + # Submit & Clear # ### with gr.Row(): @@ -1491,72 +1491,72 @@ Interactive segmentation demo with gr.Column(): segment_btn_p = gr.Button("Segment with Point Prompt", variant="primary") clear_btn_p = gr.Button("Clear", variant="secondary") - + gr.Markdown("Try some of the examples below ⬇️") gr.Examples( examples=examples, inputs=[cond_img_p], examples_per_page=4, ) - + with gr.Column(): # Description gr.Markdown(description_p) - + with gr.Tab("Box mode"): # Images with gr.Row(variant="panel"): with gr.Column(scale=1): cond_img_b.render() - + with gr.Column(scale=1): segm_img_b.render() - + # Submit & Clear with gr.Row(): with gr.Column(): with gr.Column(): segment_btn_b = gr.Button("Segment with Box Prompt", variant="primary") clear_btn_b = gr.Button("Clear", variant="secondary") - + gr.Markdown("Try some of the examples below ⬇️") gr.Examples( examples=examples, inputs=[cond_img_b], examples_per_page=4, ) - + with gr.Column(): # Description gr.Markdown(description_p) - + cond_img_p.select( get_points_with_draw, inputs=[segm_img_p, cond_img_p, global_points, global_point_label], outputs=[segm_img_p, global_points, global_point_label], ) - + cond_img_b.select( get_points_with_draw_, [segm_img_b, cond_img_b, global_points, global_point_label], [segm_img_b, global_points, global_point_label], ) - + segment_btn_p.click( segment_with_points, inputs=[cond_img_p, global_points, global_point_label], outputs=[segm_img_p, global_points, global_point_label], ) - + segment_btn_b.click( segment_with_boxs, inputs=[cond_img_b, segm_img_b, global_points, global_point_label], outputs=[segm_img_b, global_points, global_point_label], ) - + clear_btn_p.click(clear, outputs=[cond_img_p, segm_img_p, global_points, global_point_label]) clear_btn_b.click(clear, outputs=[cond_img_b, segm_img_b, global_points, global_point_label]) - + demo.queue() try: demo.launch(debug=False) @@ -1570,7 +1570,7 @@ Interactive segmentation demo .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 - + To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_16_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_16_1.png index ff05849fba88f0..a579f7cf8d9b79 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_16_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_16_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:45a2aec656ead3328284564339cc490e379c5ad16d1a1a4922f99eb84037b818 -size 1259656 +oid sha256:e23a04e4766a7681a0ccb191624e27243abc5b472a93f8c75913482b2e4677b7 +size 1260060 diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_24_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_24_1.png index 59851a8fe68139..e9dde9eaa7df76 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_24_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_24_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7dd1210236f949071574ecd9097962a3db3156b0c44810aa2f2b6d03576f6ca3 -size 1260203 +oid sha256:5bb4c9271a08dd69c873d9cc7f980750e19bf7c385193389edee6ecd43bc6c01 +size 1261465 diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_35_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_35_1.png index 903ec2ee00b8b1..6a0e8658b23756 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_35_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_35_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:89fc9efa319522df2dd0638e28e020e8695a8c791db61da44f75703b1d493470 -size 1262684 +oid sha256:ab8dd706c2c1f138e44def1caddc86762a5c055d1c528bd8b4fd35934e307ef7 +size 1261457 diff --git a/docs/notebooks/encodec-audio-compression-with-output.rst b/docs/notebooks/encodec-audio-compression-with-output.rst index 41f2b975c3bc98..62970a41a19a32 100644 --- a/docs/notebooks/encodec-audio-compression-with-output.rst +++ b/docs/notebooks/encodec-audio-compression-with-output.rst @@ -26,8 +26,8 @@ and original `repo `__. image.png -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Instantiate audio compression @@ -129,7 +129,7 @@ bandwidth. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -289,7 +289,7 @@ similar as possible to the original. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -389,13 +389,13 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! max_pad = max(padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if length <= max_pad: @@ -415,11 +415,11 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. quantized_out = torch.tensor(0.0, device=q_indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). for i, indices in enumerate(q_indices): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert (padding_left + padding_right) <= x.shape[-1] diff --git a/docs/notebooks/fast-segment-anything-with-output.rst b/docs/notebooks/fast-segment-anything-with-output.rst index cac2e1485c6fac..dec0dd864785db 100644 --- a/docs/notebooks/fast-segment-anything-with-output.rst +++ b/docs/notebooks/fast-segment-anything-with-output.rst @@ -28,8 +28,8 @@ the prompt. pipeline -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ @@ -145,7 +145,7 @@ model and generate a segmentation map. .. parsed-literal:: - 100%|██████████| 138M/138M [00:01<00:00, 100MB/s] + 100%|██████████| 138M/138M [00:01<00:00, 101MB/s] @@ -157,8 +157,8 @@ model and generate a segmentation map. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 658.4ms - Speed: 4.0ms preprocess, 658.4ms inference, 586.5ms postprocess per image at shape (1, 3, 768, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 645.4ms + Speed: 4.0ms preprocess, 645.4ms inference, 583.2ms postprocess per image at shape (1, 3, 768, 1024) The model returns segmentation maps for all the objects on the image. @@ -198,13 +198,13 @@ tracing. The FastSAM model itself is based on YOLOv8 model. Ultralytics YOLOv8.2.24 🚀 Python-3.8.10 torch-2.3.1+cpu CPU (Intel Core(TM) i9-10920X 3.50GHz) - PyTorch: starting from 'FastSAM-x.pt' with input shape (1, 3, 1024, 1024) BCHW and output shape(s) ((1, 37, 21504), (1, 32, 256, 256)) (138.2 MB) + PyTorch: starting from 'FastSAM-x.pt' with input shape (1, 3, 1024, 1024) BCHW and output shape(s) ((1, 37, 21504), (1, 32, 256, 256)) (138.3 MB) - OpenVINO: starting export with openvino 2024.1.0-15008-f4afc983258-releases/2024/1... + OpenVINO: starting export with openvino 2024.2.0-15519-5c0f38f83f6-releases/2024/2... OpenVINO: export success ✅ 6.2s, saved as 'FastSAM-x_openvino_model/' (276.1 MB) - Export complete (9.2s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/fast-segment-anything + Export complete (9.1s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/fast-segment-anything Predict: yolo predict task=segment model=FastSAM-x_openvino_model imgsz=1024 Validate: yolo val task=segment model=FastSAM-x_openvino_model imgsz=1024 data=ultralytics/datasets/sa.yaml Visualize: https://netron.app @@ -313,8 +313,8 @@ pipeline. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 1024x1024 42 objects, 503.6ms - Speed: 7.6ms preprocess, 503.6ms inference, 38.0ms postprocess per image at shape (1, 3, 1024, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 1024x1024 42 objects, 502.2ms + Speed: 7.4ms preprocess, 502.2ms inference, 33.8ms postprocess per image at shape (1, 3, 1024, 1024) One can observe the converted model outputs in the next cell, they is @@ -539,15 +539,15 @@ repo <../yolov8-optimization/>`__. INFO:nncf:3 ignored nodes were found by name in the NNCFGraph INFO:nncf:8 ignored nodes were found by types in the NNCFGraph - INFO:nncf:Not adding activation input quantizer for operation: 275 __module.model.22/aten::sigmoid/Sigmoid - INFO:nncf:Not adding activation input quantizer for operation: 325 __module.model.22.dfl.conv/aten::_convolution/Convolution - INFO:nncf:Not adding activation input quantizer for operation: 351 __module.model.22/aten::sub/Subtract - INFO:nncf:Not adding activation input quantizer for operation: 352 __module.model.22/aten::add/Add - INFO:nncf:Not adding activation input quantizer for operation: 365 __module.model.22/aten::add/Add_1 - 378 __module.model.22/aten::div/Divide + INFO:nncf:Not adding activation input quantizer for operation: 271 __module.model.22/aten::sigmoid/Sigmoid + INFO:nncf:Not adding activation input quantizer for operation: 312 __module.model.22.dfl.conv/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 349 __module.model.22/aten::sub/Subtract + INFO:nncf:Not adding activation input quantizer for operation: 350 __module.model.22/aten::add/Add + INFO:nncf:Not adding activation input quantizer for operation: 362 __module.model.22/aten::add/Add_1 + 374 __module.model.22/aten::div/Divide - INFO:nncf:Not adding activation input quantizer for operation: 366 __module.model.22/aten::sub/Subtract_1 - INFO:nncf:Not adding activation input quantizer for operation: 389 __module.model.22/aten::mul/Multiply + INFO:nncf:Not adding activation input quantizer for operation: 363 __module.model.22/aten::sub/Subtract_1 + INFO:nncf:Not adding activation input quantizer for operation: 386 __module.model.22/aten::mul/Multiply @@ -557,17 +557,17 @@ repo <../yolov8-optimization/>`__. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -578,17 +578,17 @@ repo <../yolov8-optimization/>`__. -.. raw:: html -

 
 
 
 
-.. raw:: html
 
-    
-    
+ + + + + @@ -656,9 +656,9 @@ calibration dataset to measure the performance. .. parsed-literal:: - Segmented in 23 seconds - Resulting in 5.57 fps - That is 2.96 times faster! + Segmented in 22 seconds + Resulting in 5.82 fps + That is 3.09 times faster! Try out the converted pipeline diff --git a/docs/notebooks/film-slowmo-with-output.rst b/docs/notebooks/film-slowmo-with-output.rst index 5a3c12de3717db..51342e3cc06830 100644 --- a/docs/notebooks/film-slowmo-with-output.rst +++ b/docs/notebooks/film-slowmo-with-output.rst @@ -38,8 +38,8 @@ a model source. video encoder. Ubuntu has it preinstalled, but for Windows, you should install it manually. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Prerequisites <#prerequisites>`__ - `Prepare images <#prepare-images>`__ diff --git a/docs/notebooks/freevc-voice-conversion-with-output.rst b/docs/notebooks/freevc-voice-conversion-with-output.rst index a70c8f37121789..8b50a7449daabc 100644 --- a/docs/notebooks/freevc-voice-conversion-with-output.rst +++ b/docs/notebooks/freevc-voice-conversion-with-output.rst @@ -28,8 +28,8 @@ devices. It consists of the following steps: - Convert models to OpenVINO Intermediate Representation. - Inference using only OpenVINO’s IR models. -Table of contents: -^^^^^^^^^^^^^^^^^^ +**Table of contents:** + - `Pre-requisites <#pre-requisites>`__ - `Imports and settings <#imports-and-settings>`__ @@ -77,12 +77,12 @@ Check if FreeVC is installed and append its path to ``sys.path`` from pathlib import Path import sys - - + + free_vc_repo = "FreeVC" if not Path(free_vc_repo).exists(): !git clone https://github.com/OlaWod/FreeVC.git - + sys.path.append(free_vc_repo) @@ -93,7 +93,7 @@ Check if FreeVC is installed and append its path to ``sys.path`` remote: Counting objects: 100% (65/65), done. remote: Compressing objects: 100% (41/41), done. remote: Total 131 (delta 39), reused 24 (delta 24), pack-reused 66 - Receiving objects: 100% (131/131), 15.28 MiB | 27.90 MiB/s, done. + Receiving objects: 100% (131/131), 15.28 MiB | 21.50 MiB/s, done. Resolving deltas: 100% (43/43), done. @@ -102,19 +102,19 @@ Check if FreeVC is installed and append its path to ``sys.path`` # Fetch `notebook_utils` module import requests import gdown - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) - + open("notebook_utils.py", "w").write(r.text) from notebook_utils import download_file - + wavlm_large_dir_path = Path("FreeVC/wavlm") wavlm_large_path = wavlm_large_dir_path / "WavLM-Large.pt" - + wavlm_url = "https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU&confirm=t&uuid=a703c43c-ccce-436c-8799-c11b88e9e7e4" - + if not wavlm_large_path.exists(): gdown.download(wavlm_url, str(wavlm_large_path)) @@ -123,8 +123,8 @@ Check if FreeVC is installed and append its path to ``sys.path`` Downloading... From: https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU&confirm=t&uuid=a703c43c-ccce-436c-8799-c11b88e9e7e4 - To: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/WavLM-Large.pt - 100%|██████████| 1.26G/1.26G [00:34<00:00, 36.3MB/s] + To: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/WavLM-Large.pt + 100%|██████████| 1.26G/1.26G [00:34<00:00, 36.7MB/s] .. code:: ipython3 @@ -132,7 +132,7 @@ Check if FreeVC is installed and append its path to ``sys.path`` freevc_chpt_dir = Path("checkpoints") freevc_chpt_name = "freevc.pth" freevc_chpt_path = freevc_chpt_dir / freevc_chpt_name - + if not freevc_chpt_path.exists(): download_file( f"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/freevc/{freevc_chpt_name}", @@ -152,10 +152,10 @@ Check if FreeVC is installed and append its path to ``sys.path`` audio1_url = f"https://huggingface.co/spaces/OlaWod/FreeVC/resolve/main/{audio1_name}" audio2_name = "p226_002.wav" audio2_url = f"https://huggingface.co/spaces/OlaWod/FreeVC/resolve/main/{audio2_name}" - + if not Path(audio1_name).exists(): download_file(audio1_url) - + if not Path(audio2_name).exists(): download_file(audio2_url) @@ -182,20 +182,20 @@ Imports and settings import logging import os import time - + import librosa import numpy as np import torch from scipy.io.wavfile import write from tqdm import tqdm - + import openvino as ov - + import utils from models import SynthesizerTrn from speaker_encoder.voice_encoder import SpeakerEncoder from wavlm import WavLM, WavLMConfig - + logger = logging.getLogger() logger.setLevel(logging.CRITICAL) @@ -209,7 +209,7 @@ Redefine function ``get_model`` from ``utils`` to exclude CUDA cmodel = WavLM(cfg) cmodel.load_state_dict(checkpoint["model"]) cmodel.eval() - + return cmodel Models initialization @@ -218,9 +218,9 @@ Models initialization hps = utils.get_hparams_from_file("FreeVC/configs/freevc.json") os.makedirs("outputs/freevc", exist_ok=True) - + net_g = SynthesizerTrn(hps.data.filter_length // 2 + 1, hps.train.segment_size // hps.data.hop_length, **hps.model) - + utils.load_checkpoint(freevc_chpt_path, net_g, optimizer=None, strict=True) cmodel = get_cmodel() smodel = SpeakerEncoder("FreeVC/speaker_encoder/ckpt/pretrained_bak_5805000.pt", device="cpu") @@ -228,7 +228,7 @@ Models initialization .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -254,19 +254,19 @@ Inference # tgt wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) - + g_tgt = smodel.embed_utterance(wav_tgt) g_tgt = torch.from_numpy(g_tgt).unsqueeze(0) - + # src wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate) wav_src = torch.from_numpy(wav_src).unsqueeze(0) - + c = utils.get_content(cmodel, wav_src) - + tgt_audio = net_g.infer(c, g=g_tgt) tgt_audio = tgt_audio[0][0].data.cpu().float().numpy() - + timestamp = time.strftime("%m-%d_%H-%M", time.localtime()) write( os.path.join("outputs/freevc", "{}.wav".format(timestamp)), @@ -277,7 +277,7 @@ Inference .. parsed-literal:: - 2it [00:03, 1.55s/it] + 2it [00:03, 1.61s/it] Result audio files should be available in ‘outputs/freevc’ @@ -315,13 +315,13 @@ OpenVINO’s IR format. We keep the original name of the model in code: OUTPUT_DIR = Path("output") BASE_MODEL_NAME = "cmodel" - + OUTPUT_DIR.mkdir(exist_ok=True) - + ir_cmodel_path = Path(OUTPUT_DIR / (BASE_MODEL_NAME + "_ir")).with_suffix(".xml") - + length = 32000 - + dummy_input = torch.randn(1, length) Converting to OpenVINO’s IR format. @@ -329,17 +329,17 @@ Converting to OpenVINO’s IR format. .. code:: ipython3 core = ov.Core() - - + + class ModelWrapper(torch.nn.Module): def __init__(self, model): super().__init__() self.model = model - + def forward(self, input): return self.model(input)[0] - - + + if not ir_cmodel_path.exists(): ir_cmodel = ov.convert_model(ModelWrapper(cmodel), example_input=dummy_input) ov.save_model(ir_cmodel, ir_cmodel_path) @@ -349,13 +349,13 @@ Converting to OpenVINO’s IR format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:495: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:495: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert embed_dim == self.embed_dim - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:496: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:496: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert list(query.size()) == [tgt_len, bsz, embed_dim] - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:500: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:500: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert key_bsz == bsz - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:502: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:502: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert src_len, bsz == value.shape[:2] @@ -364,14 +364,14 @@ Select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value="AUTO", description="Device:", disabled=False, ) - + device @@ -396,16 +396,16 @@ Convert ``SpeakerEncoder`` OUTPUT_DIR = Path("output") BASE_MODEL_NAME = "smodel" - + OUTPUT_DIR.mkdir(exist_ok=True) - + ir_smodel_path = Path(OUTPUT_DIR / (BASE_MODEL_NAME + "ir")).with_suffix(".xml") - - + + length = 32000 - + dummy_input = torch.randn(1, length, 40) - + if not ir_smodel_path.exists(): ir_smodel = ov.convert_model(smodel, example_input=dummy_input) ov.save_model(ir_smodel, ir_smodel_path) @@ -419,18 +419,18 @@ based on ``speaker_encoder.voice_encoder.SpeakerEncoder`` class methods from speaker_encoder.hparams import sampling_rate, mel_window_step, partials_n_frames from speaker_encoder import audio - - + + def compute_partial_slices(n_samples: int, rate, min_coverage): """ Computes where to split an utterance waveform and its corresponding mel spectrogram to obtain partial utterances of each. Both the waveform and the mel spectrogram slices are returned, so as to make each partial utterance waveform correspond to its spectrogram. - + The returned ranges may be indexing further than the length of the waveform. It is recommended that you pad the waveform with zeros up to wav_slices[-1].stop. - + :param n_samples: the number of samples in the waveform :param rate: how many partial utterances should occur per second. Partial utterances must cover the span of the entire utterance, thus the rate should not be lower than the inverse @@ -446,14 +446,14 @@ based on ``speaker_encoder.voice_encoder.SpeakerEncoder`` class methods utterances. """ assert 0 < min_coverage <= 1 - + # Compute how many frames separate two partial utterances samples_per_frame = int((sampling_rate * mel_window_step / 1000)) n_frames = int(np.ceil((n_samples + 1) / samples_per_frame)) frame_step = int(np.round((sampling_rate / rate) / samples_per_frame)) assert 0 < frame_step, "The rate is too high" assert frame_step <= partials_n_frames, "The rate is too low, it should be %f at least" % (sampling_rate / (samples_per_frame * partials_n_frames)) - + # Compute the slices wav_slices, mel_slices = [], [] steps = max(1, n_frames - partials_n_frames + frame_step + 1) @@ -462,17 +462,17 @@ based on ``speaker_encoder.voice_encoder.SpeakerEncoder`` class methods wav_range = mel_range * samples_per_frame mel_slices.append(slice(*mel_range)) wav_slices.append(slice(*wav_range)) - + # Evaluate whether extra padding is warranted or not last_wav_range = wav_slices[-1] coverage = (n_samples - last_wav_range.start) / (last_wav_range.stop - last_wav_range.start) if coverage < min_coverage and len(mel_slices) > 1: mel_slices = mel_slices[:-1] wav_slices = wav_slices[:-1] - + return wav_slices, mel_slices - - + + def embed_utterance( wav: np.ndarray, smodel: ov.CompiledModel, @@ -484,7 +484,7 @@ based on ``speaker_encoder.voice_encoder.SpeakerEncoder`` class methods Computes an embedding for a single utterance. The utterance is divided in partial utterances and an embedding is computed for each. The complete utterance embedding is the L2-normed average embedding of the partial utterances. - + :param wav: a preprocessed utterance waveform as a numpy array of float32 :param smodel: compiled speaker encoder model. :param return_partials: if True, the partial embeddings will also be returned along with @@ -509,7 +509,7 @@ based on ``speaker_encoder.voice_encoder.SpeakerEncoder`` class methods max_wave_length = wav_slices[-1].stop if max_wave_length >= len(wav): wav = np.pad(wav, (0, max_wave_length - len(wav)), "constant") - + # Split the utterance into partials and forward them through the model mel = audio.wav_to_mel_spectrogram(wav) mels = np.array([mel[s] for s in mel_slices]) @@ -517,11 +517,11 @@ based on ``speaker_encoder.voice_encoder.SpeakerEncoder`` class methods mels = torch.from_numpy(mels).to(torch.device("cpu")) output_layer = smodel.output(0) partial_embeds = smodel(mels)[output_layer] - + # Compute the utterance embedding from the partial embeddings raw_embed = np.mean(partial_embeds, axis=0) embed = raw_embed / np.linalg.norm(raw_embed, 2) - + if return_partials: return embed, partial_embeds, wav_slices return embed @@ -561,14 +561,14 @@ function to OpenVINO IR format. BASE_MODEL_NAME = "net_g" onnx_net_g_path = Path(OUTPUT_DIR / (BASE_MODEL_NAME + "_fp32")).with_suffix(".onnx") ir_net_g_path = Path(OUTPUT_DIR / (BASE_MODEL_NAME + "ir")).with_suffix(".xml") - + dummy_input_1 = torch.randn(1, 1024, 81) dummy_input_2 = torch.randn(1, 256) - + # define forward as infer net_g.forward = net_g.infer - - + + if not ir_net_g_path.exists(): ir_net_g_model = ov.convert_model(net_g, example_input=(dummy_input_1, dummy_input_2)) ov.save_model(ir_net_g_model, ir_net_g_path) @@ -578,12 +578,12 @@ function to OpenVINO IR format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-697/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-717/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - - Mismatched elements: 25632 / 25920 (98.9%) - Greatest absolute difference: 0.05565535183995962 at index (0, 0, 18928) (up to 1e-05 allowed) - Greatest relative difference: 506443.2686567164 at index (0, 0, 10753) (up to 1e-05 allowed) + + Mismatched elements: 25917 / 25920 (100.0%) + Greatest absolute difference: 0.6957888901233673 at index (0, 0, 6412) (up to 1e-05 allowed) + Greatest relative difference: 26109.2899320933 at index (0, 0, 23414) (up to 1e-05 allowed) _check_trace( @@ -600,22 +600,22 @@ Define function for synthesizing. def synthesize_audio(src, tgt): wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) - + g_tgt = embed_utterance(wav_tgt, compiled_smodel) g_tgt = torch.from_numpy(g_tgt).unsqueeze(0) - + # src wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate) wav_src = np.expand_dims(wav_src, axis=0) - + output_layer = compiled_cmodel.output(0) c = compiled_cmodel(wav_src)[output_layer] c = c.transpose((0, 2, 1)) - + output_layer = compiled_ir_net_g_model.output(0) tgt_audio = compiled_ir_net_g_model((c, g_tgt))[output_layer] tgt_audio = tgt_audio[0][0] - + return tgt_audio And now we can check inference using only IR models. @@ -623,13 +623,13 @@ And now we can check inference using only IR models. .. code:: ipython3 result_wav_names = [] - + with torch.no_grad(): for line in tqdm(zip(srcs, tgts)): src, tgt = line - + output_audio = synthesize_audio(src, tgt) - + timestamp = time.strftime("%m-%d_%H-%M", time.localtime()) result_name = f"{timestamp}.wav" result_wav_names.append(result_name) @@ -642,7 +642,7 @@ And now we can check inference using only IR models. .. parsed-literal:: - 2it [00:01, 1.31it/s] + 2it [00:01, 1.21it/s] Result audio files should be available in ‘outputs/freevc’ and you can @@ -654,7 +654,7 @@ Source audio (source of text): .. code:: ipython3 import IPython.display as ipd - + ipd.Audio(srcs[0]) @@ -662,7 +662,7 @@ Source audio (source of text): .. raw:: html - +