diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index 20b7fc4de5476d..997db08fb2b4fa 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -243,6 +243,7 @@ jobs: -DCMAKE_C_LINKER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang + -DENABLE_SYSTEM_SNAPPY=ON -DCPACK_GENERATOR=$(CMAKE_CPACK_GENERATOR) -DBUILD_nvidia_plugin=OFF -S $(REPO_DIR) @@ -389,17 +390,16 @@ jobs: - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_conditional_compilation_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ConditionalCompilation.xml displayName: 'Conditional Compilation Tests' - - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/paddle_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-PaddleTests.xml - displayName: 'Paddle Tests' - - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_ir_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-IRFrontend.xml displayName: 'IR Frontend Tests' - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ONNXFrontend.xml displayName: 'ONNX Frontend Tests' + # TODO Reenable PDPD after paddlepaddle==2.5.0 with compliant protobuf is released (ticket 95904) - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/paddle_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-Paddle.xml displayName: 'Paddle Frontend UT' + enabled: 'false' - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_tensorflow_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-Tensorflow.xml displayName: 'TensorFlow Frontend Unit Tests' diff --git a/.ci/azure/linux_debian.yml b/.ci/azure/linux_debian.yml index 6a636be467ca01..93db8c0ca71446 100644 --- a/.ci/azure/linux_debian.yml +++ b/.ci/azure/linux_debian.yml @@ -159,6 +159,7 @@ jobs: -DENABLE_TESTS=ON -DENABLE_FASTER_BUILD=ON -DENABLE_STRICT_DEPENDENCIES=OFF + -DENABLE_SYSTEM_SNAPPY=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_LINKER_LAUNCHER=ccache @@ -304,11 +305,12 @@ jobs: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'ONNX Frontend Tests' - - script: | - $(INSTALL_TEST_DIR)/paddle_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-Paddle.xml + # TODO Reenable PDPD after paddlepaddle==2.5.0 with compliant protobuf is released (ticket 95904) + - script: $(INSTALL_TEST_DIR)/paddle_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-Paddle.xml env: LD_LIBRARY_PATH: $(INSTALL_TEST_DIR) displayName: 'Paddle Frontend UT' + enabled: 'false' - script: $(INSTALL_TEST_DIR)/ov_tensorflow_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-Tensorflow.xml env: diff --git a/.ci/azure/windows.yml b/.ci/azure/windows.yml index cb42e0f8e30814..c9b8036e83d253 100644 --- a/.ci/azure/windows.yml +++ b/.ci/azure/windows.yml @@ -267,8 +267,9 @@ jobs: - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ONNXFrontend.xml displayName: 'ONNX Frontend Tests' - - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\paddle_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-Paddle.xml - displayName: 'Paddle Frontend UT' + # TODO Reenable PDPD after paddlepaddle==2.5.0 with compliant protobuf is released (ticket 95904) + #- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\paddle_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-Paddle.xml + # displayName: 'Paddle Frontend UT' - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_tensorflow_frontend_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-Tensorflow.xml displayName: 'TensorFlow Frontend Unit Tests' diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 137d5131cdcf66..74661a77af5794 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: # Python product dependencies # - # Python API requirements + # Python API, Frontends - package-ecosystem: pip directory: "/src/bindings/python/" schedule: @@ -17,97 +17,33 @@ updates: assignees: - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" - versioning-strategy: increase-if-necessary - - # Model Optimizer requirements - - package-ecosystem: pip - directory: "/tools/mo" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Dubai" - open-pull-requests-limit: 3 - assignees: - - "rkazants" - - "andrei-kochin" - - "jiwaszki" - - "p-wysocki" - allow: - - dependency-name: "*" - dependency-type: "production" - versioning-strategy: increase-if-necessary - - # POT requirements - - package-ecosystem: pip - directory: "/tools/pot" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Dubai" - open-pull-requests-limit: 3 - assignees: - - "AlexKoff88" - - "KodiaqQ" - - "jiwaszki" - - "p-wysocki" - - "rkazants" - versioning-strategy: increase-if-necessary - - # benchmark_tool requirements - - package-ecosystem: pip - directory: "/tools/benchmark_tool" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Dubai" - open-pull-requests-limit: 3 - assignees: - - "Wovchena" - - "jiwaszki" - - "p-wysocki" - - "rkazants" - versioning-strategy: increase-if-necessary - - # - # Tests requirements for frontends - # - - # PaddlePaddle FE tests requirements - - package-ecosystem: pip - directory: "/src/frontends/paddle/tests/" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Shanghai" - open-pull-requests-limit: 3 - assignees: - "ceciliapeng2011" - "meiyang-intel" - - "jiwaszki" - - "p-wysocki" - - "rkazants" + - "mbencer" + - "tomdol" + - "jane-intel" versioning-strategy: increase-if-necessary - # ONNX FE tests requirements + # Tests - package-ecosystem: pip - directory: "/src/frontends/onnx/tests/" + directory: "/tests" schedule: interval: "daily" time: "09:00" timezone: "Poland" open-pull-requests-limit: 3 assignees: - - "mbencer" - - "tomdol" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary - # TensorFlow FE tests requirements + # Model Optimizer, openvino_dev and Benchmark tool - package-ecosystem: pip - directory: "/src/frontends/tensorflow/tests/" + directory: "/tools" schedule: interval: "daily" time: "09:00" @@ -115,23 +51,31 @@ updates: open-pull-requests-limit: 3 assignees: - "rkazants" + - "andrei-kochin" - "jiwaszki" - "p-wysocki" + - "akuporos" + - "Wovchena" + allow: + - dependency-name: "*" + dependency-type: "production" versioning-strategy: increase-if-necessary - # TensorFlow Lite FE tests requirements + # POT requirements - package-ecosystem: pip - directory: "/src/frontends/tensorflow_lite/tests/" + directory: "/tools/pot" schedule: interval: "daily" time: "09:00" timezone: "Asia/Dubai" open-pull-requests-limit: 3 assignees: - - "jane-intel" - - "rkazants" + - "AlexKoff88" + - "KodiaqQ" - "jiwaszki" - "p-wysocki" + - "akuporos" + - "rkazants" versioning-strategy: increase-if-necessary # @@ -149,6 +93,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary @@ -163,6 +108,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary @@ -177,6 +123,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary @@ -191,6 +138,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index 16936e2a3559e0..f0a34186ce4537 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -11,7 +11,7 @@ env: DOXYREST_VER: '2.1.3' concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} cancel-in-progress: true jobs: @@ -25,7 +25,7 @@ jobs: lfs: true - name: Install apt-get dependencies - uses: awalsh128/cache-apt-pkgs-action@v1.2.4 + uses: awalsh128/cache-apt-pkgs-action@v1.3.0 with: packages: graphviz texlive liblua5.2-0 libclang1-9 libclang-cpp9 version: 3.0 diff --git a/.github/workflows/code_snippets.yml b/.github/workflows/code_snippets.yml index 783202d1c908bf..559d3c064a7942 100644 --- a/.github/workflows/code_snippets.yml +++ b/.github/workflows/code_snippets.yml @@ -31,7 +31,7 @@ jobs: lfs: true - name: Install OpenCL - uses: awalsh128/cache-apt-pkgs-action@v1.2.4 + uses: awalsh128/cache-apt-pkgs-action@v1.3.0 if: runner.os == 'Linux' with: packages: ocl-icd-opencl-dev opencl-headers diff --git a/.gitmodules b/.gitmodules index c166b67a762d7c..09894aceba7475 100644 --- a/.gitmodules +++ b/.gitmodules @@ -66,3 +66,6 @@ [submodule "thirdparty/flatbuffers/flatbuffers"] path = thirdparty/flatbuffers/flatbuffers url = https://github.com/google/flatbuffers.git +[submodule "thirdparty/snappy"] + path = thirdparty/snappy + url = https://github.com/google/snappy.git diff --git a/CMakeLists.txt b/CMakeLists.txt index de4f80b8cf9882..92bb7b0f19b4e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,10 @@ else() endif() endif() +if(POLICY CMP0091) + cmake_policy(SET CMP0091 NEW) # Enables use of MSVC_RUNTIME_LIBRARY +endif() + project(OpenVINO DESCRIPTION "OpenVINO toolkit") find_package(IEDevScripts REQUIRED diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 0cef392e167f95..f548332ebd8ccf 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -97,10 +97,10 @@ function(ov_download_tbb) if(WIN32 AND X86_64) # TODO: add target_path to be platform specific as well, to avoid following if RESOLVE_DEPENDENCY(TBB - ARCHIVE_WIN "tbb2020_617e9a71_win.zip" + ARCHIVE_WIN "oneapi-tbb-2021.2.1-win.zip" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "01cac3cc48705bd52b83a6e1fa1ed95c708928be76160f5b9c5c37f954d56df4" + SHA256 "d81591673bd7d3d9454054642f8ef799e1fdddc7b4cee810a95e6130eb7323d4" USE_NEW_LOCATION TRUE) elseif(ANDROID AND X86_64) RESOLVE_DEPENDENCY(TBB @@ -110,10 +110,10 @@ function(ov_download_tbb) SHA256 "f42d084224cc2d643314bd483ad180b081774608844000f132859fca3e9bf0ce") elseif(LINUX AND X86_64) RESOLVE_DEPENDENCY(TBB - ARCHIVE_LIN "tbb2020_617e9a71_lin_strip.tgz" + ARCHIVE_LIN "oneapi-tbb-2021.2.1-lin.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "e7a38f68059fb36de8b59d40b283a849f26275e34a58d2acadfdb84d49e31b9b" + SHA256 "0a56f73baaa40d72e06949ea6d593ae63a19f7580ce71c08287c1f59d2e5b988" USE_NEW_LOCATION TRUE) elseif(YOCTO_AARCH64) RESOLVE_DEPENDENCY(TBB @@ -123,10 +123,10 @@ function(ov_download_tbb) SHA256 "321261ff2eda6d4568a473cb883262bce77a93dac599f7bd65d2918bdee4d75b") elseif(APPLE AND X86_64) RESOLVE_DEPENDENCY(TBB - ARCHIVE_MAC "tbb2020_617e9a71_mac.tgz" + ARCHIVE_MAC "oneapi-tbb-2021.2.1-mac.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "67a44b695bef3348416eaf5bf2baca2b1401576c0e09c394304eba1e0eee96cd" + SHA256 "c57ce4b97116cd3093c33e6dcc147fb1bbb9678d0ee6c61a506b2bfe773232cb" USE_NEW_LOCATION TRUE) else() message(WARNING "Prebuilt TBB is not available on current platform") @@ -184,10 +184,11 @@ function(ov_download_tbbbind_2_5) USE_NEW_LOCATION TRUE) elseif(LINUX AND X86_64) RESOLVE_DEPENDENCY(TBBBIND_2_5 - ARCHIVE_LIN "tbbbind_2_5_static_lin_v2.tgz" + ARCHIVE_LIN "tbbbind_2_5_static_lin_v3.tgz" TARGET_PATH "${TEMP}/tbbbind_2_5" ENVIRONMENT "TBBBIND_2_5_ROOT" - SHA256 "865e7894c58402233caf0d1b288056e0e6ab2bf7c9d00c9dc60561c484bc90f4") + SHA256 "d39deb262c06981b5e2d2e3c593e9fc9be62ce4feb91dd4e648e92753659a6b3" + USE_NEW_LOCATION TRUE) else() # TMP: for Apple Silicon TBB does not provide TBBBind if(NOT (APPLE AND AARCH64)) diff --git a/cmake/developer_package/check_python_requirements.py b/cmake/developer_package/check_python_requirements.py new file mode 100644 index 00000000000000..8db944ee6e4888 --- /dev/null +++ b/cmake/developer_package/check_python_requirements.py @@ -0,0 +1,52 @@ +import pkg_resources +import re +import os + + +def check_python_requirements(requirements_path: str) -> None: + """ + Checks if the requirements defined in `requirements_path` are installed + in the active Python environment, while also taking constraints.txt files + into account. + """ + + constraints = {} + constraints_path = None + requirements = [] + + # read requirements and find constraints file + with open(requirements_path) as f: + raw_requirements = f.readlines() + for line in raw_requirements: + if line.startswith("-c"): + constraints_path = os.path.join(os.path.dirname(requirements_path), line.split(' ')[1][:-1]) + + # read constraints if they exist + if constraints_path: + with open(constraints_path) as f: + raw_constraints = f.readlines() + for line in raw_constraints: + if line.startswith("#") or line=="\n": + continue + line = line.replace("\n", "") + package, delimiter, constraint = re.split("(~|=|<|>|;)", line, maxsplit=1) + if constraints.get(package) is None: + constraints[package] = [delimiter + constraint] + else: + constraints[package].extend([delimiter + constraint]) + for line in raw_requirements: + if line.startswith(("#", "-c")): + continue + line = line.replace("\n", "") + if re.search("\W", line): + requirements.append(line) + else: + constraint = constraints.get(line) + if constraint: + for marker in constraint: + requirements.append(line+marker) + else: + requirements.append(line) + else: + requirements = raw_requirements + pkg_resources.require(requirements) diff --git a/cmake/developer_package/compile_flags/os_flags.cmake b/cmake/developer_package/compile_flags/os_flags.cmake index 03564d6b142671..a5dbb05de41685 100644 --- a/cmake/developer_package/compile_flags/os_flags.cmake +++ b/cmake/developer_package/compile_flags/os_flags.cmake @@ -455,14 +455,6 @@ else() endif() endif() -# if(OV_COMPILER_IS_CLANG) -# ie_add_compiler_flags(-Wshorten-64-to-32) -# endif() -# TODO -if(OV_COMPILER_IS_CLANG) - ie_add_compiler_flags(-Wno-delete-non-abstract-non-virtual-dtor) -endif() - check_cxx_compiler_flag("-Wsuggest-override" SUGGEST_OVERRIDE_SUPPORTED) if(SUGGEST_OVERRIDE_SUPPORTED) set(CMAKE_CXX_FLAGS "-Wsuggest-override ${CMAKE_CXX_FLAGS}") diff --git a/cmake/developer_package/compile_flags/sdl.cmake b/cmake/developer_package/compile_flags/sdl.cmake index 7e992f6b9cdd9f..3973ba74db655d 100644 --- a/cmake/developer_package/compile_flags/sdl.cmake +++ b/cmake/developer_package/compile_flags/sdl.cmake @@ -50,6 +50,9 @@ else() if(ENABLE_INTEGRITYCHECK) set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /INTEGRITYCHECK") endif() + if(ENABLE_QSPECTRE) + ie_add_compiler_flags(/Qspectre) + endif() endif() set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${IE_C_CXX_FLAGS}") diff --git a/cmake/developer_package/features.cmake b/cmake/developer_package/features.cmake index cc1406a24c1fb9..5c810c713c6a31 100644 --- a/cmake/developer_package/features.cmake +++ b/cmake/developer_package/features.cmake @@ -8,7 +8,7 @@ include(target_flags) # FIXME: there are compiler failures with LTO and Cross-Compile toolchains. Disabling for now, but # this must be addressed in a proper way ie_dependent_option (ENABLE_LTO "Enable Link Time Optimization" OFF - "LINUX OR (APPLE AND AARCH64);EMSCRIPTEN OR NOT CMAKE_CROSSCOMPILING;CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9" OFF) + "LINUX;EMSCRIPTEN OR NOT CMAKE_CROSSCOMPILING;CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9" OFF) ie_option (OS_FOLDER "create OS dedicated folder in output" OFF) @@ -26,6 +26,8 @@ endif() ie_option (CMAKE_COMPILE_WARNING_AS_ERROR "Enable warnings as errors" ${CMAKE_COMPILE_WARNING_AS_ERROR_DEFAULT}) +ie_dependent_option (ENABLE_QSPECTRE "Enable Qspectre mitigation" OFF "CMAKE_CXX_COMPILER_ID STREQUAL MSVC" OFF) + ie_dependent_option (ENABLE_INTEGRITYCHECK "build DLLs with /INTEGRITYCHECK flag" OFF "CMAKE_CXX_COMPILER_ID STREQUAL MSVC" OFF) ie_option (ENABLE_SANITIZER "enable checking memory errors via AddressSanitizer" OFF) diff --git a/cmake/developer_package/native_compile.cmake b/cmake/developer_package/native_compile.cmake index 7b23986ddb87f4..638d2b15cf1a3b 100644 --- a/cmake/developer_package/native_compile.cmake +++ b/cmake/developer_package/native_compile.cmake @@ -72,6 +72,15 @@ function(ov_native_compile_external_project) list(APPEND ARG_CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") endif() + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21) + if(DEFINED CMAKE_CXX_LINKER_LAUNCHER) + list(APPEND ARG_CMAKE_ARGS "-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}") + endif() + if(DEFINED CMAKE_C_LINKER_LAUNCHER) + list(APPEND ARG_CMAKE_ARGS "-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}") + endif() + endif() + ExternalProject_Add(${ARG_TARGET_NAME} # Directory Options SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" @@ -84,8 +93,6 @@ function(ov_native_compile_external_project) CMAKE_ARGS "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}" "-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}" - "-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}" - "-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}" "-DCMAKE_CXX_FLAGS=${compile_flags}" "-DCMAKE_C_FLAGS=${compile_flags}" "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW" diff --git a/cmake/developer_package/plugins/create_plugins_hpp.cmake b/cmake/developer_package/plugins/create_plugins_hpp.cmake index cddcad738470ca..1fedf858ce58ca 100644 --- a/cmake/developer_package/plugins/create_plugins_hpp.cmake +++ b/cmake/developer_package/plugins/create_plugins_hpp.cmake @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # -foreach(var IE_DEVICE_MAPPING IE_PLUGINS_HPP_HEADER IE_PLUGINS_HPP_HEADER_IN) +foreach(var OV_DEVICE_MAPPING BUILD_SHARED_LIBS OV_PLUGINS_HPP_HEADER OV_PLUGINS_HPP_HEADER_IN) if(NOT DEFINED ${var}) message(FATAL_ERROR "${var} is required, but not defined") endif() @@ -10,29 +10,15 @@ endforeach() # configure variables -set(IE_PLUGINS_DECLARATIONS "") -set(IE_PLUGINS_MAP_DEFINITION +set(OV_PLUGINS_DECLARATIONS "") +set(OV_PLUGINS_MAP_DEFINITION " static const std::map plugins_hpp = {") -foreach(dev_map IN LISTS IE_DEVICE_MAPPING) +foreach(dev_map IN LISTS OV_DEVICE_MAPPING) string(REPLACE ":" ";" dev_map "${dev_map}") list(GET dev_map 0 mapped_dev_name) list(GET dev_map 1 actual_dev_name) - # common - set(_IE_CREATE_PLUGIN_FUNC "CreatePluginEngine${actual_dev_name}") - set(_IE_CREATE_EXTENSION_FUNC "CreateExtensionShared${actual_dev_name}") - - # declarations - set(IE_PLUGINS_DECLARATIONS "${IE_PLUGINS_DECLARATIONS} -IE_DEFINE_PLUGIN_CREATE_FUNCTION_DECLARATION(${_IE_CREATE_PLUGIN_FUNC});") - if(${actual_dev_name}_AS_EXTENSION) - set(IE_PLUGINS_DECLARATIONS "${IE_PLUGINS_DECLARATIONS} -IE_DEFINE_EXTENSION_CREATE_FUNCTION_DECLARATION(${_IE_CREATE_EXTENSION_FUNC});") - else() - set(_IE_CREATE_EXTENSION_FUNC "nullptr") - endif() - # definitions set(dev_config "{") if(${mapped_dev_name}_CONFIG) @@ -48,11 +34,31 @@ IE_DEFINE_EXTENSION_CREATE_FUNCTION_DECLARATION(${_IE_CREATE_EXTENSION_FUNC});") endif() set(dev_config "${dev_config}}") - set(IE_PLUGINS_MAP_DEFINITION "${IE_PLUGINS_MAP_DEFINITION} - { \"${mapped_dev_name}\", Value { ${_IE_CREATE_PLUGIN_FUNC}, ${_IE_CREATE_EXTENSION_FUNC}, ${dev_config} } },") + + if(NOT BUILD_SHARED_LIBS) + # common + set(_OV_CREATE_PLUGIN_FUNC "CreatePluginEngine${actual_dev_name}") + set(_OV_CREATE_EXTENSION_FUNC "CreateExtensionShared${actual_dev_name}") + + # declarations + set(OV_PLUGINS_DECLARATIONS "${OV_PLUGINS_DECLARATIONS} + IE_DEFINE_PLUGIN_CREATE_FUNCTION_DECLARATION(${_OV_CREATE_PLUGIN_FUNC});") + if(${actual_dev_name}_AS_EXTENSION) + set(OV_PLUGINS_DECLARATIONS "${OV_PLUGINS_DECLARATIONS} + IE_DEFINE_EXTENSION_CREATE_FUNCTION_DECLARATION(${_OV_CREATE_EXTENSION_FUNC});") + else() + set(_OV_CREATE_EXTENSION_FUNC "nullptr") + endif() + + set(OV_PLUGINS_MAP_DEFINITION "${OV_PLUGINS_MAP_DEFINITION} + { \"${mapped_dev_name}\", Value { ${_OV_CREATE_PLUGIN_FUNC}, ${_OV_CREATE_EXTENSION_FUNC}, ${dev_config} } },") + else() + set(OV_PLUGINS_MAP_DEFINITION "${OV_PLUGINS_MAP_DEFINITION} + { \"${mapped_dev_name}\", Value { \"${actual_dev_name}\", ${dev_config} } },") + endif() endforeach() -set(IE_PLUGINS_MAP_DEFINITION "${IE_PLUGINS_MAP_DEFINITION} +set(OV_PLUGINS_MAP_DEFINITION "${OV_PLUGINS_MAP_DEFINITION} };\n") -configure_file("${IE_PLUGINS_HPP_HEADER_IN}" "${IE_PLUGINS_HPP_HEADER}" @ONLY) +configure_file("${OV_PLUGINS_HPP_HEADER_IN}" "${OV_PLUGINS_HPP_HEADER}" @ONLY) diff --git a/cmake/developer_package/plugins/plugins.cmake b/cmake/developer_package/plugins/plugins.cmake index b4cfe20bd024e5..80a8266e83292a 100644 --- a/cmake/developer_package/plugins/plugins.cmake +++ b/cmake/developer_package/plugins/plugins.cmake @@ -49,10 +49,6 @@ function(ie_add_plugin) # create and configure target if(NOT IE_PLUGIN_PSEUDO_PLUGIN_FOR) - if(IE_PLUGIN_VERSION_DEFINES_FOR) - addVersionDefines(${IE_PLUGIN_VERSION_DEFINES_FOR} CI_BUILD_NUMBER) - endif() - set(input_files ${IE_PLUGIN_SOURCES}) foreach(obj_lib IN LISTS IE_PLUGIN_OBJECT_LIBRARIES) list(APPEND input_files $) @@ -67,6 +63,10 @@ function(ie_add_plugin) add_library(${IE_PLUGIN_NAME} ${library_type} ${input_files}) + if(IE_PLUGIN_VERSION_DEFINES_FOR) + ov_add_version_defines(${IE_PLUGIN_VERSION_DEFINES_FOR} ${IE_PLUGIN_NAME}) + endif() + target_compile_definitions(${IE_PLUGIN_NAME} PRIVATE IMPLEMENT_INFERENCE_ENGINE_PLUGIN) if(NOT BUILD_SHARED_LIBS) # to distinguish functions creating plugin objects @@ -113,7 +113,7 @@ function(ie_add_plugin) if(IE_PLUGIN_PSEUDO_DEVICE) set(plugin_hidden HIDDEN) endif() - ie_cpack_add_component(${install_component} + ie_cpack_add_component(${install_component} DISPLAY_NAME "${IE_PLUGIN_DEVICE_NAME} runtime" DESCRIPTION "${IE_PLUGIN_DEVICE_NAME} runtime" ${plugin_hidden} @@ -227,16 +227,18 @@ macro(ie_register_plugins_dynamic) # Combine all .xml files into plugins.xml - add_custom_command(TARGET ${IE_REGISTER_MAIN_TARGET} POST_BUILD - COMMAND - "${CMAKE_COMMAND}" - -D "CMAKE_SHARED_MODULE_PREFIX=${CMAKE_SHARED_MODULE_PREFIX}" - -D "IE_CONFIG_OUTPUT_FILE=${config_output_file}" - -D "IE_CONFIGS_DIR=${CMAKE_BINARY_DIR}/plugins" - -P "${IEDevScripts_DIR}/plugins/register_plugin_cmake.cmake" - COMMENT - "Registering plugins to plugins.xml config file" - VERBATIM) + if(ENABLE_PLUGINS_XML) + add_custom_command(TARGET ${IE_REGISTER_MAIN_TARGET} POST_BUILD + COMMAND + "${CMAKE_COMMAND}" + -D "CMAKE_SHARED_MODULE_PREFIX=${CMAKE_SHARED_MODULE_PREFIX}" + -D "IE_CONFIG_OUTPUT_FILE=${config_output_file}" + -D "IE_CONFIGS_DIR=${CMAKE_BINARY_DIR}/plugins" + -P "${IEDevScripts_DIR}/plugins/register_plugin_cmake.cmake" + COMMENT + "Registering plugins to plugins.xml config file" + VERBATIM) + endif() endmacro() # @@ -279,13 +281,9 @@ function(ie_target_link_plugins TARGET_NAME) endfunction() # -# ie_generate_plugins_hpp() +# ov_generate_plugins_hpp() # -function(ie_generate_plugins_hpp) - if(BUILD_SHARED_LIBS) - return() - endif() - +function(ov_generate_plugins_hpp) set(device_mapping) set(device_configs) set(as_extension) @@ -296,17 +294,23 @@ function(ie_generate_plugins_hpp) message(FATAL_ERROR "Unexpected error, please, contact developer of this script") endif() - # create device mapping: preudo device => actual device + # create device mapping: pseudo device => actual device list(GET name 0 device_name) - if(${device_name}_PSEUDO_PLUGIN_FOR) - list(APPEND device_mapping "${device_name}:${${device_name}_PSEUDO_PLUGIN_FOR}") + if(BUILD_SHARED_LIBS) + list(GET name 1 library_name) + ie_plugin_get_file_name(${library_name} library_name) + list(APPEND device_mapping "${device_name}:${library_name}") else() - list(APPEND device_mapping "${device_name}:${device_name}") - endif() + if(${device_name}_PSEUDO_PLUGIN_FOR) + list(APPEND device_mapping "${device_name}:${${device_name}_PSEUDO_PLUGIN_FOR}") + else() + list(APPEND device_mapping "${device_name}:${device_name}") + endif() - # register plugin as extension - if(${device_name}_AS_EXTENSION) - list(APPEND as_extension -D "${device_name}_AS_EXTENSION=ON") + # register plugin as extension + if(${device_name}_AS_EXTENSION) + list(APPEND as_extension -D "${device_name}_AS_EXTENSION=ON") + endif() endif() # add default plugin config options @@ -317,21 +321,22 @@ function(ie_generate_plugins_hpp) endif() endforeach() - # add plugins to libraries including ie_plugins.hpp + # add plugins to libraries including ov_plugins.hpp ie_target_link_plugins(openvino) if(TARGET inference_engine_s) ie_target_link_plugins(inference_engine_s) endif() - set(ie_plugins_hpp "${CMAKE_BINARY_DIR}/src/inference/ie_plugins.hpp") + set(ov_plugins_hpp "${CMAKE_BINARY_DIR}/src/inference/ov_plugins.hpp") set(plugins_hpp_in "${IEDevScripts_DIR}/plugins/plugins.hpp.in") - add_custom_command(OUTPUT "${ie_plugins_hpp}" + add_custom_command(OUTPUT "${ov_plugins_hpp}" COMMAND "${CMAKE_COMMAND}" - -D "IE_DEVICE_MAPPING=${device_mapping}" - -D "IE_PLUGINS_HPP_HEADER_IN=${plugins_hpp_in}" - -D "IE_PLUGINS_HPP_HEADER=${ie_plugins_hpp}" + -D "BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}" + -D "OV_DEVICE_MAPPING=${device_mapping}" + -D "OV_PLUGINS_HPP_HEADER_IN=${plugins_hpp_in}" + -D "OV_PLUGINS_HPP_HEADER=${ov_plugins_hpp}" ${device_configs} ${as_extension} -P "${IEDevScripts_DIR}/plugins/create_plugins_hpp.cmake" @@ -339,28 +344,11 @@ function(ie_generate_plugins_hpp) "${plugins_hpp_in}" "${IEDevScripts_DIR}/plugins/create_plugins_hpp.cmake" COMMENT - "Generate ie_plugins.hpp for static build" + "Generate ov_plugins.hpp for build" VERBATIM) # for some reason dependency on source files does not work # so, we have to use explicit target and make it dependency for inference_engine - add_custom_target(_ie_plugins_hpp DEPENDS ${ie_plugins_hpp}) - add_dependencies(inference_engine_obj _ie_plugins_hpp) - - # add dependency for object files - get_target_property(sources inference_engine_obj SOURCES) - foreach(source IN LISTS sources) - if("${source}" MATCHES "\\$\\") - # object library - set(obj_library ${CMAKE_MATCH_1}) - get_target_property(obj_sources ${obj_library} SOURCES) - list(APPEND all_sources ${obj_sources}) - else() - # usual source - list(APPEND all_sources ${source}) - endif() - endforeach() - - # add dependency on header file generation for all inference_engine source files - set_source_files_properties(${all_sources} PROPERTIES OBJECT_DEPENDS ${ie_plugins_hpp}) + add_custom_target(_ov_plugins_hpp DEPENDS ${ov_plugins_hpp}) + add_dependencies(inference_engine_obj _ov_plugins_hpp) endfunction() diff --git a/cmake/developer_package/plugins/plugins.hpp.in b/cmake/developer_package/plugins/plugins.hpp.in index fa8119756b82e8..224f77c8cb980b 100644 --- a/cmake/developer_package/plugins/plugins.hpp.in +++ b/cmake/developer_package/plugins/plugins.hpp.in @@ -4,9 +4,14 @@ #pragma once +#include +#include + +#ifdef OPENVINO_STATIC_LIBRARY + #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" -@IE_PLUGINS_DECLARATIONS@ +@OV_PLUGINS_DECLARATIONS@ struct Value { InferenceEngine::CreatePluginEngineFunc * m_create_plugin_func; @@ -14,10 +19,20 @@ struct Value { std::map m_default_config; }; +#else + +struct Value { + std::string m_plugin_path; + std::map m_default_config; +}; + +#endif + using Key = std::string; using PluginsStaticRegistry = std::map; -inline const std::map getStaticPluginsRegistry() { -@IE_PLUGINS_MAP_DEFINITION@ + +inline const std::map getCompiledPluginsRegistry() { +@OV_PLUGINS_MAP_DEFINITION@ return plugins_hpp; } diff --git a/cmake/developer_package/python_requirements.cmake b/cmake/developer_package/python_requirements.cmake index d187669d6ffbe9..3f5dcd6bd0d5dd 100644 --- a/cmake/developer_package/python_requirements.cmake +++ b/cmake/developer_package/python_requirements.cmake @@ -97,7 +97,11 @@ function(ov_check_pip_packages) if(PYTHONINTERP_FOUND) execute_process( - COMMAND ${PYTHON_EXECUTABLE} -c "import pkg_resources ; pkg_resources.require(open('${ARG_REQUIREMENTS_FILE}', mode='r'))" + COMMAND ${PYTHON_EXECUTABLE} -c " +from check_python_requirements import check_python_requirements ; +check_python_requirements('${ARG_REQUIREMENTS_FILE}') ; + " + WORKING_DIRECTORY "${IEDevScripts_DIR}" RESULT_VARIABLE EXIT_CODE OUTPUT_VARIABLE OUTPUT_TEXT ERROR_VARIABLE ERROR_TEXT) diff --git a/cmake/developer_package/version.cmake b/cmake/developer_package/version.cmake index 35f1616e5e89e2..9a461c43a73499 100644 --- a/cmake/developer_package/version.cmake +++ b/cmake/developer_package/version.cmake @@ -185,6 +185,46 @@ macro (addVersionDefines FILE) unset(__version_file) endmacro() +macro (ov_add_version_defines FILE TARGET) + set(__version_file ${FILE}) + if(NOT IS_ABSOLUTE ${__version_file}) + set(__version_file "${CMAKE_CURRENT_SOURCE_DIR}/${__version_file}") + endif() + if(NOT EXISTS ${__version_file}) + message(FATAL_ERROR "${FILE} does not exists in current source directory") + endif() + _remove_source_from_target(${TARGET} ${FILE}) + _remove_source_from_target(${TARGET} ${__version_file}) + if (BUILD_SHARED_LIBS) + add_library(${TARGET}_version OBJECT ${__version_file}) + else() + add_library(${TARGET}_version STATIC ${__version_file}) + endif() + if(SUGGEST_OVERRIDE_SUPPORTED) + set_source_files_properties(${__version_file} + PROPERTIES COMPILE_OPTIONS -Wno-suggest-override) + endif() + + target_compile_definitions(${TARGET}_version PRIVATE + CI_BUILD_NUMBER=\"${CI_BUILD_NUMBER}\" + $ + $) + target_include_directories(${TARGET}_version PRIVATE + $ + $) + target_link_libraries(${TARGET}_version PRIVATE + $) + target_compile_options(${TARGET}_version PRIVATE + $ + $) + set_target_properties(${TARGET}_version + PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE + $) + + target_sources(${TARGET} PRIVATE $) + unset(__version_file) +endmacro() + function(ov_add_library_version library) if(NOT DEFINED OpenVINO_SOVERSION) message(FATAL_ERROR "Internal error: OpenVINO_SOVERSION is not defined") diff --git a/cmake/extra_modules.cmake b/cmake/extra_modules.cmake index 7b843341c11159..a5b1cd22e82ccf 100644 --- a/cmake/extra_modules.cmake +++ b/cmake/extra_modules.cmake @@ -169,9 +169,9 @@ ov_generate_dev_package_config() # with all imported developer targets register_extra_modules() -# for static libraries case we need to generate final ie_plugins.hpp +# for static libraries case we need to generate final ov_plugins.hpp # with all the information about plugins -ie_generate_plugins_hpp() +ov_generate_plugins_hpp() # used for static build ov_generate_frontends_hpp() diff --git a/cmake/features.cmake b/cmake/features.cmake index 17c5ccc1b3c7e5..402c848a2b2bc8 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -94,6 +94,8 @@ ie_option (ENABLE_HETERO "Enables Hetero Device Plugin" ON) ie_option (ENABLE_TEMPLATE "Enable template plugin" ON) +ie_dependent_option (ENABLE_PLUGINS_XML "Generate plugins.xml configuration file or not" OFF "NOT BUILD_SHARED_LIBS" OFF) + ie_dependent_option (GAPI_TEST_PERF "if GAPI unit tests should examine performance" OFF "ENABLE_TESTS;ENABLE_GAPI_PREPROCESSING" OFF) ie_dependent_option (ENABLE_DATA "fetch models from testdata repo" ON "ENABLE_FUNCTIONAL_TESTS;NOT ANDROID" OFF) @@ -152,13 +154,16 @@ ie_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON) ie_option(ENABLE_OV_PYTORCH_FRONTEND "Enable PyTorch FrontEnd" ON) ie_option(ENABLE_OV_TF_FRONTEND "Enable TensorFlow FrontEnd" ON) ie_option(ENABLE_OV_TF_LITE_FRONTEND "Enable TensorFlow Lite FrontEnd" ON) -ie_dependent_option(ENABLE_SYSTEM_PROTOBUF "Use system protobuf" OFF - "ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_TF_FRONTEND;BUILD_SHARED_LIBS" OFF) ie_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON) -ie_dependent_option(ENABLE_SYSTEM_FLATBUFFERS "Use system flatbuffers" ON + +ie_dependent_option(ENABLE_SNAPPY_COMPRESSION "Enables compression support for TF FE" ON + "ENABLE_OV_TF_FRONTEND" ON) +ie_dependent_option(ENABLE_SYSTEM_PROTOBUF "Enables use of system protobuf" OFF + "ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_TF_FRONTEND;BUILD_SHARED_LIBS" OFF) +ie_dependent_option(ENABLE_SYSTEM_FLATBUFFERS "Enables use of system flatbuffers" ON "ENABLE_OV_TF_LITE_FRONTEND" OFF) +ie_dependent_option(ENABLE_SYSTEM_SNAPPY "Enables use of system version of snappy" OFF "ENABLE_SNAPPY_COMPRESSION;BUILD_SHARED_LIBS" OFF) -ie_dependent_option(ENABLE_OV_CORE_UNIT_TESTS "Enables OpenVINO core unit tests" ON "ENABLE_TESTS" OFF) ie_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF) if(NOT BUILD_SHARED_LIBS AND ENABLE_OV_TF_FRONTEND) diff --git a/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in b/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in index 24238be0604c1b..d530ea36d1d9c8 100644 --- a/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in +++ b/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in @@ -28,6 +28,9 @@ foreach(option IN LISTS ov_options) endforeach() message(" ") +# activate generation of plugins.xml +set(ENABLE_PLUGINS_XML ON) + # for samples in 3rd party projects if(ENABLE_SAMPLES) set_and_check(gflags_DIR "@gflags_BINARY_DIR@") diff --git a/docs/Documentation/deployment_guide_introduction.md b/docs/Documentation/deployment_guide_introduction.md index 4e3162d6de5383..6496a3cf494250 100644 --- a/docs/Documentation/deployment_guide_introduction.md +++ b/docs/Documentation/deployment_guide_introduction.md @@ -9,12 +9,9 @@ Run and Deploy Locally Deploy via Model Serving -@endsphinxdirective - Once you have a model that meets both OpenVINO™ and your requirements, you can choose how to deploy it with your application. -@sphinxdirective .. panels:: :doc:`Deploy via OpenVINO Runtime ` @@ -30,8 +27,7 @@ Once you have a model that meets both OpenVINO™ and your requirements, you can Deployment via OpenVINO Model Server allows the application to connect to the inference server set up remotely. This way inference can use external resources instead of those available to the application itself. -@endsphinxdirective - +Apart from the default deployment options, you may also :doc:`deploy your application for the TensorFlow framework with OpenVINO Integration ` -Apart from the default deployment options, you may also [deploy your application for the TensorFlow framework with OpenVINO Integration](./openvino_ecosystem_ovtf.md). +@endsphinxdirective \ No newline at end of file diff --git a/docs/Documentation/model_introduction.md b/docs/Documentation/model_introduction.md index fb094a4578e6d2..5d524718591828 100644 --- a/docs/Documentation/model_introduction.md +++ b/docs/Documentation/model_introduction.md @@ -9,22 +9,23 @@ openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide omz_tools_downloader -@endsphinxdirective - -Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as OpenVINO's [Open Model Zoo](../model_zoo.md). +Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as OpenVINO's :doc:`Open Model Zoo `. -[OpenVINO™ supports several model formats](../MO_DG/prepare_model/convert_model/supported_model_formats.md) and allows to convert them to it's own, OpenVINO IR, providing a tool dedicated to this task. +:doc:`OpenVINO™ supports several model formats ` and allows to convert them to it's own, OpenVINO IR, providing a tool dedicated to this task. -[Model Optimizer](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) reads the original model and creates the OpenVINO IR model (.xml and .bin files) so that inference can ultimately be performed without delays due to format conversion. Optionally, Model Optimizer can adjust the model to be more suitable for inference, for example, by [alternating input shapes](../MO_DG/prepare_model/convert_model/Converting_Model.md), [embedding preprocessing](../MO_DG/prepare_model/Additional_Optimizations.md) and [cutting training parts off](../MO_DG/prepare_model/convert_model/Cutting_Model.md). +:doc:`Model Optimizer ` reads the original model and creates the OpenVINO IR model (.xml and .bin files) so that inference can ultimately be performed without delays due to format conversion. Optionally, Model Optimizer can adjust the model to be more suitable for inference, for example, by :doc:`alternating input shapes `, :doc:`embedding preprocessing ` and :doc:`cutting training parts off `. The approach to fully convert a model is considered the default choice, as it allows the full extent of OpenVINO features. The OpenVINO IR model format is used by other conversion and preparation tools, such as the Post-Training Optimization Tool, for further optimization of the converted model. -Conversion is not required for ONNX, PaddlePaddle, and TensorFlow models (check [TensorFlow Frontend Capabilities and Limitations](../resources/tensorflow_frontend.md)), as OpenVINO provides C++ and Python APIs for importing them to OpenVINO Runtime directly. It provides a convenient way to quickly switch from framework-based code to OpenVINO-based code in your inference application. +Conversion is not required for ONNX, PaddlePaddle, and TensorFlow models (check :doc:`TensorFlow Frontend Capabilities and Limitations `), as OpenVINO provides C++ and Python APIs for importing them to OpenVINO Runtime directly. It provides a convenient way to quickly switch from framework-based code to OpenVINO-based code in your inference application. This section describes how to obtain and prepare your model for work with OpenVINO to get the best inference results: -* [See the supported formats and how to use them in your project](../MO_DG/prepare_model/convert_model/supported_model_formats.md) -* [Convert different model formats to the OpenVINO IR format](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -* [Automate model-related tasks with Model Downloader and additional OMZ Tools](https://docs.openvino.ai/latest/omz_tools_downloader.html). -To begin with, you may want to [browse a database of models for use in your projects](../model_zoo.md). +* :doc:`See the supported formats and how to use them in your project `. +* :doc:`Convert different model formats to the OpenVINO IR format `. +* `Automate model-related tasks with Model Downloader and additional OMZ Tools `__. + +To begin with, you may want to :doc:`browse a database of models for use in your projects `. + +@endsphinxdirective diff --git a/docs/Doxyfile.config b/docs/Doxyfile.config index 3dc3c07d6409a7..16cb335679bfaa 100644 --- a/docs/Doxyfile.config +++ b/docs/Doxyfile.config @@ -1002,7 +1002,6 @@ EXCLUDE_SYMBOLS = InferenceEngine::details \ ie_api::BlobBuffer \ *impl* \ *device_name* \ - *num_requests* \ *exec_net* \ *c_config* \ *ie_core_impl* \ diff --git a/docs/Extensibility_UG/frontend_extensions.md b/docs/Extensibility_UG/frontend_extensions.md index ba24f9edc83d55..e791766a6114d6 100644 --- a/docs/Extensibility_UG/frontend_extensions.md +++ b/docs/Extensibility_UG/frontend_extensions.md @@ -116,6 +116,33 @@ So the conclusion is that each attribute of target OpenVINO operation should be This is achieved by specifying maps as arguments for `OpExtension` constructor. +### Mapping custom operations to frontends with OPENVINO_FRAMEWORK_MAP macro + +> **NOTE**: Below solution works only for ONNX and Tensorflow frontends. + +`OPENVINO_FRAMEWORK_MAP` is a macro that should be used inside OpenVINO operation's class definition and that lets you specify the mapping between this operation to a frontend operation. + +Let's consider the following example. Imagine you have an ONNX model with `CustomOp` operation (and this operation has `mode` attribute) and a Tensorflow model with `CustomOpV3` operation (this operation has `axis` attribute) and both of them can be implemented with a single OpenVINO operation `CustomOp` like follows: + +@snippet ov_extensions.cpp frontend_extension_framework_map_macro_headers +@snippet ov_extensions.cpp frontend_extension_framework_map_macro_CustomOp + +Let's take a closer look at the parameters this macro takes: +```cpp +OPENVINO_FRAMEWORK_MAP(framework, name, attributes_map, attributes_values) +``` +- `framework` - framework name. +- `name` - the framework operation name. It's optional if the OpenVINO custom operation name (that is the name that is passed as the first parameter to `OPENVINO_OP` macro) is the same as the framework operation name and both `attributes_map` and `attributes_values` are not provided. +- `attributes_map` - used to provide a mapping between OpenVINO operation attribute and framework operation attribute. Contains key-value pairs, where key is an OpenVINO operation attribute name and value is its corresponding framework operation attribute name. This parameter is optional if the number of OpenVINO operation attributes and their names match one-to-one with framework operation attributes. +- `attributes_values` - used to provide default values for OpenVINO operation attributes that are not specified in `attributes_map`. Contains key-value pairs, where key is an OpenVINO operation attribute name and the value is this attribute value. This parameter cannot be provided if `attributes_map` contains all of OpenVINO operation attributes or if `attributes_map` is not provided. + +In the example above, `OPENVINO_FRAMEWORK_MAP` is used twice. +First, OpenVINO `CustomOp` is mapped to ONNX `CustomOp` operation, `m_mode` attribute is mapped to `mode` attribute, while `m_axis` attribute gets the default value `-1`. +Secondly, OpenVINO `CustomOp` is mapped to Tensorflow `CustomOpV3` operation, `m_axis` attribute is mapped to `axis` attribute, while `m_mode` attribute gets the default value `"linear"`. + +The last step is to register this custom operation by following: +@snippet ov_extensions.cpp frontend_extension_framework_map_macro_add_extension + Mapping to Multiple Operations with ConversionExtension ####################################################### diff --git a/docs/IE_PLUGIN_DG/AsyncInferRequest.md b/docs/IE_PLUGIN_DG/AsyncInferRequest.md index d8f45d528b22c9..e45a5799deea8b 100644 --- a/docs/IE_PLUGIN_DG/AsyncInferRequest.md +++ b/docs/IE_PLUGIN_DG/AsyncInferRequest.md @@ -1,49 +1,45 @@ -# Asynchronous Inference Request {#openvino_docs_ie_plugin_dg_async_infer_request} +# Asynchronous Inference Request {#openvino_docs_ov_plugin_dg_async_infer_request} Asynchronous Inference Request runs an inference pipeline asynchronously in one or several task executors depending on a device pipeline structure. -OpenVINO Runtime Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class: +OpenVINO Runtime Plugin API provides the base ov::IAsyncInferRequest class: -- The class has the `_pipeline` field of `std::vector >`, which contains pairs of an executor and executed task. +- The class has the `m_pipeline` field of `std::vector, ov::threading::Task> >`, which contains pairs of an executor and executed task. - All executors are passed as arguments to a class constructor and they are in the running state and ready to run tasks. -- The class has the InferenceEngine::AsyncInferRequestThreadSafeDefault::StopAndWait method, which waits for `_pipeline` to finish in a class destructor. The method does not stop task executors and they are still in the running stage, because they belong to the executable network instance and are not destroyed. +- The class has the ov::IAsyncInferRequest::stop_and_wait method, which waits for `m_pipeline` to finish in a class destructor. The method does not stop task executors and they are still in the running stage, because they belong to the compiled model instance and are not destroyed. -`AsyncInferRequest` Class +AsyncInferRequest Class ------------------------ -OpenVINO Runtime Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class for a custom asynchronous inference request implementation: +OpenVINO Runtime Plugin API provides the base ov::IAsyncInferRequest class for a custom asynchronous inference request implementation: @snippet src/async_infer_request.hpp async_infer_request:header -#### Class Fields +### Class Fields -- `_inferRequest` - a reference to the [synchronous inference request](@ref openvino_docs_ov_plugin_dg_infer_request) implementation. Its methods are reused in the `AsyncInferRequest` constructor to define a device pipeline. -- `_waitExecutor` - a task executor that waits for a response from a device about device tasks completion +- `m_wait_executor` - a task executor that waits for a response from a device about device tasks completion -> **NOTE**: If a plugin can work with several instances of a device, `_waitExecutor` must be device-specific. Otherwise, having a single task executor for several devices does not allow them to work in parallel. +> **NOTE**: If a plugin can work with several instances of a device, `m_wait_executor` must be device-specific. Otherwise, having a single task executor for several devices does not allow them to work in parallel. -### `AsyncInferRequest()` +### AsyncInferRequest() -The main goal of the `AsyncInferRequest` constructor is to define a device pipeline `_pipeline`. The example below demonstrates `_pipeline` creation with the following stages: +The main goal of the `AsyncInferRequest` constructor is to define a device pipeline `m_pipeline`. The example below demonstrates `m_pipeline` creation with the following stages: -- `inferPreprocess` is a CPU compute task. -- `startPipeline` is a CPU ligthweight task to submit tasks to a remote device. -- `waitPipeline` is a CPU non-compute task that waits for a response from a remote device. -- `inferPostprocess` is a CPU compute task. +- `infer_preprocess_and_start_pipeline` is a CPU ligthweight task to submit tasks to a remote device. +- `wait_pipeline` is a CPU non-compute task that waits for a response from a remote device. +- `infer_postprocess` is a CPU compute task. @snippet src/async_infer_request.cpp async_infer_request:ctor The stages are distributed among two task executors in the following way: -- `inferPreprocess` and `startPipeline` are combined into a single task and run on `_requestExecutor`, which computes CPU tasks. +- `infer_preprocess_and_start_pipeline` prepare input tensors and run on `m_request_executor`, which computes CPU tasks. - You need at least two executors to overlap compute tasks of a CPU and a remote device the plugin works with. Otherwise, CPU and device tasks are executed serially one by one. -- `waitPipeline` is sent to `_waitExecutor`, which works with the device. +- `wait_pipeline` is sent to `m_wait_executor`, which works with the device. -> **NOTE**: `callbackExecutor` is also passed to the constructor and it is used in the base InferenceEngine::AsyncInferRequestThreadSafeDefault class, which adds a pair of `callbackExecutor` and a callback function set by the user to the end of the pipeline. +> **NOTE**: `m_callback_executor` is also passed to the constructor and it is used in the base ov::IAsyncInferRequest class, which adds a pair of `callback_executor` and a callback function set by the user to the end of the pipeline. -Inference request stages are also profiled using IE_PROFILING_AUTO_SCOPE, which shows how pipelines of multiple asynchronous inference requests are run in parallel via the [Intel® VTune™ Profiler](https://software.intel.com/en-us/vtune) tool. +### ~AsyncInferRequest() -### `~AsyncInferRequest()` - -In the asynchronous request destructor, it is necessary to wait for a pipeline to finish. It can be done using the InferenceEngine::AsyncInferRequestThreadSafeDefault::StopAndWait method of the base class. +In the asynchronous request destructor, it is necessary to wait for a pipeline to finish. It can be done using the ov::IAsyncInferRequest::stop_and_wait method of the base class. @snippet src/async_infer_request.cpp async_infer_request:dtor diff --git a/docs/IE_PLUGIN_DG/Building.md b/docs/IE_PLUGIN_DG/Building.md index 9330d658a248f0..f40f2296e85122 100644 --- a/docs/IE_PLUGIN_DG/Building.md +++ b/docs/IE_PLUGIN_DG/Building.md @@ -30,6 +30,7 @@ Once the commands above are executed, the OpenVINO Developer Package is generate * `openvino::unitTestUtils` - static library with unit tests utilities * `openvino::ngraphFunctions` - static library with the set of `ov::Model` builders * `openvino::funcSharedTests` - static library with common functional tests + * `openvino::ngraph_reference` - static library with operation reference implementations. > **NOTE**: it's enough just to run `cmake --build . --target ov_dev_targets` command to build only targets from the > OpenVINO Developer package. @@ -61,7 +62,7 @@ $ cmake -DENABLE_FUNCTIONAL_TESTS=OFF -DOpenVINODeveloperPackage_DIR=../openvino - `src/CMakeLists.txt` to build a plugin shared library from sources: @snippet template/src/CMakeLists.txt cmake:plugin - > **NOTE**: `openvino::runtime` target is imported from the OpenVINO Developer Package. + > **NOTE**: `openvino::...` targets are imported from the OpenVINO Developer Package. - `tests/functional/CMakeLists.txt` to build a set of functional plugin tests: @snippet template/tests/functional/CMakeLists.txt cmake:functional_tests diff --git a/docs/IE_PLUGIN_DG/CompiledModel.md b/docs/IE_PLUGIN_DG/CompiledModel.md index 45d990101d132a..fa80ee81cb75ab 100644 --- a/docs/IE_PLUGIN_DG/CompiledModel.md +++ b/docs/IE_PLUGIN_DG/CompiledModel.md @@ -54,7 +54,7 @@ The method creates an synchronous inference request and returns it. While the public OpenVINO API has a single interface for inference request, which can be executed in synchronous and asynchronous modes, a plugin library implementation has two separate classes: - [Synchronous inference request](@ref openvino_docs_ov_plugin_dg_infer_request), which defines pipeline stages and runs them synchronously in the `infer` method. -- [Asynchronous inference request](@ref openvino_docs_ie_plugin_dg_async_infer_request), which is a wrapper for a synchronous inference request and can run a pipeline asynchronously. Depending on a device pipeline structure, it can has one or several stages: +- [Asynchronous inference request](@ref openvino_docs_ov_plugin_dg_async_infer_request), which is a wrapper for a synchronous inference request and can run a pipeline asynchronously. Depending on a device pipeline structure, it can has one or several stages: - For single-stage pipelines, there is no need to define this method and create a class derived from ov::IAsyncInferRequest. For single stage pipelines, a default implementation of this method creates ov::IAsyncInferRequest wrapping a synchronous inference request and runs it asynchronously in the `m_request_executor` executor. - For pipelines with multiple stages, such as performing some preprocessing on host, uploading input data to a device, running inference on a device, or downloading and postprocessing output data, schedule stages on several task executors to achieve better device use and performance. You can do it by creating a sufficient number of inference requests running in parallel. In this case, device stages of different inference requests are overlapped with preprocessing and postprocessing stage giving better performance. > **IMPORTANT**: It is up to you to decide how many task executors you need to optimally execute a device pipeline. diff --git a/docs/IE_PLUGIN_DG/InferRequest.md b/docs/IE_PLUGIN_DG/InferRequest.md index bd08278e90cd0c..0a3f467d847d6a 100644 --- a/docs/IE_PLUGIN_DG/InferRequest.md +++ b/docs/IE_PLUGIN_DG/InferRequest.md @@ -2,7 +2,7 @@ `InferRequest` class functionality: - Allocate input and output tensors needed for a backend-dependent network inference. -- Define functions for inference process stages (for example, `preprocess`, `upload`, `infer`, `download`, `postprocess`). These functions can later be used to define an execution pipeline during [Asynchronous Inference Request](@ref openvino_docs_ie_plugin_dg_async_infer_request) implementation. +- Define functions for inference process stages (for example, `preprocess`, `upload`, `infer`, `download`, `postprocess`). These functions can later be used to define an execution pipeline during [Asynchronous Inference Request](@ref openvino_docs_ov_plugin_dg_async_infer_request) implementation. - Call inference stages one by one synchronously. InferRequest Class @@ -23,7 +23,9 @@ The example class has several fields: - backend specific fields: - `m_backend_input_tensors` - input backend tensors. - `m_backend_output_tensors` - output backend tensors. - - `m_executable` - an executable object / backend computational graph. + - `m_executable` - an executable object / backend computational graph. + - `m_eval_context` - an evaluation context to save backend states after the inference. + - `m_variable_states` - a vector of variable states. ### InferRequest Constructor @@ -69,7 +71,13 @@ Executes a pipeline synchronously using `m_executable` object: @snippet src/sync_infer_request.cpp infer_request:start_pipeline -#### 3. infer_postprocess() +#### 3. wait_pipeline() + +Waits a pipeline in case of plugin asynchronous execution: + +@snippet src/sync_infer_request.cpp infer_request:wait_pipeline + +#### 4. infer_postprocess() Converts backend specific tensors to tensors passed by user: @@ -81,4 +89,4 @@ The method returns the profiling info which was measured during pipeline stages @snippet src/sync_infer_request.cpp infer_request:get_profiling_info -The next step in the plugin library implementation is the [Asynchronous Inference Request](@ref openvino_docs_ie_plugin_dg_async_infer_request) class. +The next step in the plugin library implementation is the [Asynchronous Inference Request](@ref openvino_docs_ov_plugin_dg_async_infer_request) class. diff --git a/docs/IE_PLUGIN_DG/Intro.md b/docs/IE_PLUGIN_DG/Intro.md index d1a6a12e104036..97e2acaf7b52db 100644 --- a/docs/IE_PLUGIN_DG/Intro.md +++ b/docs/IE_PLUGIN_DG/Intro.md @@ -10,7 +10,8 @@ Implement Plugin Functionality Implement Compiled Model Functionality Implement Synchronous Inference Request - Implement Asynchronous Inference Request + Implement Asynchronous Inference Request + Provide Plugin Specific Properties Implement Remote Context Implement Remote Tensor openvino_docs_ov_plugin_dg_plugin_build @@ -43,16 +44,16 @@ OpenVINO plugin dynamic library consists of several main components: 3. [Inference Request class](@ref openvino_docs_ov_plugin_dg_infer_request): - Runs an inference pipeline serially. - Can extract performance counters for an inference pipeline execution profiling. -4. [Asynchronous Inference Request class](@ref openvino_docs_ie_plugin_dg_async_infer_request): +4. [Asynchronous Inference Request class](@ref openvino_docs_ov_plugin_dg_async_infer_request): - Wraps the [Inference Request](@ref openvino_docs_ov_plugin_dg_infer_request) class and runs pipeline stages in parallel on several task executors based on a device-specific pipeline structure. -5. [Remote Context](@ref openvino_docs_ov_plugin_dg_remote_context): +5. [Plugin specific properties](@ref openvino_docs_ov_plugin_dg_properties): + - Provides the plugin specific properties. +6. [Remote Context](@ref openvino_docs_ov_plugin_dg_remote_context): - Provides the device specific remote context. Context allows to create remote tensors. -6. [Remote Tensor](@ref openvino_docs_ov_plugin_dg_remote_tensor) +7. [Remote Tensor](@ref openvino_docs_ov_plugin_dg_remote_tensor) - Provides the device specific remote tensor API and implementation. -> **NOTE**: This documentation is written based on the `Template` plugin, which demonstrates plugin - -development details. Find the complete code of the `Template`, which is fully compilable and up-to-date, +> **NOTE**: This documentation is written based on the `Template` plugin, which demonstrates plugin development details. Find the complete code of the `Template`, which is fully compilable and up-to-date, at `/src/plugins/template`. @@ -61,7 +62,7 @@ Detailed guides * [Build](@ref openvino_docs_ov_plugin_dg_plugin_build) a plugin library using CMake * Plugin and its components [testing](@ref openvino_docs_ov_plugin_dg_plugin_testing) -* [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks) +* [Quantized networks](@ref openvino_docs_ov_plugin_dg_quantized_models) * [Low precision transformations](@ref openvino_docs_OV_UG_lpt) guide * [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide diff --git a/docs/IE_PLUGIN_DG/Plugin.md b/docs/IE_PLUGIN_DG/Plugin.md index 2fa02a009e8056..124852d6cf6d79 100644 --- a/docs/IE_PLUGIN_DG/Plugin.md +++ b/docs/IE_PLUGIN_DG/Plugin.md @@ -42,6 +42,7 @@ As an example, a plugin configuration has three value parameters: - `perf_counts` - boolean value to identify whether to collect performance counters during [Inference Request](@ref openvino_docs_ov_plugin_dg_infer_request) execution. - `streams_executor_config` - configuration of `ov::threading::IStreamsExecutor` to handle settings of multi-threaded context. - `performance_mode` - configuration of `ov::hint::PerformanceMode` to set the performance mode. +- `disable_transformations` - allows to disable transformations which are applied in the process of model compilation. ### Plugin Constructor @@ -85,7 +86,7 @@ Actual model compilation is done in the `CompiledModel` constructor. Refer to th The function accepts a const shared pointer to `ov::Model` object and applies common and device-specific transformations on a copied model to make it more friendly to hardware operations. For details how to write custom device-specific transformation, please, refer to [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide. See detailed topics about model representation: * [Intermediate Representation and Operation Sets](@ref openvino_docs_MO_DG_IR_and_opsets) - * [Quantized models](@ref openvino_docs_ie_plugin_dg_quantized_networks). + * [Quantized models](@ref openvino_docs_ov_plugin_dg_quantized_models). @snippet template/src/plugin.cpp plugin:transform_model diff --git a/docs/IE_PLUGIN_DG/PluginTesting.md b/docs/IE_PLUGIN_DG/PluginTesting.md index 6a0cecfb05a1c4..ca19d5ea2fbbae 100644 --- a/docs/IE_PLUGIN_DG/PluginTesting.md +++ b/docs/IE_PLUGIN_DG/PluginTesting.md @@ -8,7 +8,7 @@ OpenVINO Plugin tests are included in the `openvino::funcSharedTests` CMake targ Test definitions are split into tests class declaration (see `src/tests/functional/plugin/shared/include`) and tests class implementation (see `src/tests/functional/plugin/shared/src`) and include the following scopes of plugin conformance tests: -1. **Behavior tests** (`behavior` sub-folder), which are a separate test group to check that a plugin satisfies basic OpenVINO concepts: plugin creation, multiple executable networks support, multiple synchronous and asynchronous inference requests support, and so on. See the next section with details how to instantiate the tests definition class with plugin-specific parameters. +1. **Behavior tests** (`behavior` sub-folder), which are a separate test group to check that a plugin satisfies basic OpenVINO concepts: plugin creation, multiple compiled models support, multiple synchronous and asynchronous inference requests support, and so on. See the next section with details how to instantiate the tests definition class with plugin-specific parameters. 2. **Single layer tests** (`single_layer_tests` sub-folder). This groups of tests checks that a particular single layer can be inferenced on a device. An example of test instantiation based on test definition from `openvino::funcSharedTests` library: diff --git a/docs/IE_PLUGIN_DG/Properties.md b/docs/IE_PLUGIN_DG/Properties.md new file mode 100644 index 00000000000000..a8459181e74c42 --- /dev/null +++ b/docs/IE_PLUGIN_DG/Properties.md @@ -0,0 +1,10 @@ +# Plugin Properties {#openvino_docs_ov_plugin_dg_properties} + +Plugin can provide own device specific properties. + +Property Class +------------------------ + +OpenVINO API provides the interface ov::Property which allows to define the property and access rights. Based on that, a declaration of plugin specific properties can look as follows: + +@snippet include/template/properties.hpp properties:public_header diff --git a/docs/IE_PLUGIN_DG/QuantizedNetworks.md b/docs/IE_PLUGIN_DG/QuantizedNetworks.md index 57deb94281de05..f3c712e2f618b5 100644 --- a/docs/IE_PLUGIN_DG/QuantizedNetworks.md +++ b/docs/IE_PLUGIN_DG/QuantizedNetworks.md @@ -1,8 +1,8 @@ -# Quantized networks compute and restrictions {#openvino_docs_ie_plugin_dg_quantized_networks} +# Quantized models compute and restrictions {#openvino_docs_ov_plugin_dg_quantized_models} -One of the feature of Inference Engine is the support of quantized networks with different precisions: INT8, INT4, etc. +One of the feature of OpenVINO is the support of quantized models with different precisions: INT8, INT4, etc. However, it is up to the plugin to define what exact precisions are supported by the particular HW. -All quantized networks which can be expressed in IR have a unified representation by means of *FakeQuantize* operation. +All quantized models which can be expressed in IR have a unified representation by means of *FakeQuantize* operation. For more details about low-precision model representation please refer to this [document](@ref openvino_docs_ie_plugin_dg_lp_representation). ### Interpreting FakeQuantize at runtime @@ -44,6 +44,6 @@ Below we define these rules as follows: - Per-channel quantization of activations for channel-wise and element-wise operations, e.g. Depthwise Convolution, Eltwise Add/Mul, ScaleShift. - Symmetric and asymmetric quantization of weights and activations with the support of per-channel scales and zero-points. - Non-unified quantization parameters for Eltwise and Concat operations. -- Non-quantized network output, i.e. there are no quantization parameters for it. +- Non-quantized models output, i.e. there are no quantization parameters for it. [qdq_propagation]: images/qdq_propagation.png diff --git a/docs/IE_PLUGIN_DG/detailed_guides.md b/docs/IE_PLUGIN_DG/detailed_guides.md index 934c53cc1e2ca5..2076afb0d94447 100644 --- a/docs/IE_PLUGIN_DG/detailed_guides.md +++ b/docs/IE_PLUGIN_DG/detailed_guides.md @@ -6,13 +6,13 @@ :maxdepth: 1 :hidden: - openvino_docs_ie_plugin_dg_quantized_networks + openvino_docs_ov_plugin_dg_quantized_models openvino_docs_OV_UG_lpt @endsphinxdirective The guides below provides extra information about specific features of OpenVINO needed for understanding during OpenVINO plugin development: -* [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks) +* [Quantized networks](@ref openvino_docs_ov_plugin_dg_quantized_models) * [Low precision transformations](@ref openvino_docs_OV_UG_lpt) guide * [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide diff --git a/docs/IE_PLUGIN_DG/layout.xml b/docs/IE_PLUGIN_DG/layout.xml index 44137896ee794b..5b84575851e11b 100644 --- a/docs/IE_PLUGIN_DG/layout.xml +++ b/docs/IE_PLUGIN_DG/layout.xml @@ -25,6 +25,7 @@ + @@ -79,6 +80,7 @@ + diff --git a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md index c695f20ce15643..0b7c24ab8e1cf2 100644 --- a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md +++ b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/lpt.md @@ -128,6 +128,7 @@ The model on this step is changed. There are more details in developer guide [Pr ### Step 2. Markup This step creates runtime attributes for operations. These attributes will be used in next step. Transformations: +* [MarkupBias](@ref openvino_docs_OV_UG_lpt_MarkupBias) * [MarkupCanBeQuantized](@ref openvino_docs_OV_UG_lpt_MarkupCanBeQuantized) * [MarkupPrecisions](@ref openvino_docs_OV_UG_lpt_MarkupPrecisions) * [MarkupPerTensorQuantization](@ref openvino_docs_OV_UG_lpt_MarkupPerTensorQuantization) diff --git a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/pipeline/step2_markup.md b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/pipeline/step2_markup.md index f1590c229c8d26..df93ec6445460a 100644 --- a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/pipeline/step2_markup.md +++ b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/pipeline/step2_markup.md @@ -2,18 +2,20 @@ This step defines the optimal `FakeQuantize` decomposition precisions for the best inference performance via operations markup with runtime attribute instances. Attributes are created for input and output ports and operations. Transformations do not change the operation output port precisions. A model markup low precision logic is decomposed and implemented into the following common markup transformations. The order of transformations is important: -1. [MarkupCanBeQuantized](@ref openvino_docs_OV_UG_lpt_MarkupCanBeQuantized) -2. [MarkupPrecisions](@ref openvino_docs_OV_UG_lpt_MarkupPrecisions) -3. [MarkupPerTensorQuantization](@ref openvino_docs_OV_UG_lpt_MarkupPerTensorQuantization) -4. [MarkupAvgPoolPrecisionPreserved](@ref openvino_docs_OV_UG_lpt_MarkupAvgPoolPrecisionPreserved) -5. [PropagatePrecisions](@ref openvino_docs_OV_UG_lpt_PropagatePrecisions) -6. [AlignQuantizationIntervals](@ref openvino_docs_OV_UG_lpt_AlignQuantizationIntervals) -7. [AlignQuantizationParameters](@ref openvino_docs_OV_UG_lpt_AlignQuantizationParameters) +1. [MarkupBias](@ref openvino_docs_OV_UG_lpt_MarkupBias) +2. [MarkupCanBeQuantized](@ref openvino_docs_OV_UG_lpt_MarkupCanBeQuantized) +3. [MarkupPrecisions](@ref openvino_docs_OV_UG_lpt_MarkupPrecisions) +4. [MarkupPerTensorQuantization](@ref openvino_docs_OV_UG_lpt_MarkupPerTensorQuantization) +5. [MarkupAvgPoolPrecisionPreserved](@ref openvino_docs_OV_UG_lpt_MarkupAvgPoolPrecisionPreserved) +6. [PropagatePrecisions](@ref openvino_docs_OV_UG_lpt_PropagatePrecisions) +7. [AlignQuantizationIntervals](@ref openvino_docs_OV_UG_lpt_AlignQuantizationIntervals) +8. [AlignQuantizationParameters](@ref openvino_docs_OV_UG_lpt_AlignQuantizationParameters) The table of transformations and used attributes: | Transformation name | Create attributes | Use attributes | |---------------------------------|-------------------------------|-------------------------------------------| +| MarkupBias | Bias | | | MarkupCanBeQuantized | Precisions | | | MarkupPrecisions | Precisions,PrecisionPreserved | | | MarkupPerTensorQuantization | PerTensorQuantization | | diff --git a/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/transformations/step2_markup/markup_bias.md b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/transformations/step2_markup/markup_bias.md new file mode 100644 index 00000000000000..632d50cee6c38e --- /dev/null +++ b/docs/IE_PLUGIN_DG/plugin_transformation_pipeline/low_precision_transformations/transformations/step2_markup/markup_bias.md @@ -0,0 +1,3 @@ +# MarkupBias transformation {#openvino_docs_OV_UG_lpt_MarkupBias} + +ngraph::pass::low_precision::MarkupBias class represents the `MarkupBias` transformation. diff --git a/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md b/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md index 91e31630183975..9a39933fde6fb4 100644 --- a/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md +++ b/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md @@ -15,102 +15,128 @@ openvino_docs_MO_DG_FP16_Compression openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ -@endsphinxdirective + Model Optimizer is a cross-platform command-line tool that facilitates the transition between training and deployment environments, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices. -To use it, you need a pre-trained deep learning model in one of the supported formats: TensorFlow, PyTorch, PaddlePaddle, MXNet, Caffe, Kaldi, or ONNX. Model Optimizer converts the model to the OpenVINO Intermediate Representation format (IR), which you can infer later with [OpenVINO™ Runtime](../OV_Runtime_UG/openvino_intro.md). +To use it, you need a pre-trained deep learning model in one of the supported formats: TensorFlow, PyTorch, PaddlePaddle, MXNet, Caffe, Kaldi, or ONNX. Model Optimizer converts the model to the OpenVINO Intermediate Representation format (IR), which you can infer later with :doc:`OpenVINO™ Runtime `. Note that Model Optimizer does not infer models. The figure below illustrates the typical workflow for deploying a trained deep learning model: -![](img/BASIC_FLOW_MO_simplified.svg) +.. image:: _static/images/BASIC_FLOW_MO_simplified.svg where IR is a pair of files describing the model: -* .xml - Describes the network topology. +* ``.xml`` - Describes the network topology. -* .bin - Contains the weights and biases binary data. +* ``.bin`` - Contains the weights and biases binary data. -The OpenVINO IR can be additionally optimized for inference by [Post-training optimization](../../tools/pot/docs/Introduction.md) that applies post-training quantization methods. +The OpenVINO IR can be additionally optimized for inference by :doc:`Post-training optimization ` that applies post-training quantization methods. -## How to Run Model Optimizer +How to Run Model Optimizer +########################## To convert a model to IR, you can run Model Optimizer by using the following command: -```sh -mo --input_model INPUT_MODEL -``` +.. code-block:: sh + + mo --input_model INPUT_MODEL -If the out-of-the-box conversion (only the `--input_model` parameter is specified) is not successful, use the parameters mentioned below to override input shapes and cut the model: -- Model Optimizer provides two parameters to override original input shapes for model conversion: `--input` and `--input_shape`. -For more information about these parameters, refer to the [Setting Input Shapes](prepare_model/convert_model/Converting_Model.md) guide. +If the out-of-the-box conversion (only the ``--input_model`` parameter is specified) is not successful, use the parameters mentioned below to override input shapes and cut the model: + +- Model Optimizer provides two parameters to override original input shapes for model conversion: ``--input`` and ``--input_shape``. +For more information about these parameters, refer to the :doc:`Setting Input Shapes ` guide. - To cut off unwanted parts of a model (such as unsupported operations and training sub-graphs), -use the `--input` and `--output` parameters to define new inputs and outputs of the converted model. -For a more detailed description, refer to the [Cutting Off Parts of a Model](prepare_model/convert_model/Cutting_Model.md) guide. +use the ``--input`` and ``--output`` parameters to define new inputs and outputs of the converted model. +For a more detailed description, refer to the :doc:`Cutting Off Parts of a Model ` guide. You can also insert additional input pre-processing sub-graphs into the converted model by using -the `--mean_values`, `scales_values`, `--layout`, and other parameters described -in the [Embedding Preprocessing Computation](prepare_model/Additional_Optimizations.md) article. +the ``--mean_values``, ``scales_values``, ``--layout``, and other parameters described +in the :doc:`Embedding Preprocessing Computation ` article. -The `--compress_to_fp16` compression parameter in Model Optimizer allows generating IR with constants (for example, weights for convolutions and matrix multiplications) compressed to `FP16` data type. For more details, refer to the [Compression of a Model to FP16](prepare_model/FP16_Compression.md) guide. +The ``--compress_to_fp16`` compression parameter in Model Optimizer allows generating IR with constants (for example, weights for convolutions and matrix multiplications) compressed to ``FP16`` data type. For more details, refer to the :doc:`Compression of a Model to FP16 ` guide. To get the full list of conversion parameters available in Model Optimizer, run the following command: -```sh -mo --help -``` +.. code-block:: sh + + mo --help -## Examples of CLI Commands + +Examples of CLI Commands +######################## Below is a list of separate examples for different frameworks and Model Optimizer parameters: 1. Launch Model Optimizer for a TensorFlow MobileNet model in the binary protobuf format: -```sh -mo --input_model MobileNet.pb -``` -Launch Model Optimizer for a TensorFlow BERT model in the SavedModel format with three inputs. Specify input shapes explicitly -where the batch size and the sequence length equal 2 and 30 respectively: -```sh -mo --saved_model_dir BERT --input mask,word_ids,type_ids --input_shape [2,30],[2,30],[2,30] -``` -For more information, refer to the [Converting a TensorFlow Model](prepare_model/convert_model/Convert_Model_From_TensorFlow.md) guide. + + .. code-block:: sh + + mo --input_model MobileNet.pb + + + Launch Model Optimizer for a TensorFlow BERT model in the SavedModel format with three inputs. Specify input shapes explicitly where the batch size and the sequence length equal 2 and 30 respectively: + + .. code-block:: sh + + mo --saved_model_dir BERT --input mask,word_ids,type_ids --input_shape [2,30],[2,30],[2,30] + + For more information, refer to the :doc:`Converting a TensorFlow Model ` guide. 2. Launch Model Optimizer for an ONNX OCR model and specify new output explicitly: -```sh -mo --input_model ocr.onnx --output probabilities -``` -For more information, refer to the [Converting an ONNX Model](prepare_model/convert_model/Convert_Model_From_ONNX.md) guide. -> **NOTE**: PyTorch models must be exported to the ONNX format before conversion into IR. More information can be found in [Converting a PyTorch Model](prepare_model/convert_model/Convert_Model_From_PyTorch.md). + .. code-block:: sh + + mo --input_model ocr.onnx --output probabilities + + + For more information, refer to the :doc:`Converting an ONNX Model ` guide. + + .. note:: + + PyTorch models must be exported to the ONNX format before conversion into IR. More information can be found in :doc:`Converting a PyTorch Model `. 3. Launch Model Optimizer for a PaddlePaddle UNet model and apply mean-scale normalization to the input: -```sh -mo --input_model unet.pdmodel --mean_values [123,117,104] --scale 255 -``` -For more information, refer to the [Converting a PaddlePaddle Model](prepare_model/convert_model/Convert_Model_From_Paddle.md) guide. + + .. code-block:: sh + + mo --input_model unet.pdmodel --mean_values [123,117,104] --scale 255 + + + For more information, refer to the :doc:`Converting a PaddlePaddle Model ` guide. 4. Launch Model Optimizer for an Apache MXNet SSD Inception V3 model and specify first-channel layout for the input: -```sh -mo --input_model ssd_inception_v3-0000.params --layout NCHW -``` -For more information, refer to the [Converting an Apache MXNet Model](prepare_model/convert_model/Convert_Model_From_MxNet.md) guide. + + .. code-block:: sh + + mo --input_model ssd_inception_v3-0000.params --layout NCHW + + + For more information, refer to the :doc:`Converting an Apache MXNet Model ` guide. 5. Launch Model Optimizer for a Caffe AlexNet model with input channels in the RGB format which needs to be reversed: -```sh -mo --input_model alexnet.caffemodel --reverse_input_channels -``` -For more information, refer to the [Converting a Caffe Model](prepare_model/convert_model/Convert_Model_From_Caffe.md) guide. + + .. code-block:: sh + + mo --input_model alexnet.caffemodel --reverse_input_channels + + + For more information, refer to the :doc:`Converting a Caffe Model ` guide. 6. Launch Model Optimizer for a Kaldi LibriSpeech nnet2 model: -```sh -mo --input_model librispeech_nnet2.mdl --input_shape [1,140] -``` -For more information, refer to the [Converting a Kaldi Model](prepare_model/convert_model/Convert_Model_From_Kaldi.md) guide. - -- To get conversion recipes for specific TensorFlow, ONNX, PyTorch, Apache MXNet, and Kaldi models, -refer to the [Model Conversion Tutorials](prepare_model/convert_model/Convert_Model_Tutorials.md). -- For more information about IR, see [Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](IR_and_opsets.md). + + .. code-block:: sh + + mo --input_model librispeech_nnet2.mdl --input_shape [1,140] + + + For more information, refer to the :doc:`Converting a Kaldi Model ` guide. + +- To get conversion recipes for specific TensorFlow, ONNX, PyTorch, Apache MXNet, and Kaldi models, refer to the :doc:`Model Conversion Tutorials `. +- For more information about IR, see :doc:`Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™ `. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/Additional_Optimizations.md b/docs/MO_DG/prepare_model/Additional_Optimizations.md index 8f2459ebdf20bc..213ec14ab5d7af 100644 --- a/docs/MO_DG/prepare_model/Additional_Optimizations.md +++ b/docs/MO_DG/prepare_model/Additional_Optimizations.md @@ -1,102 +1,156 @@ # Embedding Preprocessing Computation {#openvino_docs_MO_DG_Additional_Optimization_Use_Cases} -Input data for inference can be different from the training dataset and requires additional preprocessing before inference. -To accelerate the whole pipeline including preprocessing and inference, Model Optimizer provides special parameters such as `--mean_values`, +@sphinxdirective -`--scale_values`, `--reverse_input_channels`, and `--layout`. Based on these parameters, Model Optimizer generates OpenVINO IR with additionally -inserted sub-graphs to perform the defined preprocessing. This preprocessing block can perform mean-scale normalization of input data, -reverting data along channel dimension, and changing the data layout. -See the following sections for details on the parameters, or the [Overview of Preprocessing API](../../OV_Runtime_UG/preprocessing_overview.md) for the same functionality in OpenVINO Runtime. +Input data for inference can be different from the training dataset and requires +additional preprocessing before inference. To accelerate the whole pipeline including +preprocessing and inference, Model Optimizer provides special parameters such as ``--mean_values``, -## Specifying Layout +``--scale_values``, ``--reverse_input_channels``, and ``--layout``. Based on these +parameters, Model Optimizer generates OpenVINO IR with additionally inserted sub-graphs +to perform the defined preprocessing. This preprocessing block can perform mean-scale +normalization of input data, reverting data along channel dimension, and changing +the data layout. See the following sections for details on the parameters, or the +:doc:`Overview of Preprocessing API ` +for the same functionality in OpenVINO Runtime. -You may need to set input layouts, as it is required by some preprocessing, for example, setting a batch, applying mean or scales, and reversing input channels (BGR<->RGB). +Specifying Layout +################# -Layout defines the meaning of dimensions in shape and can be specified for both inputs and outputs. Some preprocessing requires to set input layouts, for example, setting a batch, applying mean or scales, and reversing input channels (BGR<->RGB). +You may need to set input layouts, as it is required by some preprocessing, for +example, setting a batch, applying mean or scales, and reversing input channels (BGR<->RGB). -For the layout syntax, check the [Layout API overview](../../OV_Runtime_UG/layout_overview.md). -To specify the layout, you can use the `--layout` option followed by the layout value. +Layout defines the meaning of dimensions in shape and can be specified for both +inputs and outputs. Some preprocessing requires to set input layouts, for example, +setting a batch, applying mean or scales, and reversing input channels (BGR<->RGB). -For example, the following command specifies the `NHWC` layout for a Tensorflow `nasnet_large` model that was exported to the ONNX format: +For the layout syntax, check the :doc:`Layout API overview `. +To specify the layout, you can use the ``--layout`` option followed by the layout value. -``` -mo --input_model tf_nasnet_large.onnx --layout nhwc -``` +For example, the following command specifies the ``NHWC`` layout for a Tensorflow +``nasnet_large`` model that was exported to the ONNX format: -Additionally, if a model has more than one input or needs both input and output layouts specified, you need to provide the name of each input or output to apply the layout. +.. code-block:: sh -For example, the following command specifies the layout for an ONNX `Yolo v3 Tiny` model with its first input `input_1` in `NCHW` layout and second input `image_shape` having two dimensions: batch and size of the image expressed as the `N?` layout: + mo --input_model tf_nasnet_large.onnx --layout nhwc -``` -mo --input_model yolov3-tiny.onnx --layout input_1(nchw),image_shape(n?) -``` -## Changing Model Layout +Additionally, if a model has more than one input or needs both input and output +layouts specified, you need to provide the name of each input or output to apply the layout. + +For example, the following command specifies the layout for an ONNX ``Yolo v3 Tiny`` +model with its first input ``input_1`` in ``NCHW`` layout and second input ``image_shape`` +having two dimensions: batch and size of the image expressed as the ``N?`` layout: + +.. code-block:: sh + + mo --input_model yolov3-tiny.onnx --layout input_1(nchw),image_shape(n?) + + +Changing Model Layout +##################### Changing the model layout may be necessary if it differs from the one presented by input data. -Use either `--layout` or `--source_layout` with `--target_layout` to change the layout. +Use either ``--layout`` or ``--source_layout`` with ``--target_layout`` to change the layout. + +For example, for the same ``nasnet_large`` model mentioned previously, you can use +the following commands to provide data in the ``NCHW`` layout: -For example, for the same `nasnet_large` model mentioned previously, you can use the following commands to provide data in the `NCHW` layout: +.. code-block:: sh -``` -mo --input_model tf_nasnet_large.onnx --source_layout nhwc --target_layout nchw -mo --input_model tf_nasnet_large.onnx --layout "nhwc->nchw" -``` + mo --input_model tf_nasnet_large.onnx --source_layout nhwc --target_layout nchw + mo --input_model tf_nasnet_large.onnx --layout "nhwc->nchw" -Again, if a model has more than one input or needs both input and output layouts specified, you need to provide the name of each input or output to apply the layout. -For example, to provide data in the `NHWC` layout for the `Yolo v3 Tiny` model mentioned earlier, use the following commands: +Again, if a model has more than one input or needs both input and output layouts +specified, you need to provide the name of each input or output to apply the layout. -``` -mo --input_model yolov3-tiny.onnx --source_layout "input_1(nchw),image_shape(n?)" --target_layout "input_1(nhwc)" -mo --input_model yolov3-tiny.onnx --layout "input_1(nchw->nhwc),image_shape(n?)" -``` +For example, to provide data in the ``NHWC`` layout for the `Yolo v3 Tiny` model +mentioned earlier, use the following commands: -## Specifying Mean and Scale Values -Neural network models are usually trained with the normalized input data. This means that the input data values are converted to be in a specific range, -for example, `[0, 1]` or `[-1, 1]`. Sometimes, the mean values (mean images) are subtracted from the input data values as part of the preprocessing. +.. code-block:: sh + + mo --input_model yolov3-tiny.onnx --source_layout "input_1(nchw),image_shape(n?)" --target_layout "input_1(nhwc)" + mo --input_model yolov3-tiny.onnx --layout "input_1(nchw->nhwc),image_shape(n?)" + + +Specifying Mean and Scale Values +################################ + +Neural network models are usually trained with the normalized input data. This +means that the input data values are converted to be in a specific range, for example, +``[0, 1]`` or ``[-1, 1]``. Sometimes, the mean values (mean images) are subtracted +from the input data values as part of the preprocessing. There are two cases of how the input data preprocessing is implemented. - * The input preprocessing operations are a part of a model. - In this case, the application does not perform a separate preprocessing step: everything is embedded into the model itself. Model Optimizer will generate the OpenVINO IR format with required preprocessing operations, and no `mean` and `scale` parameters are required. - * The input preprocessing operations are not a part of a model and the preprocessing is performed within the application which feeds the model with input data. +* The input preprocessing operations are a part of a model. + + In this case, the application does not perform a separate preprocessing step: + everything is embedded into the model itself. Model Optimizer will generate the + OpenVINO IR format with required preprocessing operations, and no ``mean`` and + ``scale`` parameters are required. +* The input preprocessing operations are not a part of a model and the preprocessing + is performed within the application which feeds the model with input data. - In this case, information about mean/scale values should be provided to Model Optimizer to embed it to the generated OpenVINO IR format. + In this case, information about mean/scale values should be provided to Model + Optimizer to embed it to the generated OpenVINO IR format. -Model Optimizer provides command-line parameters to specify the values: `--mean_values`, `--scale_values`, `--scale`. -Using these parameters, Model Optimizer embeds the corresponding preprocessing block for mean-value normalization of the input data +Model Optimizer provides command-line parameters to specify the values: ``--mean_values``, +``--scale_values``, ``--scale``. Using these parameters, Model Optimizer embeds the +corresponding preprocessing block for mean-value normalization of the input data and optimizes this block so that the preprocessing takes negligible time for inference. -For example, the following command runs Model Optimizer for the PaddlePaddle UNet model and applies mean-scale normalization to the input data: +For example, the following command runs Model Optimizer for the PaddlePaddle UNet +model and applies mean-scale normalization to the input data: + +.. code-block:: sh + + mo --input_model unet.pdmodel --mean_values [123,117,104] --scale 255 + + +Reversing Input Channels +######################## + +Sometimes, input images for your application can be of the RGB (or BGR) format +and the model is trained on images of the BGR (or RGB) format, which is in the +opposite order of color channels. In this case, it is important to preprocess the +input images by reverting the color channels before inference. + +To embed this preprocessing step into OpenVINO IR, Model Optimizer provides the +``--reverse_input_channels`` command-line parameter to shuffle the color channels. + +The ``--reverse_input_channels`` parameter can be used to preprocess the model +input in the following cases: + +* Only one dimension in the input shape has a size equal to ``3``. +* One dimension has an undefined size and is marked as ``C`` channel using ``layout`` parameters. + +Using the ``--reverse_input_channels`` parameter, Model Optimizer embeds the corresponding +preprocessing block for reverting the input data along channel dimension and optimizes +this block so that the preprocessing takes only negligible time for inference. + +For example, the following command launches Model Optimizer for the TensorFlow AlexNet +model and embeds the ``reverse_input_channel`` preprocessing block into OpenVINO IR: -```sh -mo --input_model unet.pdmodel --mean_values [123,117,104] --scale 255 -``` +.. code-block:: sh -## Reversing Input Channels -Sometimes, input images for your application can be of the RGB (or BGR) format and the model is trained on images of the BGR (or RGB) format, -which is in the opposite order of color channels. In this case, it is important to preprocess the input images by reverting the color channels before inference. + mo --input_model alexnet.pb --reverse_input_channels -To embed this preprocessing step into OpenVINO IR, Model Optimizer provides the `--reverse_input_channels` command-line parameter to shuffle the color channels. -The `--reverse_input_channels` parameter can be used to preprocess the model input in the following cases: - * Only one dimension in the input shape has a size equal to 3. - * One dimension has an undefined size and is marked as `C` channel using `layout` parameters. +.. note:: -Using the `--reverse_input_channels` parameter, Model Optimizer embeds the corresponding preprocessing block for reverting -the input data along channel dimension and optimizes this block so that the preprocessing takes only negligible time for inference. + If both mean and scale values are specified, the mean is subtracted first and + then the scale is applied regardless of the order of options in the command-line. + Input values are *divided* by the scale value(s). If the ``--reverse_input_channels`` + option is also used, ``reverse_input_channels`` will be applied first, then ``mean`` + and after that ``scale``. The data flow in the model looks as follows: + ``Parameter -> ReverseInputChannels -> Mean apply-> Scale apply -> the original body of the model``. -For example, the following command launches Model Optimizer for the TensorFlow AlexNet model and embeds the `reverse_input_channel` preprocessing block into OpenVINO IR: +Additional Resources +#################### -```sh -mo --input_model alexnet.pb --reverse_input_channels -``` +* :doc:`Overview of Preprocessing API ` -> **NOTE**: If both mean and scale values are specified, the mean is subtracted first and then the scale is applied regardless of the order of options -in the command-line. Input values are *divided* by the scale value(s). If the `--reverse_input_channels` option is also used, `reverse_input_channels` -will be applied first, then `mean` and after that `scale`. The data flow in the model looks as follows: -`Parameter -> ReverseInputChannels -> Mean apply-> Scale apply -> the original body of the model`. +@endsphinxdirective -## Additional Resources -* [Overview of Preprocessing API](../../OV_Runtime_UG/preprocessing_overview.md) diff --git a/docs/MO_DG/prepare_model/FP16_Compression.md b/docs/MO_DG/prepare_model/FP16_Compression.md index 9216243f85a712..96086e596b7686 100644 --- a/docs/MO_DG/prepare_model/FP16_Compression.md +++ b/docs/MO_DG/prepare_model/FP16_Compression.md @@ -1,16 +1,29 @@ # Compressing a Model to FP16 {#openvino_docs_MO_DG_FP16_Compression} -Model Optimizer by default converts all floating-point weights to `FP16` data type. The resulting IR is called -compressed `FP16` model. The resulting model will occupy about twice as less space in the file system, -but it may have some accuracy drop. For most models, the accuracy drop is negligible. -But in case if accuracy drop is significant user can disable compression explicitly. +@sphinxdirective -By default, models are compressed to `FP16`, but you can disable compression by specifying `--compress_to_fp16=False`: -``` -mo --input_model INPUT_MODEL --compress_to_fp16=False -``` +Model Optimizer by default converts all floating-point weights to ``FP16`` data type. +The resulting IR is called compressed ``FP16`` model. The resulting model will occupy +about twice as less space in the file system, but it may have some accuracy drop. +For most models, the accuracy drop is negligible. But in case if accuracy drop is +significant user can disable compression explicitly. -For details on how plugins handle compressed `FP16` models, see [Working with devices](../../OV_Runtime_UG/supported_plugins/Device_Plugins.md). +By default, models are compressed to ``FP16``, but you can disable compression by +specifying ``--compress_to_fp16=False``: -> **NOTE**: `FP16` compression is sometimes used as the initial step for `INT8` quantization. -> Refer to the [Post-training optimization](../../../tools/pot/docs/Introduction.md) guide for more information about that. +.. code-block:: sh + + mo --input_model INPUT_MODEL --compress_to_fp16=False + + +For details on how plugins handle compressed ``FP16`` models, see +:doc:`Working with devices `. + +.. note:: + + ``FP16`` compression is sometimes used as the initial step for ``INT8`` quantization. + Refer to the :doc:`Post-training optimization ` guide for more + information about that. + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md b/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md index 855e124fab80fa..8fe92fae1bba52 100644 --- a/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md +++ b/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md @@ -1,642 +1,989 @@ # Model Optimizer Frequently Asked Questions {#openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ} -If your question is not covered by the topics below, use the [OpenVINO Support page](https://software.intel.com/en-us/openvino-toolkit/documentation/get-started), where you can participate on a free forum. +@sphinxdirective -#### Q1. What does the message "[ ERROR ]: Current caffe.proto does not contain field" mean? +If your question is not covered by the topics below, use the `OpenVINO Support page `__, where you can participate on a free forum. -**A** : Internally, Model Optimizer uses a protobuf library to parse and load Caffe models. This library requires a file grammar and a generated parser. For a Caffe fallback, Model Optimizer uses a Caffe-generated parser for a Caffe-specific `.proto` file (which is usually located in the `src/caffe/proto` directory). Make sure that you install exactly the same version of Caffe (with Python interface) as that was used to create the model. +.. _question-1: + +Q1. What does the message "[ ERROR ]: Current caffe.proto does not contain field" mean? +##################################################################################################################################################### + +**A:** Internally, Model Optimizer uses a protobuf library to parse and load Caffe models. This library requires a file grammar and a generated parser. For a Caffe fallback, Model Optimizer uses a Caffe-generated parser for a Caffe-specific ``.proto`` file (which is usually located in the ``src/caffe/proto`` directory). Make sure that you install exactly the same version of Caffe (with Python interface) as that was used to create the model. If you just want to experiment with Model Optimizer and test a Python extension for working with your custom -layers without building Caffe, add the layer description to the `caffe.proto` file and generate a parser for it. +layers without building Caffe, add the layer description to the ``caffe.proto`` file and generate a parser for it. + +For example, to add the description of the ``CustomReshape`` layer, which is an artificial layer not present in any ``caffe.proto`` files: -For example, to add the description of the `CustomReshape` layer, which is an artificial layer not present in any `caffe.proto` files: +1. Add the following lines to the ``caffe.proto`` file: -1. Add the following lines to the `caffe.proto` file: -```shell - package mo_caffe; // To avoid conflict with Caffe system, it is highly recommended to specify different package name. - ... - message LayerParameter { - // Other layers parameters description. + .. code-block:: shell + + package mo_caffe; // To avoid conflict with Caffe system, it is highly recommended to specify different package name. ... - optional CustomReshapeParameter custom_reshape_param = 546; // 546 - ID is any number not present in caffe.proto. - } - // The lines from here to the end of the file are describing contents of this parameter. - message CustomReshapeParameter { - optional BlobShape shape = 1; // Just use the same parameter type as some other Caffe layers. - } -``` - -2. Generate a new parser: -```shell -cd /openvino/tools/mo/front/caffe/proto -python3 generate_caffe_pb2.py --input_proto /src/caffe/proto/caffe.proto -``` -where `PATH_TO_CUSTOM_CAFFE` is the path to the root directory of custom Caffe. - -3. Now, Model Optimizer is able to load the model into memory and start working with your extensions if there are any. - -However, since your model has custom layers, you must register them as custom. To learn more about it, refer to [Custom Layers in Model Optimizer](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer). - -#### Q2. How do I create a bare caffemodel, if I have only prototxt? - -**A** : You need the Caffe Python interface. In this case, do the following: -```shell -python3 -import caffe -net = caffe.Net('/my_net.prototxt', caffe.TEST) -net.save('/my_net.caffemodel') -``` -#### Q3. What does the message "[ ERROR ]: Unable to create ports for node with id" mean? - -**A** : Most likely, Model Optimizer does not know how to infer output shapes of some layers in the given topology. + message LayerParameter { + // Other layers parameters description. + ... + optional CustomReshapeParameter custom_reshape_param = 546; // 546 - ID is any number not present in caffe.proto. + } + // The lines from here to the end of the file are describing contents of this parameter. + message CustomReshapeParameter { + optional BlobShape shape = 1; // Just use the same parameter type as some other Caffe layers. + } + + +2. Generate a new parser: + + .. code-block:: shell + + cd /openvino/tools/mo/front/caffe/proto + python3 generate_caffe_pb2.py --input_proto /src/caffe/proto/caffe.proto + + + where ``PATH_TO_CUSTOM_CAFFE` is the path to the root directory of custom Caffe. + +3. Now, Model Optimizer is able to load the model into memory and start working with your extensions if there are any. + + However, since your model has custom layers, you must register them as custom. To learn more about it, refer to :doc:`Custom Layers in Model Optimizer `. + +.. _question-2: + +Q2. How do I create a bare caffemodel, if I have only prototxt? +##################################################################################################################################################### + +**A:** You need the Caffe Python interface. In this case, do the following: + +.. code-block:: shell + + python3 + import caffe + net = caffe.Net('/my_net.prototxt', caffe.TEST) + net.save('/my_net.caffemodel') + + +.. _question-3: + +Q3. What does the message "[ ERROR ]: Unable to create ports for node with id" mean? +##################################################################################################################################################### + +**A:** Most likely, Model Optimizer does not know how to infer output shapes of some layers in the given topology. To lessen the scope, compile the list of layers that are custom for Model Optimizer: present in the topology, -absent in the [list of supported layers](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) for the target framework. Then, refer to available options in the corresponding section in the [Custom Layers in Model Optimizer](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) page. - -#### Q4. What does the message "Input image of shape is larger than mean image from file" mean? - -**A** : Your model input shapes must be smaller than or equal to the shapes of the mean image file you provide. The idea behind the mean file is to subtract its values from the input image in an element-wise manner. When the mean file is smaller than the input image, there are not enough values to perform element-wise subtraction. Also, make sure you use the mean file that was used during the network training phase. Note that the mean file is dependent on dataset. - -#### Q5. What does the message "Mean file is empty" mean? - -**A** : Most likely, the mean file specified with the `--mean_file` flag is empty while Model Optimizer is launched. Make sure that this is exactly the required mean file and try to regenerate it from the given dataset if possible. - -#### Q6. What does the message "Probably mean file has incorrect format" mean? - -**A** : The mean file that you provide for Model Optimizer must be in the `.binaryproto` format. You can try to check the content, using recommendations from the BVLC Caffe ([#290](https://github.com/BVLC/caffe/issues/290)). - -#### Q7. What does the message "Invalid proto file: there is neither 'layer' nor 'layers' top-level messages" mean? - -**A** : The structure of any Caffe topology is described in the `caffe.proto` file of any Caffe version. For example, the following `.proto` file in Model Optimizer is used by default: `mo/front/caffe/proto/my_caffe.proto`, with the structure: -``` -message NetParameter { - // ... some other parameters - // The layers that make up the net. Each of their configurations, including - // connectivity and behavior, is specified as a LayerParameter. - repeated LayerParameter layer = 100; // ID 100 so layers are printed last. - // DEPRECATED: use 'layer' instead. - repeated V1LayerParameter layers = 2; -} -``` -This means that any topology should contain layers as top-level structures in `prototxt`. For example, see the [LeNet topology](https://github.com/BVLC/caffe/blob/master/examples/mnist/lenet.prototxt). - -#### Q8. What does the message "Old-style inputs (via 'input_dims') are not supported. Please specify inputs via 'input_shape'" mean? - -**A** : The structure of any Caffe topology is described in the `caffe.proto` file for any Caffe version. For example, the following `.proto` file in Model Optimizer is used by default: `mo/front/caffe/proto/my_caffe.proto`, with the structure: -```sh -message NetParameter { - - optional string name = 1; // consider giving the network a name - // DEPRECATED. See InputParameter. The input blobs to the network. - repeated string input = 3; - // DEPRECATED. See InputParameter. The shape of the input blobs. - repeated BlobShape input_shape = 8; - // 4D input dimensions -- deprecated. Use "input_shape" instead. - // If specified, for each input blob there should be four - // values specifying the num, channels, height and width of the input blob. - // Thus, there should be a total of (4 * #input) numbers. - repeated int32 input_dim = 4; - // ... other parameters -} -``` +absent in the :doc:`list of supported layers ` for the target framework. Then, refer to available options in the corresponding section in the :doc:`Custom Layers in Model Optimizer ` page. + +.. _question-4: + +Q4. What does the message "Input image of shape is larger than mean image from file" mean? +##################################################################################################################################################### + +**A:** Your model input shapes must be smaller than or equal to the shapes of the mean image file you provide. The idea behind the mean file is to subtract its values from the input image in an element-wise manner. When the mean file is smaller than the input image, there are not enough values to perform element-wise subtraction. Also, make sure you use the mean file that was used during the network training phase. Note that the mean file is dependent on dataset. + +.. _question-5: + +Q5. What does the message "Mean file is empty" mean? +##################################################################################################################################################### + +**A:** Most likely, the mean file specified with the ``--mean_file`` flag is empty while Model Optimizer is launched. Make sure that this is exactly the required mean file and try to regenerate it from the given dataset if possible. + +.. _question-6: + +Q6. What does the message "Probably mean file has incorrect format" mean? +##################################################################################################################################################### + +**A:** The mean file that you provide for Model Optimizer must be in the ``.binaryproto`` format. You can try to check the content, using recommendations from the BVLC Caffe (`#290 `__). + +.. _question-7: + +Q7. What does the message "Invalid proto file: there is neither 'layer' nor 'layers' top-level messages" mean? +##################################################################################################################################################### + +**A:** The structure of any Caffe topology is described in the ``caffe.proto`` file of any Caffe version. For example, the following ``.proto`` file in Model Optimizer is used by default: ``mo/front/caffe/proto/my_caffe.proto``, with the structure: + +.. code-block:: sh + + message NetParameter { + // ... some other parameters + // The layers that make up the net. Each of their configurations, including + // connectivity and behavior, is specified as a LayerParameter. + repeated LayerParameter layer = 100; // ID 100 so layers are printed last. + // DEPRECATED: use 'layer' instead. + repeated V1LayerParameter layers = 2; + } + + +This means that any topology should contain layers as top-level structures in ``prototxt``. For example, see the `LeNet topology `__. + +.. _question-8: + +Q8. What does the message "Old-style inputs (via 'input_dims') are not supported. Please specify inputs via 'input_shape'" mean? +##################################################################################################################################################### + +**A:** The structure of any Caffe topology is described in the ``caffe.proto`` file for any Caffe version. For example, the following ``.proto`` file in Model Optimizer is used by default: ``mo/front/caffe/proto/my_caffe.proto``, with the structure: + +.. code-block:: sh + + message NetParameter { + + optional string name = 1; // consider giving the network a name + // DEPRECATED. See InputParameter. The input blobs to the network. + repeated string input = 3; + // DEPRECATED. See InputParameter. The shape of the input blobs. + repeated BlobShape input_shape = 8; + // 4D input dimensions -- deprecated. Use "input_shape" instead. + // If specified, for each input blob there should be four + // values specifying the num, channels, height and width of the input blob. + // Thus, there should be a total of (4 * #input) numbers. + repeated int32 input_dim = 4; + // ... other parameters + } + + Therefore, the input layer of the provided model must be specified in one of the following styles: * -```sh -input: "data" -input_shape -{ - dim: 1 - dim: 3 - dim: 227 - dim: 227 -} -``` + + .. code-block:: sh + + input: "data" + input_shape + { + dim: 1 + dim: 3 + dim: 227 + dim: 227 + } + * -```sh -input: "data" -input_shape -{ - dim: 1 - dim: 3 - dim: 600 - dim: 1000 -} -input: "im_info" -input_shape -{ - dim: 1 - dim: 3 -} -``` + + .. code-block:: sh + + input: "data" + input_shape + { + dim: 1 + dim: 3 + dim: 600 + dim: 1000 + } + input: "im_info" + input_shape + { + dim: 1 + dim: 3 + } + * -```sh -layer -{ - name: "data" - type: "Input" - top: "data" - input_param {shape: {dim: 1 dim: 3 dim: 600 dim: 1000}} -} -layer -{ - name: "im_info" - type: "Input" - top: "im_info" - input_param {shape: {dim: 1 dim: 3}} -} -``` + + .. code-block:: sh + + layer + { + name: "data" + type: "Input" + top: "data" + input_param {shape: {dim: 1 dim: 3 dim: 600 dim: 1000}} + } + layer + { + name: "im_info" + type: "Input" + top: "im_info" + input_param {shape: {dim: 1 dim: 3}} + } + * -```sh -input: "data" -input_dim: 1 -input_dim: 3 -input_dim: 500 -``` + + .. code-block:: sh + + input: "data" + input_dim: 1 + input_dim: 3 + input_dim: 500 + However, if your model contains more than one input, Model Optimizer is able to convert the model with inputs specified in one of the first three forms in the above list. The 4th form is not supported for multi-input topologies. -#### Q9. What does the message "Mean file for topologies with multiple inputs is not supported" mean? +.. _question-9: + +Q9. What does the message "Mean file for topologies with multiple inputs is not supported" mean? +##################################################################################################################################################### + +**A:** Model Optimizer does not support mean file processing for topologies with more than one input. In this case, you need to perform preprocessing of the inputs for a generated Intermediate Representation in OpenVINO Runtime to perform subtraction for every input of your multi-input model. See the :doc:`Overview of Preprocessing ` for details. + +.. _question-10: + +Q10. What does the message "Cannot load or process mean file: value error" mean? +##################################################################################################################################################### + +**A:** There are multiple reasons why Model Optimizer does not accept the mean file. +See FAQs :ref:`#4 `, :ref:`#5 `, and :ref:`#6 `. + +.. _question-11: + +Q11. What does the message "Invalid prototxt file: value error" mean? +##################################################################################################################################################### + +**A:** There are multiple reasons why Model Optimizer does not accept a Caffe topology. See FAQs :ref:`#7 ` and :ref:`#20 `. + +.. _question-12: + +Q12. What does the message "Error happened while constructing caffe.Net in the Caffe fallback function" mean? +##################################################################################################################################################### + +**A:** Model Optimizer tried to infer a specified layer via the Caffe framework. However, it cannot construct a net using the Caffe Python interface. Make sure that your ``caffemodel`` and ``prototxt`` files are correct. To ensure that the problem is not in the ``prototxt`` file, see FAQ :ref:`#2 `. + +.. _question-13: -**A** : Model Optimizer does not support mean file processing for topologies with more than one input. In this case, you need to perform preprocessing of the inputs for a generated Intermediate Representation in OpenVINO Runtime to perform subtraction for every input of your multi-input model. See the [Overview of Preprocessing](@ref openvino_docs_OV_UG_Preprocessing_Overview) for details. +Q13. What does the message "Cannot infer shapes due to exception in Caffe" mean? +##################################################################################################################################################### -#### Q10. What does the message "Cannot load or process mean file: value error" mean? +**A:** Model Optimizer tried to infer a custom layer via the Caffe framework, but the model could not be inferred using Caffe. This might happen if you try to convert the model with some noise weights and biases, which conflict with layers that have dynamic shapes. You should write your own extension for every custom layer your topology might have. For more details, refer to the :doc:`Model Optimizer Extensibility ` page. -**A** : There are multiple reasons why Model Optimizer does not accept the mean file. See FAQs [#4](#question-4), [#5](#question-5), and [#6](#question-6). +.. _question-14: -#### Q11. What does the message "Invalid prototxt file: value error" mean? +Q14. What does the message "Cannot infer shape for node {} because there is no Caffe available. Please register python infer function for op or use Caffe for shape inference" mean? +#################################################################################################################################################################################### -**A** : There are multiple reasons why Model Optimizer does not accept a Caffe topology. See FAQs [#7](#question-7) and [#20](#question-20). +**A:** Your model contains a custom layer and you have correctly registered it with the ``CustomLayersMapping.xml`` file. These steps are required to offload shape inference of the custom layer with the help of the system Caffe. However, Model Optimizer could not import a Caffe package. Make sure that you have built Caffe with a ``pycaffe`` target and added it to the ``PYTHONPATH`` environment variable. At the same time, it is highly recommended to avoid dependency on Caffe and write your own Model Optimizer extension for your custom layer. For more information, refer to FAQ :ref:`#44 `. -#### Q12. What does the message "Error happened while constructing caffe.Net in the Caffe fallback function" mean? +.. _question-15: -**A** : Model Optimizer tried to infer a specified layer via the Caffe framework. However, it cannot construct a net using the Caffe Python interface. Make sure that your `caffemodel` and `prototxt` files are correct. To ensure that the problem is not in the `prototxt` file, see FAQ [#2](#question-2). +Q15. What does the message "Framework name can not be deduced from the given options. Use --framework to choose one of Caffe, TensorFlow, MXNet" mean? +###################################################################################################################################################### -#### Q13. What does the message "Cannot infer shapes due to exception in Caffe" mean? +**A:** You have run Model Optimizer without a flag ``--framework caffe|tf|mxnet``. Model Optimizer tries to deduce the framework by the extension of input model file (``.pb`` for TensorFlow, ``.caffemodel`` for Caffe, ``.params`` for Apache MXNet). Your input model might have a different extension and you need to explicitly set the source framework. For example, use ``--framework caffe``. -**A** : Model Optimizer tried to infer a custom layer via the Caffe framework, but the model could not be inferred using Caffe. This might happen if you try to convert the model with some noise weights and biases, which conflict with layers that have dynamic shapes. You should write your own extension for every custom layer your topology might have. For more details, refer to the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) page. +.. _question-16: -#### Q14. What does the message "Cannot infer shape for node {} because there is no Caffe available. Please register python infer function for op or use Caffe for shape inference" mean? +Q16. What does the message "Input shape is required to convert MXNet model. Please provide it with --input_shape" mean? +##################################################################################################################################################### -**A** : Your model contains a custom layer and you have correctly registered it with the `CustomLayersMapping.xml` file. These steps are required to offload shape inference of the custom layer with the help of the system Caffe. However, Model Optimizer could not import a Caffe package. Make sure that you have built Caffe with a `pycaffe` target and added it to the `PYTHONPATH` environment variable. At the same time, it is highly recommended to avoid dependency on Caffe and write your own Model Optimizer extension for your custom layer. For more information, refer to FAQ [#44](#question-44). +**A:** Input shape was not provided. That is mandatory for converting an MXNet model to the OpenVINO Intermediate Representation, because MXNet models do not contain information about input shapes. Use the ``--input_shape`` flag to specify it. For more information about using the ``--input_shape``, refer to FAQ :ref:`#56 `. -#### Q15. What does the message "Framework name can not be deduced from the given options. Use --framework to choose one of Caffe, TensorFlow, MXNet" mean? +.. _question-17: -**A** : You have run Model Optimizer without a flag `--framework caffe|tf|mxnet`. Model Optimizer tries to deduce the framework by the extension of input model file (`.pb` for TensorFlow, `.caffemodel` for Caffe, `.params` for Apache MXNet). Your input model might have a different extension and you need to explicitly set the source framework. For example, use `--framework caffe`. +Q17. What does the message "Both --mean_file and mean_values are specified. Specify either mean file or mean values" mean? +##################################################################################################################################################### -#### Q16. What does the message "Input shape is required to convert MXNet model. Please provide it with --input_shape" mean? +**A:** The ``--mean_file`` and ``--mean_values`` options are two ways of specifying preprocessing for the input. However, they cannot be used together, as it would mean double subtraction and lead to ambiguity. Choose one of these options and pass it with the corresponding CLI option. -**A** : Input shape was not provided. That is mandatory for converting an MXNet model to the OpenVINO Intermediate Representation, because MXNet models do not contain information about input shapes. Use the `--input_shape` flag to specify it. For more information about using the `--input_shape`, refer to FAQ [#56](#question-56). +.. _question-18: -#### Q17. What does the message "Both --mean_file and mean_values are specified. Specify either mean file or mean values" mean? +Q18. What does the message "Negative value specified for --mean_file_offsets option. Please specify positive integer values in format '(x,y)'" mean? +##################################################################################################################################################### -**A** : The `--mean_file` and `--mean_values` options are two ways of specifying preprocessing for the input. However, they cannot be used together, as it would mean double subtraction and lead to ambiguity. Choose one of these options and pass it with the corresponding CLI option. +**A:** You might have specified negative values with ``--mean_file_offsets``. Only positive integer values in format '(x,y)' must be used. -#### Q18. What does the message "Negative value specified for --mean_file_offsets option. Please specify positive integer values in format '(x,y)'" mean? +.. _question-19: -**A** : You might have specified negative values with `--mean_file_offsets`. Only positive integer values in format '(x,y)' must be used. +Q19. What does the message "Both --scale and --scale_values are defined. Specify either scale factor or scale values per input channels" mean? +##################################################################################################################################################### -#### Q19. What does the message "Both --scale and --scale_values are defined. Specify either scale factor or scale values per input channels" mean? +**A:** The ``--scale`` option sets a scaling factor for all channels, while ``--scale_values`` sets a scaling factor per each channel. Using both of them simultaneously produces ambiguity, so you must use only one of them. For more information, refer to the **Using Framework-Agnostic Conversion Parameters** section: for :doc:`Converting a Caffe Model `, :doc:`Converting a TensorFlow Model `, :doc:`Converting an MXNet Model `. -**A** : The `--scale` option sets a scaling factor for all channels, while `--scale_values` sets a scaling factor per each channel. Using both of them simultaneously produces ambiguity, so you must use only one of them. For more information, refer to the **Using Framework-Agnostic Conversion Parameters** section: for [Converting a Caffe Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe), [Converting a TensorFlow Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow), [Converting an MXNet Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet). +.. _question-20: -#### Q20. What does the message "Cannot find prototxt file: for Caffe please specify --input_proto - a protobuf file that stores topology and --input_model that stores pre-trained weights" mean? +Q20. What does the message "Cannot find prototxt file: for Caffe please specify --input_proto - a protobuf file that stores topology and --input_model that stores pre-trained weights" mean? +############################################################################################################################################################################################## -**A** : Model Optimizer cannot find a `.prototxt` file for a specified model. By default, it must be located in the same directory as the input model with the same name (except extension). If any of these conditions is not satisfied, use `--input_proto` to specify the path to the `.prototxt` file. +**A:** Model Optimizer cannot find a ``.prototxt`` file for a specified model. By default, it must be located in the same directory as the input model with the same name (except extension). If any of these conditions is not satisfied, use ``--input_proto`` to specify the path to the ``.prototxt`` file. -#### Q21. What does the message "Failed to create directory .. . Permission denied!" mean? +.. _question-21: -**A** : Model Optimizer cannot create a directory specified via `--output_dir`. Make sure that you have enough permissions to create the specified directory. +Q21. What does the message "Failed to create directory .. . Permission denied!" mean? +##################################################################################################################################################### -#### Q22. What does the message "Discovered data node without inputs and value" mean? +**A:** Model Optimizer cannot create a directory specified via ``--output_dir``. Make sure that you have enough permissions to create the specified directory. -**A** : One of the layers in the specified topology might not have inputs or values. Make sure that the provided `caffemodel` and `protobuf` files are correct. +.. _question-22: -#### Q23. What does the message "Part of the nodes was not translated to IE. Stopped" mean? +Q22. What does the message "Discovered data node without inputs and value" mean? +##################################################################################################################################################### -**A** : Some of the operations are not supported by OpenVINO Runtime and cannot be translated to OpenVINO Intermediate Representation. You can extend Model Optimizer by allowing generation of new types of operations and implement these operations in the dedicated OpenVINO plugins. For more information, refer to the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +**A:** One of the layers in the specified topology might not have inputs or values. Make sure that the provided ``caffemodel`` and ``protobuf`` files are correct. -#### Q24. What does the message "While creating an edge from .. to .. : node name is undefined in the graph. Check correctness of the input model" mean? +.. _question-23: -**A** : Model Optimizer cannot build a graph based on a specified model. Most likely, it is incorrect. +Q23. What does the message "Part of the nodes was not translated to IE. Stopped" mean? +##################################################################################################################################################### -#### Q25. What does the message "Node does not exist in the graph" mean? +**A:** Some of the operations are not supported by OpenVINO Runtime and cannot be translated to OpenVINO Intermediate Representation. You can extend Model Optimizer by allowing generation of new types of operations and implement these operations in the dedicated OpenVINO plugins. For more information, refer to the :doc:`OpenVINO Extensibility Mechanism ` guide. -**A** : You might have specified an output node via the `--output` flag that does not exist in a provided model. Make sure that the specified output is correct and this node exists in the current model. +.. _question-24: -#### Q26. What does the message "--input parameter was provided. Other inputs are needed for output computation. Provide more inputs or choose another place to cut the net" mean? +Q24. What does the message "While creating an edge from .. to .. : node name is undefined in the graph. Check correctness of the input model" mean? +##################################################################################################################################################### -**A** : Most likely, Model Optimizer tried to cut the model by a specified input. However, other inputs are needed. +**A:** Model Optimizer cannot build a graph based on a specified model. Most likely, it is incorrect. -#### Q27. What does the message "Placeholder node does not have an input port, but input port was provided" mean? +.. _question-25: -**A** : You might have specified a placeholder node with an input node, while the placeholder node does not have it in the model. +Q25. What does the message "Node does not exist in the graph" mean? +##################################################################################################################################################### -#### Q28. What does the message "Port index is out of number of available input ports for node" mean? +**A:** You might have specified an output node via the ``--output`` flag that does not exist in a provided model. Make sure that the specified output is correct and this node exists in the current model. -**A** : This error occurs when an incorrect input port is specified with the `--input` command line argument. When using `--input`, you may optionally specify an input port in the form: `X:node_name`, where `X` is an integer index of the input port starting from 0 and `node_name` is the name of a node in the model. This error occurs when the specified input port `X` is not in the range 0..(n-1), where n is the number of input ports for the node. Specify a correct port index, or do not use it if it is not needed. +.. _question-26: -#### Q29. What does the message "Node has more than 1 input and input shapes were provided. Try not to provide input shapes or specify input port with PORT:NODE notation, where PORT is an integer" mean? +Q26. What does the message "--input parameter was provided. Other inputs are needed for output computation. Provide more inputs or choose another place to cut the net" mean? +############################################################################################################################################################################## -**A** : This error occurs when an incorrect combination of the `--input` and `--input_shape` command line options is used. Using both `--input` and `--input_shape` is valid only if `--input` points to the `Placeholder` node, a node with one input port or `--input` has the form `PORT:NODE`, where `PORT` is an integer port index of input for node `NODE`. Otherwise, the combination of `--input` and `--input_shape` is incorrect. +**A:** Most likely, Model Optimizer tried to cut the model by a specified input. However, other inputs are needed. -@anchor FAQ30 -#### Q30. What does the message "Input port > 0 in --input is not supported if --input_shape is not provided. Node: NAME_OF_THE_NODE. Omit port index and all input ports will be replaced by placeholders. Or provide --input_shape" mean? +.. _question-27: -**A** : When using the `PORT:NODE` notation for the `--input` command line argument and `PORT` > 0, you should specify `--input_shape` for this input. This is a limitation of the current Model Optimizer implementation. +Q27. What does the message "Placeholder node does not have an input port, but input port was provided" mean? +##################################################################################################################################################### + +**A:** You might have specified a placeholder node with an input node, while the placeholder node does not have it in the model. + +.. _question-28: + +Q28. What does the message "Port index is out of number of available input ports for node" mean? +##################################################################################################################################################### + +**A:** This error occurs when an incorrect input port is specified with the ``--input`` command line argument. When using ``--input``, you may optionally specify an input port in the form: ``X:node_name``, where ``X`` is an integer index of the input port starting from 0 and ``node_name`` is the name of a node in the model. This error occurs when the specified input port ``X`` is not in the range 0..(n-1), where n is the number of input ports for the node. Specify a correct port index, or do not use it if it is not needed. + +.. _question-29: + +Q29. What does the message "Node has more than 1 input and input shapes were provided. Try not to provide input shapes or specify input port with PORT:NODE notation, where PORT is an integer" mean? +###################################################################################################################################################################################################### + +**A:** This error occurs when an incorrect combination of the ``--input`` and ``--input_shape`` command line options is used. Using both ``--input`` and ``--input_shape`` is valid only if ``--input`` points to the ``Placeholder`` node, a node with one input port or ``--input`` has the form ``PORT:NODE``, where ``PORT`` is an integer port index of input for node ``NODE``. Otherwise, the combination of ``--input`` and ``--input_shape`` is incorrect. + + +.. _question-30: + +Q30. What does the message "Input port > 0 in --input is not supported if --input_shape is not provided. Node: NAME_OF_THE_NODE. Omit port index and all input ports will be replaced by placeholders. Or provide --input_shape" mean? +####################################################################################################################################################################################################################################### + +**A:** When using the ``PORT:NODE`` notation for the ``--input`` command line argument and ``PORT`` > 0, you should specify ``--input_shape`` for this input. This is a limitation of the current Model Optimizer implementation. > **NOTE**: It is no longer relevant message since the limitation on input port index for model truncation has been resolved. -#### Q31. What does the message "No or multiple placeholders in the model, but only one shape is provided, cannot set it" mean? +.. _question-31: + +Q31. What does the message "No or multiple placeholders in the model, but only one shape is provided, cannot set it" mean? +##################################################################################################################################################### + +**A:** You might have provided only one shape for the placeholder, while there are none or multiple inputs in the model. Make sure that you have provided the correct data for placeholder nodes. + +.. _question-32: + +Q32. What does the message "The amount of input nodes for port is not equal to 1" mean? +##################################################################################################################################################### + +**A:** This error occurs when the ``SubgraphMatch.single_input_node`` function is used for an input port that supplies more than one node in a sub-graph. The ``single_input_node`` function can be used only for ports that has a single consumer inside the matching sub-graph. When multiple nodes are connected to the port, use the ``input_nodes`` function or ``node_by_pattern`` function instead of ``single_input_node``. For more details, refer to the **Graph Transformation Extensions** section in the :doc:`Model Optimizer Extensibility ` guide. + +.. _question-33: + +Q33. What does the message "Output node for port has already been specified" mean? +##################################################################################################################################################### + +**A:** This error occurs when the ``SubgraphMatch._add_output_node`` function is called manually from user's extension code. This is an internal function, and you should not call it directly. + +.. _question-34: + +Q34. What does the message "Unsupported match kind.... Match kinds "points" or "scope" are supported only" mean? +##################################################################################################################################################### + +**A:** While using configuration file to implement a TensorFlow front replacement extension, an incorrect match kind was used. Only ``points`` or ``scope`` match kinds are supported. For more details, refer to the :doc:`Model Optimizer Extensibility ` guide. -**A** : You might have provided only one shape for the placeholder, while there are none or multiple inputs in the model. Make sure that you have provided the correct data for placeholder nodes. +.. _question-35: -#### Q32. What does the message "The amount of input nodes for port is not equal to 1" mean? +Q35. What does the message "Cannot write an event file for the TensorBoard to directory" mean? +##################################################################################################################################################### -**A** : This error occurs when the `SubgraphMatch.single_input_node` function is used for an input port that supplies more than one node in a sub-graph. The `single_input_node` function can be used only for ports that has a single consumer inside the matching sub-graph. When multiple nodes are connected to the port, use the `input_nodes` function or `node_by_pattern` function instead of `single_input_node`. For more details, refer to the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) guide. +**A:** Model Optimizer tried to write an event file in the specified directory but failed to do that. That could happen when the specified directory does not exist or you do not have permissions to write in it. -#### Q33. What does the message "Output node for port has already been specified" mean? +.. _question-36: -**A** : This error occurs when the `SubgraphMatch._add_output_node` function is called manually from user's extension code. This is an internal function, and you should not call it directly. +Q36. What does the message "There is no registered 'infer' function for node with op = .. . Please implement this function in the extensions" mean? +##################################################################################################################################################### -#### Q34. What does the message "Unsupported match kind.... Match kinds "points" or "scope" are supported only" mean? +**A** Most likely, you tried to extend Model Optimizer with a new primitive, but you did not specify an infer function. For more information on extensions, see the :doc:`OpenVINO Extensibility Mechanism ` guide. -**A** : While using configuration file to implement a TensorFlow front replacement extension, an incorrect match kind was used. Only `points` or `scope` match kinds are supported. For more details, refer to the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) guide. +.. _question-37: -#### Q35. What does the message "Cannot write an event file for the TensorBoard to directory" mean? +Q37. What does the message "Stopped shape/value propagation at node" mean? +##################################################################################################################################################### -**A** : Model Optimizer tried to write an event file in the specified directory but failed to do that. That could happen when the specified directory does not exist or you do not have permissions to write in it. +**A:** Model Optimizer cannot infer shapes or values for the specified node. It can happen because of the following reasons: a bug exists in the custom shape infer function, the node inputs have incorrect values/shapes, or the input shapes are incorrect. -#### Q36. What does the message "There is no registered 'infer' function for node with op = .. . Please implement this function in the extensions" mean? +.. _question-38: -**A** Most likely, you tried to extend Model Optimizer with a new primitive, but you did not specify an infer function. For more information on extensions, see the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +Q38. What does the message "The input with shape .. does not have the batch dimension" mean? +##################################################################################################################################################### -#### Q37. What does the message "Stopped shape/value propagation at node" mean? +**A:** Batch dimension is the first dimension in the shape and it should be equal to 1 or undefined. In your case, it is not either equal to 1 or undefined, which is why the ``-b`` shortcut produces undefined and unspecified behavior. To resolve the issue, specify full shapes for each input with the ``--input_shape`` option. Run Model Optimizer with the ``--help`` option to learn more about the notation for input shapes. -**A** : Model Optimizer cannot infer shapes or values for the specified node. It can happen because of the following reasons: a bug exists in the custom shape infer function, the node inputs have incorrect values/shapes, or the input shapes are incorrect. +.. _question-39: -#### Q38. What does the message "The input with shape .. does not have the batch dimension" mean? +Q39. What does the message "Not all output shapes were inferred or fully defined for node" mean? +##################################################################################################################################################### -**A** : Batch dimension is the first dimension in the shape and it should be equal to 1 or undefined. In your case, it is not either equal to 1 or undefined, which is why the `-b` shortcut produces undefined and unspecified behavior. To resolve the issue, specify full shapes for each input with the `--input_shape` option. Run Model Optimizer with the `--help` option to learn more about the notation for input shapes. +**A:** Most likely, the shape is not defined (partially or fully) for the specified node. You can use ``--input_shape`` with positive integers to override model input shapes. -#### Q39. What does the message "Not all output shapes were inferred or fully defined for node" mean? +.. _question-40: -**A** : Most likely, the shape is not defined (partially or fully) for the specified node. You can use `--input_shape` with positive integers to override model input shapes. +Q40. What does the message "Shape for tensor is not defined. Can not proceed" mean? +##################################################################################################################################################### -#### Q40. What does the message "Shape for tensor is not defined. Can not proceed" mean? +**A:** This error occurs when the ``--input`` command-line option is used to cut a model and ``--input_shape`` is not used to override shapes for a node, so a shape for the node cannot be inferred by Model Optimizer. You need to help Model Optimizer by specifying shapes with ``--input_shape`` for each node specified with the ``--input`` command-line option. -**A** : This error occurs when the `--input` command-line option is used to cut a model and `--input_shape` is not used to override shapes for a node, so a shape for the node cannot be inferred by Model Optimizer. You need to help Model Optimizer by specifying shapes with `--input_shape` for each node specified with the `--input` command-line option. +.. _question-41: -#### Q41. What does the message "Module TensorFlow was not found. Please install TensorFlow 1.2 or higher" mean? +Q41. What does the message "Module TensorFlow was not found. Please install TensorFlow 1.2 or higher" mean? +##################################################################################################################################################### -**A** : To convert TensorFlow models with Model Optimizer, TensorFlow 1.2 or newer must be installed. For more information on prerequisites, see the [Configuring Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) guide. +**A:** To convert TensorFlow models with Model Optimizer, TensorFlow 1.2 or newer must be installed. For more information on prerequisites, see the :doc:`Configuring Model Optimizer ` guide. -#### Q42. What does the message "Cannot read the model file: it is incorrect TensorFlow model file or missing" mean? +.. _question-42: -**A** : The model file should contain a frozen TensorFlow graph in the text or binary format. Make sure that `--input_model_is_text` is provided for a model in the text format. By default, a model is interpreted as binary file. +Q42. What does the message "Cannot read the model file: it is incorrect TensorFlow model file or missing" mean? +##################################################################################################################################################### -#### Q43. What does the message "Cannot pre-process TensorFlow graph after reading from model file. File is corrupt or has unsupported format" mean? +**A:** The model file should contain a frozen TensorFlow graph in the text or binary format. Make sure that ``--input_model_is_text`` is provided for a model in the text format. By default, a model is interpreted as binary file. -**A** : Most likely, there is a problem with the specified file for the model. The file exists, but it has an invalid format or is corrupted. +.. _question-43: -#### Q44. What does the message "Found custom layer. Model Optimizer does not support this layer. Please, register it in CustomLayersMapping.xml or implement extension" mean? +Q43. What does the message "Cannot pre-process TensorFlow graph after reading from model file. File is corrupt or has unsupported format" mean? +##################################################################################################################################################### -**A** : This means that the layer `{layer_name}` is not supported in Model Optimizer. You will find a list of all unsupported layers in the corresponding section. You should implement the extensions for this layer. See [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) for more information. +**A:** Most likely, there is a problem with the specified file for the model. The file exists, but it has an invalid format or is corrupted. -#### Q45. What does the message "Custom replacement configuration file does not exist" mean? +.. _question-44: -**A** : A path to the custom replacement configuration file was provided with the `--transformations_config` flag, but the file could not be found. Make sure the specified path is correct and the file exists. +Q44. What does the message "Found custom layer. Model Optimizer does not support this layer. Please, register it in CustomLayersMapping.xml or implement extension" mean? +########################################################################################################################################################################## -#### Q46. What does the message "Extractors collection have case insensitive duplicates" mean? +**A:** This means that the layer ``{layer_name}`` is not supported in Model Optimizer. You will find a list of all unsupported layers in the corresponding section. You should implement the extensions for this layer. See :doc:`OpenVINO Extensibility Mechanism ` for more information. -**A** : When extending Model Optimizer with new primitives, keep in mind that their names are case-insensitive. Most likely, another operation with the same name is already defined. For more information, see the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +.. _question-45: -#### Q47. What does the message "Input model name is not in an expected format, cannot extract iteration number" mean? +Q45. What does the message "Custom replacement configuration file does not exist" mean? +##################################################################################################################################################### -**A** : Model Optimizer cannot load an MXNet model in the specified file format. Make sure you use the `.json` or `.param` format. +**A:** A path to the custom replacement configuration file was provided with the ``--transformations_config`` flag, but the file could not be found. Make sure the specified path is correct and the file exists. -#### Q48. What does the message "Cannot convert type of placeholder because not all of its outputs are 'Cast' to float operations" mean? +.. _question-46: -**A** : There are models where `Placeholder` has the UINT8 type and the first operation after it is 'Cast', which casts the input to FP32. Model Optimizer detected that the `Placeholder` has the UINT8 type, but the next operation is not 'Cast' to float. Model Optimizer does not support such a case. Make sure you change the model to have `Placeholder` for FP32. +Q46. What does the message "Extractors collection have case insensitive duplicates" mean? +##################################################################################################################################################### -#### Q49. What does the message "Data type is unsupported" mean? +**A:** When extending Model Optimizer with new primitives, keep in mind that their names are case-insensitive. Most likely, another operation with the same name is already defined. For more information, see the :doc:`OpenVINO Extensibility Mechanism ` guide. -**A** : Model Optimizer cannot read the value with the specified data type. Currently, the following types are supported: bool, float16, float32, double, int8, int16, int32, int64, uint8, uint16, uint32, uint64, str. +.. _question-47: -#### Q50. What does the message "No node with name ..." mean? +Q47. What does the message "Input model name is not in an expected format, cannot extract iteration number" mean? +##################################################################################################################################################### -**A** : Model Optimizer tried to access a node that does not exist. This could happen if you have incorrectly specified placeholder, input or output node name. +**A:** Model Optimizer cannot load an MXNet model in the specified file format. Make sure you use the ``.json`` or ``.param`` format. -#### Q51. What does the message "Module MXNet was not found. Please install MXNet 1.0.0" mean? +.. _question-48: -**A** : To convert MXNet models with Model Optimizer, Apache MXNet 1.0.0 must be installed. For more information about prerequisites, see the[Configuring Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) guide. +Q48. What does the message "Cannot convert type of placeholder because not all of its outputs are 'Cast' to float operations" mean? +##################################################################################################################################################### -#### Q52. What does the message "The following error happened while loading MXNet model .." mean? +**A:** There are models where ``Placeholder`` has the UINT8 type and the first operation after it is 'Cast', which casts the input to FP32. Model Optimizer detected that the ``Placeholder`` has the UINT8 type, but the next operation is not 'Cast' to float. Model Optimizer does not support such a case. Make sure you change the model to have ``Placeholder`` for FP32. -**A** : Most likely, there is a problem with loading of the MXNet model. Make sure the specified path is correct, the model exists and is not corrupted, and you have sufficient permissions to work with it. +.. _question-49: -#### Q53. What does the message "The following error happened while processing input shapes: .." mean? +Q49. What does the message "Data type is unsupported" mean? +##################################################################################################################################################### -**A** : Make sure inputs are defined and have correct shapes. You can use `--input_shape` with positive integers to override model input shapes. +**A:** Model Optimizer cannot read the value with the specified data type. Currently, the following types are supported: bool, float16, float32, double, int8, int16, int32, int64, uint8, uint16, uint32, uint64, str. -#### Q54. What does the message "Attempt to register of custom name for the second time as class. Note that custom names are case-insensitive" mean? +.. _question-50: -**A** : When extending Model Optimizer with new primitives, keep in mind that their names are case-insensitive. Most likely, another operation with the same name is already defined. For more information, see the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +Q50. What does the message "No node with name ..." mean? +##################################################################################################################################################### -#### Q55. What does the message "Both --input_shape and --batch were provided. Please, provide only one of them" mean? +**A:** Model Optimizer tried to access a node that does not exist. This could happen if you have incorrectly specified placeholder, input or output node name. -**A** : Specifying the batch and the input shapes at the same time is not supported. You must specify a desired batch as the first value of the input shape. +.. _question-51: -#### Q56. What does the message "Input shape .. cannot be parsed" mean? +Q51. What does the message "Module MXNet was not found. Please install MXNet 1.0.0" mean? +##################################################################################################################################################### -**A** : The specified input shape cannot be parsed. Define it in one of the following ways: +**A:** To convert MXNet models with Model Optimizer, Apache MXNet 1.0.0 must be installed. For more information about prerequisites, see the :doc:`Configuring Model Optimizer ` guide. + +.. _question-52: + +Q52. What does the message "The following error happened while loading MXNet model .." mean? +##################################################################################################################################################### + +**A:** Most likely, there is a problem with loading of the MXNet model. Make sure the specified path is correct, the model exists and is not corrupted, and you have sufficient permissions to work with it. + +.. _question-53: + +Q53. What does the message "The following error happened while processing input shapes: .." mean? +##################################################################################################################################################### + +**A:** Make sure inputs are defined and have correct shapes. You can use ``--input_shape`` with positive integers to override model input shapes. + +.. _question-54: + +Q54. What does the message "Attempt to register of custom name for the second time as class. Note that custom names are case-insensitive" mean? +##################################################################################################################################################### + +**A:** When extending Model Optimizer with new primitives, keep in mind that their names are case-insensitive. Most likely, another operation with the same name is already defined. For more information, see the :doc:`OpenVINO Extensibility Mechanism ` guide. + +.. _question-55: + +Q55. What does the message "Both --input_shape and --batch were provided. Please, provide only one of them" mean? +##################################################################################################################################################### + +**A:** Specifying the batch and the input shapes at the same time is not supported. You must specify a desired batch as the first value of the input shape. + +.. _question-56: + +Q56. What does the message "Input shape .. cannot be parsed" mean? +##################################################################################################################################################### + +**A:** The specified input shape cannot be parsed. Define it in one of the following ways: * -```shell - mo --input_model .caffemodel --input_shape (1,3,227,227) -``` + + .. code-block:: shell + + mo --input_model .caffemodel --input_shape (1,3,227,227) + * -```shell - mo --input_model .caffemodel --input_shape [1,3,227,227] -``` -* In case of multi input topology you should also specify inputs: -```shell - mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),(1,6,1,1) -``` + + .. code-block:: shell + + mo --input_model .caffemodel --input_shape [1,3,227,227] + +* In case of multi input topology you should also specify inputs: + + .. code-block:: shell + + mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),(1,6,1,1) + Keep in mind that there is no space between and inside the brackets for input shapes. -#### Q57. What does the message "Please provide input layer names for input layer shapes" mean? +.. _question-57: + +Q57. What does the message "Please provide input layer names for input layer shapes" mean? +##################################################################################################################################################### + +**A:** When specifying input shapes for several layers, you must provide names for inputs, whose shapes will be overwritten. For usage examples, see the :doc:`Converting a Caffe Model `. Additional information for ``--input_shape`` is in FAQ :ref:`#56 `. + +.. _question-58: + +Q58. What does the message "Values cannot be parsed" mean? +##################################################################################################################################################### + +**A:** Mean values for the given parameter cannot be parsed. It should be a string with a list of mean values. For example, in '(1,2,3)', 1 stands for the RED channel, 2 for the GREEN channel, 3 for the BLUE channel. + +.. _question-59: + +Q59. What does the message ".. channels are expected for given values" mean? +##################################################################################################################################################### + +**A:** The number of channels and the number of given values for mean values do not match. The shape should be defined as '(R,G,B)' or '[R,G,B]'. The shape should not contain undefined dimensions (? or -1). The order of values is as follows: (value for a RED channel, value for a GREEN channel, value for a BLUE channel). + +.. _question-60: + +Q60. What does the message "You should specify input for each mean value" mean? +##################################################################################################################################################### + +**A:** Most likely, you didn't specify inputs using ``--mean_values``. Specify inputs with the ``--input`` flag. For usage examples, refer to the FAQ :ref:`#62 `. + +.. _question-61: + +Q61. What does the message "You should specify input for each scale value" mean? +##################################################################################################################################################### + +**A:** Most likely, you didn't specify inputs using ``--scale_values``. Specify inputs with the ``--input`` flag. For usage examples, refer to the FAQ :ref:`#63 `. + +.. _question-62: + +Q62. What does the message "Number of inputs and mean values does not match" mean? +##################################################################################################################################################### + +**A:** The number of specified mean values and the number of inputs must be equal. For a usage example, refer to the :doc:`Converting a Caffe Model ` guide. + +.. _question-63: + +Q63. What does the message "Number of inputs and scale values does not match" mean? +##################################################################################################################################################### + +**A:** The number of specified scale values and the number of inputs must be equal. For a usage example, refer to the :doc:`Converting a Caffe Model ` guide. -**A** : When specifying input shapes for several layers, you must provide names for inputs, whose shapes will be overwritten. For usage examples, see the [Converting a Caffe Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe). Additional information for `--input_shape` is in FAQ [#56](#question-56). +.. _question-64: -#### Q58. What does the message "Values cannot be parsed" mean? +Q64. What does the message "No class registered for match kind ... Supported match kinds are .. " mean? +##################################################################################################################################################### -**A** : Mean values for the given parameter cannot be parsed. It should be a string with a list of mean values. For example, in '(1,2,3)', 1 stands for the RED channel, 2 for the GREEN channel, 3 for the BLUE channel. +**A:** A replacement defined in the configuration file for sub-graph replacement, using node names patterns or start/end nodes, has the ``match_kind`` attribute. The attribute may have only one of the values: ``scope`` or ``points``. If a different value is provided, this error is displayed. -#### Q59. What does the message ".. channels are expected for given values" mean? +.. _question-65: -**A** : The number of channels and the number of given values for mean values do not match. The shape should be defined as '(R,G,B)' or '[R,G,B]'. The shape should not contain undefined dimensions (? or -1). The order of values is as follows: (value for a RED channel, value for a GREEN channel, value for a BLUE channel). +Q65. What does the message "No instance(s) is(are) defined for the custom replacement" mean? +##################################################################################################################################################### -#### Q60. What does the message "You should specify input for each mean value" mean? +**A:** A replacement defined in the configuration file for sub-graph replacement, using node names patterns or start/end nodes, has the ``instances`` attribute. This attribute is mandatory. This error will occur if the attribute is missing. For more details, refer to the **Graph Transformation Extensions** section in the :doc:`Model Optimizer Extensibility ` guide. -**A** : Most likely, you didn't specify inputs using `--mean_values`. Specify inputs with the `--input` flag. For usage examples, refer to the FAQ [#62](#question-62). +.. _question-66: -#### Q61. What does the message "You should specify input for each scale value" mean? +Q66. What does the message "The instance must be a single dictionary for the custom replacement with id .." mean? +##################################################################################################################################################### -**A** : Most likely, you didn't specify inputs using `--scale_values`. Specify inputs with the `--input` flag. For usage examples, refer to the FAQ [#63](#question-63). +**A:** A replacement defined in the configuration file for sub-graph replacement, using start/end nodes, has the ``instances`` attribute. For this type of replacement, the instance must be defined with a dictionary with two keys ``start_points`` and ``end_points``. Values for these keys are lists with the start and end node names, respectively. For more details, refer to the **Graph Transformation Extensions** section in the :doc:`Model Optimizer Extensibility ` guide. -#### Q62. What does the message "Number of inputs and mean values does not match" mean? +.. _question-67: -**A** : The number of specified mean values and the number of inputs must be equal. For a usage example, refer to the [Converting a Caffe Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe) guide. +Q67. What does the message "No instances are defined for replacement with id .. " mean? +##################################################################################################################################################### -#### Q63. What does the message "Number of inputs and scale values does not match" mean? +**A:** A replacement for the specified id is not defined in the configuration file. For more information, refer to the FAQ :ref:`#65 `. -**A** : The number of specified scale values and the number of inputs must be equal. For a usage example, refer to the [Converting a Caffe Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe) guide. +.. _question-68: -#### Q64. What does the message "No class registered for match kind ... Supported match kinds are .. " mean? +Q68. What does the message "Custom replacements configuration file .. does not exist" mean? +##################################################################################################################################################### -**A** : A replacement defined in the configuration file for sub-graph replacement, using node names patterns or start/end nodes, has the `match_kind` attribute. The attribute may have only one of the values: `scope` or `points`. If a different value is provided, this error is displayed. +**A:** The path to a custom replacement configuration file was provided with the ``--transformations_config`` flag, but it cannot be found. Make sure the specified path is correct and the file exists. -#### Q65. What does the message "No instance(s) is(are) defined for the custom replacement" mean? +.. _question-69: -**A** : A replacement defined in the configuration file for sub-graph replacement, using node names patterns or start/end nodes, has the `instances` attribute. This attribute is mandatory. This error will occur if the attribute is missing. For more details, refer to the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) guide. +Q69. What does the message "Failed to parse custom replacements configuration file .." mean? +##################################################################################################################################################### -#### Q66. What does the message "The instance must be a single dictionary for the custom replacement with id .." mean? +**A:** The file for custom replacement configuration provided with the ``--transformations_config`` flag cannot be parsed. In particular, it should have a valid JSON structure. For more details, refer to the `JSON Schema Reference `__ page. -**A** : A replacement defined in the configuration file for sub-graph replacement, using start/end nodes, has the `instances` attribute. For this type of replacement, the instance must be defined with a dictionary with two keys `start_points` and `end_points`. Values for these keys are lists with the start and end node names, respectively. For more details, refer to the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) guide. +.. _question-70: -#### Q67. What does the message "No instances are defined for replacement with id .. " mean? +Q70. What does the message "One of the custom replacements in the configuration file .. does not contain attribute 'id'" mean? +##################################################################################################################################################### -**A** : A replacement for the specified id is not defined in the configuration file. For more information, refer to the FAQ [#65](#question-65). +**A:** Every custom replacement should declare a set of mandatory attributes and their values. For more details, refer to FAQ :ref:`#71 `. -#### Q68. What does the message "Custom replacements configuration file .. does not exist" mean? +.. _question-71: -**A** : The path to a custom replacement configuration file was provided with the `--transformations_config` flag, but it cannot be found. Make sure the specified path is correct and the file exists. +Q71. What does the message "File .. validation failed" mean? +##################################################################################################################################################### -#### Q69. What does the message "Failed to parse custom replacements configuration file .." mean? +**A:** The file for custom replacement configuration provided with the ``--transformations_config`` flag cannot pass validation. Make sure you have specified ``id``, ``instances``, and ``match_kind`` for all the patterns. -**A** : The file for custom replacement configuration provided with the `--transformations_config` flag cannot be parsed. In particular, it should have a valid JSON structure. For more details, refer to the [JSON Schema Reference](https://spacetelescope.github.io/understanding-json-schema/reference/index.html) page. +.. _question-72: -#### Q70. What does the message "One of the custom replacements in the configuration file .. does not contain attribute 'id'" mean? +Q72. What does the message "Cannot update the file .. because it is broken" mean? +##################################################################################################################################################### -**A** : Every custom replacement should declare a set of mandatory attributes and their values. For more details, refer to FAQ [#71](#question-71). +**A:** The custom replacement configuration file provided with the ``--tensorflow_custom_operations_config_update`` cannot be parsed. Make sure that the file is correct and refer to FAQ :ref:`#68 `, :ref:`#69 `, :ref:`#70 `, and :ref:`#71 `. -#### Q71. What does the message "File .. validation failed" mean? +.. _question-73: -**A** : The file for custom replacement configuration provided with the `--transformations_config` flag cannot pass validation. Make sure you have specified `id`, `instances`, and `match_kind` for all the patterns. +Q73. What does the message "End node .. is not reachable from start nodes: .." mean? +##################################################################################################################################################### -#### Q72. What does the message "Cannot update the file .. because it is broken" mean? +**A:** This error occurs when you try to make a sub-graph match. It is detected that between the start and end nodes that were specified as inputs/outputs for the subgraph to find, there are nodes marked as outputs but there is no path from them to the input nodes. Make sure the subgraph you want to match does actually contain all the specified output nodes. -**A** : The custom replacement configuration file provided with the `--tensorflow_custom_operations_config_update` cannot be parsed. Make sure that the file is correct and refer to FAQ [#68](#question-68), [#69](#question-69), [#70](#question-70), and [#71](#question-71). +.. _question-74: -#### Q73. What does the message "End node .. is not reachable from start nodes: .." mean? +Q74. What does the message "Sub-graph contains network input node .." mean? +##################################################################################################################################################### -**A** : This error occurs when you try to make a sub-graph match. It is detected that between the start and end nodes that were specified as inputs/outputs for the subgraph to find, there are nodes marked as outputs but there is no path from them to the input nodes. Make sure the subgraph you want to match does actually contain all the specified output nodes. +**A:** The start or end node for the sub-graph replacement using start/end nodes is specified incorrectly. Model Optimizer finds internal nodes of the sub-graph strictly "between" the start and end nodes, and then adds all input nodes to the sub-graph (and the inputs of their inputs, etc.) for these "internal" nodes. This error reports that Model Optimizer reached input node during this phase. This means that the start/end points are specified incorrectly in the configuration file. For more details, refer to the **Graph Transformation Extensions** section in the :doc:`Model Optimizer Extensibility ` guide. -#### Q74. What does the message "Sub-graph contains network input node .." mean? +.. _question-75: -**A** : The start or end node for the sub-graph replacement using start/end nodes is specified incorrectly. Model Optimizer finds internal nodes of the sub-graph strictly "between" the start and end nodes, and then adds all input nodes to the sub-graph (and the inputs of their inputs, etc.) for these "internal" nodes. This error reports that Model Optimizer reached input node during this phase. This means that the start/end points are specified incorrectly in the configuration file. For more details, refer to the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) guide. +Q75. What does the message "... elements of ... were clipped to infinity while converting a blob for node [...] to ..." mean? +##################################################################################################################################################### -#### Q75. What does the message "... elements of ... were clipped to infinity while converting a blob for node [...] to ..." mean? +**A:** This message may appear when the ``--compress_to_fp16`` (or deprecated ``--data_type``) command-line option is used. This option implies compression of all the model weights, biases, and other constant values to FP16. If a value of a constant is out of the range of valid FP16 values, the value is converted to positive or negative infinity. It may lead to incorrect results of inference or may not be a problem, depending on the model. The number of such elements and the total number of elements in the constant value is printed out together with the name of the node, where this value is used. -**A** : This message may appear when the `--compress_to_fp16` (or deprecated `--data_type`) command-line option is used. This option implies compression of all the model weights, biases, and other constant values to FP16. If a value of a constant is out of the range of valid FP16 values, the value is converted to positive or negative infinity. It may lead to incorrect results of inference or may not be a problem, depending on the model. The number of such elements and the total number of elements in the constant value is printed out together with the name of the node, where this value is used. +.. _question-76: -#### Q76. What does the message "... elements of ... were clipped to zero while converting a blob for node [...] to ..." mean? +Q76. What does the message "... elements of ... were clipped to zero while converting a blob for node [...] to ..." mean? +##################################################################################################################################################### -**A** : This message may appear when the `--compress_to_fp16` (or deprecated `--data_type`) command-line option is used. This option implies conversion of all blobs in the mode to FP16. If a value in the blob is so close to zero that it cannot be represented as a valid FP16 value, it is converted to a true zero FP16 value. Depending on the model, it may lead to incorrect results of inference or may not be a problem. The number of such elements and the total number of elements in the blob are printed out together with a name of the node, where this blob is used. +**A:** This message may appear when the ``--compress_to_fp16`` (or deprecated ``--data_type``) command-line option is used. This option implies conversion of all blobs in the mode to FP16. If a value in the blob is so close to zero that it cannot be represented as a valid FP16 value, it is converted to a true zero FP16 value. Depending on the model, it may lead to incorrect results of inference or may not be a problem. The number of such elements and the total number of elements in the blob are printed out together with a name of the node, where this blob is used. -#### Q77. What does the message "The amount of nodes matched pattern ... is not equal to 1" mean? +.. _question-77: -**A** : This error occurs when the `SubgraphMatch.node_by_pattern` function is used with a pattern that does not uniquely identify a single node in a sub-graph. Try to extend the pattern string to make unambiguous match to a single sub-graph node. For more details, refer to the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) guide. +Q77. What does the message "The amount of nodes matched pattern ... is not equal to 1" mean? +##################################################################################################################################################### -#### Q78. What does the message "The topology contains no "input" layers" mean? +**A:** This error occurs when the ``SubgraphMatch.node_by_pattern`` function is used with a pattern that does not uniquely identify a single node in a sub-graph. Try to extend the pattern string to make unambiguous match to a single sub-graph node. For more details, refer to the **Graph Transformation Extensions** section in the :doc:`Model Optimizer Extensibility ` guide. -**A** : Your Caffe topology `.prototxt` file is intended for training. Model Optimizer expects a deployment-ready `.prototxt` file. To fix the problem, prepare a deployment-ready `.prototxt` file. Preparation of a deploy-ready topology usually results in removing `data` layer(s), adding `input` layer(s), and removing loss layer(s). +.. _question-78: -#### Q79. What does the message "Warning: please expect that Model Optimizer conversion might be slow" mean? +Q78. What does the message "The topology contains no "input" layers" mean? +##################################################################################################################################################### -**A** : You are using an unsupported Python version. Use only versions 3.4 - 3.6 for the C++ `protobuf` implementation that is supplied with OpenVINO toolkit. You can still boost the conversion speed by building the protobuf library from sources. For complete instructions about building `protobuf` from sources, see the appropriate section in the[Converting a Model to Intermediate Representation](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) guide. +**A:** Your Caffe topology ``.prototxt`` file is intended for training. Model Optimizer expects a deployment-ready ``.prototxt`` file. To fix the problem, prepare a deployment-ready ``.prototxt`` file. Preparation of a deploy-ready topology usually results in removing ``data`` layer(s), adding ``input`` layer(s), and removing loss layer(s). -#### Q80. What does the message "Arguments --nd_prefix_name, --pretrained_model_name and --input_symbol should be provided. Please provide all or do not use any." mean? +.. _question-79: -**A** : This error occurs if you did not provide the `--nd_prefix_name`, `--pretrained_model_name`, and `--input_symbol` parameters. -Model Optimizer requires both `.params` and `.nd` model files to merge into the result file (`.params`). -Topology description (`.json` file) should be prepared (merged) in advance and provided with the `--input_symbol` parameter. +Q79. What does the message "Warning: please expect that Model Optimizer conversion might be slow" mean? +##################################################################################################################################################### -If you add additional layers and weights that are in `.nd` files to your model, Model Optimizer can build a model -from one `.params` file and two additional `.nd` files (`*_args.nd`, `*_auxs.nd`). +**A:** You are using an unsupported Python version. Use only versions 3.4 - 3.6 for the C++ ``protobuf`` implementation that is supplied with OpenVINO toolkit. You can still boost the conversion speed by building the protobuf library from sources. For complete instructions about building ``protobuf`` from sources, see the appropriate section in the :doc:`Converting a Model to Intermediate Representation ` guide. + +.. _question-80: + +Q80. What does the message "Arguments --nd_prefix_name, --pretrained_model_name and --input_symbol should be provided. Please provide all or do not use any." mean? +#################################################################################################################################################################### + +**A:** This error occurs if you did not provide the ``--nd_prefix_name``, ``--pretrained_model_name``, and ``--input_symbol`` parameters. +Model Optimizer requires both ``.params`` and ``.nd`` model files to merge into the result file (``.params``). +Topology description (``.json`` file) should be prepared (merged) in advance and provided with the ``--input_symbol`` parameter. + +If you add additional layers and weights that are in ``.nd`` files to your model, Model Optimizer can build a model +from one ``.params`` file and two additional ``.nd`` files (``*_args.nd``, ``*_auxs.nd``). To do that, provide both CLI options or do not pass them if you want to convert an MXNet model without additional weights. -For more information, refer to the [Converting an MXNet Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet) guide. +For more information, refer to the :doc:`Converting an MXNet Model ` guide. + +.. _question-81: + +Q81. What does the message "You should specify input for mean/scale values" mean? +##################################################################################################################################################### + +**A:** When the model has multiple inputs and you want to provide mean/scale values, you need to pass those values for each input. More specifically, the number of passed values should be the same as the number of inputs of the model. +For more information, refer to the :doc:`Converting a Model to Intermediate Representation ` guide. + +.. _question-82: + +Q82. What does the message "Input with name ... not found!" mean? +##################################################################################################################################################### + +**A:** When you passed the mean/scale values and specify names of input layers of the model, you might have used the name that does not correspond to any input layer. Make sure that you list only names of the input layers of your model when passing values with the ``--input`` option. +For more information, refer to the :doc:`Converting a Model to Intermediate Representation ` guide. + -#### Q81. What does the message "You should specify input for mean/scale values" mean? +.. _question-83: -**A** : When the model has multiple inputs and you want to provide mean/scale values, you need to pass those values for each input. More specifically, the number of passed values should be the same as the number of inputs of the model. -For more information, refer to the [Converting a Model to Intermediate Representation](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. +Q83. What does the message "Specified input json ... does not exist" mean? +##################################################################################################################################################### -#### Q82. What does the message "Input with name ... not found!" mean? +**A:** Most likely, ``.json`` file does not exist or has a name that does not match the notation of Apache MXNet. Make sure the file exists and has a correct name. +For more information, refer to the :doc:`Converting an MXNet Model ` guide. -**A** : When you passed the mean/scale values and specify names of input layers of the model, you might have used the name that does not correspond to any input layer. Make sure that you list only names of the input layers of your model when passing values with the `--input` option. -For more information, refer to the [Converting a Model to Intermediate Representation](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. +.. _question-84: -#### Q83. What does the message "Specified input json ... does not exist" mean? +Q84. What does the message "Unsupported Input model file type ... Model Optimizer support only .params and .nd files format" mean? +##################################################################################################################################################### -**A** : Most likely, `.json` file does not exist or has a name that does not match the notation of Apache MXNet. Make sure the file exists and has a correct name. -For more information, refer to the [Converting an MXNet Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet) guide. +**A:** Model Optimizer for Apache MXNet supports only ``.params`` and ``.nd`` files formats. Most likely, you specified an unsupported file format in ``--input_model``. +For more information, refer to :doc:`Converting an MXNet Model `. -#### Q84. What does the message "Unsupported Input model file type ... Model Optimizer support only .params and .nd files format" mean? -**A** : Model Optimizer for Apache MXNet supports only `.params` and `.nd` files formats. Most likely, you specified an unsupported file format in `--input_model`. -For more information, refer to [Converting an MXNet Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet). +.. _question-85: -#### Q85. What does the message "Operation ... not supported. Please register it as custom op" mean? +Q85. What does the message "Operation ... not supported. Please register it as custom op" mean? +##################################################################################################################################################### -**A** : Model Optimizer tried to load the model that contains some unsupported operations. +**A:** Model Optimizer tried to load the model that contains some unsupported operations. If you want to convert model that contains unsupported operations, you need to prepare extension for all such operations. -For more information, refer to the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +For more information, refer to the :doc:`OpenVINO Extensibility Mechanism ` guide. -#### Q86. What does the message "Can not register Op ... Please, call function 'register_caffe_python_extractor' with parameter 'name'" mean? +.. _question-86: -**A** : This error appears if the class of implementation of `Op` for Python Caffe layer could not be used by Model Optimizer. Python layers should be handled differently comparing to ordinary Caffe layers. +Q86. What does the message "Can not register Op ... Please, call function 'register_caffe_python_extractor' with parameter 'name'" mean? +##################################################################################################################################################### -In particular, you need to call the function `register_caffe_python_extractor` and pass `name` as the second argument of the function. +**A:** This error appears if the class of implementation of ``Op`` for Python Caffe layer could not be used by Model Optimizer. Python layers should be handled differently comparing to ordinary Caffe layers. + +In particular, you need to call the function ``register_caffe_python_extractor`` and pass ``name`` as the second argument of the function. The name should be the compilation of the layer name with the module name separated by a dot. -For example, your topology contains this layer with type `Python`: - -``` -layer { - name: 'proposal' - type: 'Python' - ... - python_param { - module: 'rpn.proposal_layer' - layer: 'ProposalLayer' - param_str: "'feat_stride': 16" - } -} -``` - -The first step is to implement an extension for this layer in Model Optimizer as an ancestor of `Op` class: -``` -class ProposalPythonExampleOp(Op): - op = 'Proposal' - - def __init__(self, graph: nx.MultiDiGraph, attrs: dict): - ... -``` +For example, your topology contains this layer with type ``Python``: + +.. code-block:: + + layer { + name: 'proposal' + type: 'Python' + ... + python_param { + module: 'rpn.proposal_layer' + layer: 'ProposalLayer' + param_str: "'feat_stride': 16" + } + } + + +The first step is to implement an extension for this layer in Model Optimizer as an ancestor of ``Op`` class: + +.. code-block:: + + class ProposalPythonExampleOp(Op): + op = 'Proposal' + + def __init__(self, graph: nx.MultiDiGraph, attrs: dict): + ... + It is mandatory to call two functions right after the implementation of that class: -``` -class ProposalPythonExampleOp(Op): - ... -register_caffe_python_extractor(ProposalPythonExampleOp, 'rpn.proposal_layer.ProposalLayer') -Op.excluded_classes.append(ProposalPythonExampleOp) -``` +.. code-block:: + + class ProposalPythonExampleOp(Op): + ... -Note that the first call register_caffe_python_extractor(ProposalPythonExampleOp, 'rpn.proposal_layer.ProposalLayer') registers an extension of the layer in Model Optimizer, which will be found by the specific name (mandatory to join module name and layer name): rpn.proposal_layer.ProposalLayer. + register_caffe_python_extractor(ProposalPythonExampleOp, 'rpn.proposal_layer.ProposalLayer') + Op.excluded_classes.append(ProposalPythonExampleOp) + + +Note that the first call ``register_caffe_python_extractor(ProposalPythonExampleOp, 'rpn.proposal_layer.ProposalLayer')`` registers an extension of the layer in Model Optimizer, which will be found by the specific name (mandatory to join module name and layer name): ``rpn.proposal_layer.ProposalLayer``. The second call prevents Model Optimizer from using this extension as if it is an extension for -a layer with type `Proposal`. Otherwise, this layer can be chosen as an implementation of extension that can lead to potential issues. -For more information, refer to the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +a layer with type ``Proposal``. Otherwise, this layer can be chosen as an implementation of extension that can lead to potential issues. +For more information, refer to the :doc:`OpenVINO Extensibility Mechanism ` guide. + +.. _question-87: -#### Q87. What does the message "Model Optimizer is unable to calculate output shape of Memory node .." mean? +Q87. What does the message "Model Optimizer is unable to calculate output shape of Memory node .." mean? +##################################################################################################################################################### -**A** : Model Optimizer supports only `Memory` layers, in which `input_memory` goes before `ScaleShift` or the `FullyConnected` layer. -This error message means that in your model the layer after input memory is not of the `ScaleShift` or `FullyConnected` type. +**A:** Model Optimizer supports only ``Memory`` layers, in which ``input_memory`` goes before ``ScaleShift`` or the ``FullyConnected`` layer. +This error message means that in your model the layer after input memory is not of the ``ScaleShift`` or ``FullyConnected`` type. This is a known limitation. -#### Q88. What do the messages "File ... does not appear to be a Kaldi file (magic number does not match)", "Kaldi model should start with tag" mean? +.. _question-88: -**A** : These error messages mean that Model Optimizer does not support your Kaldi model, because the `checksum` of the model is not -16896 (the model should start with this number), or the model file does not contain the `` tag as a starting one. +Q88. What do the messages "File ... does not appear to be a Kaldi file (magic number does not match)", "Kaldi model should start with tag" mean? +######################################################################################################################################################### + +**A:** These error messages mean that Model Optimizer does not support your Kaldi model, because the ``checksum`` of the model is not +16896 (the model should start with this number), or the model file does not contain the ```` tag as a starting one. Make sure that you provide a path to a true Kaldi model and try again. -#### Q89. What do the messages "Expect counts file to be one-line file." or "Expect counts file to contain list of integers" mean? -**A** : These messages mean that the file counts you passed contain not one line. The count file should start with -`[` and end with `]`, and integer values should be separated by spaces between those brackets. +.. _question-89: + +Q89. What do the messages "Expect counts file to be one-line file." or "Expect counts file to contain list of integers" mean? +##################################################################################################################################################### + +**A:** These messages mean that the file counts you passed contain not one line. The count file should start with +``[`` and end with ``]``, and integer values should be separated by spaces between those brackets. -#### Q90. What does the message "Model Optimizer is not able to read Kaldi model .." mean? +.. _question-90: -**A** : There are multiple reasons why Model Optimizer does not accept a Kaldi topology, including: -the file is not available or does not exist. Refer to FAQ [#88](#question-88). +Q90. What does the message "Model Optimizer is not able to read Kaldi model .." mean? +##################################################################################################################################################### -#### Q91. What does the message "Model Optimizer is not able to read counts file .." mean? +**A:** There are multiple reasons why Model Optimizer does not accept a Kaldi topology, including: +the file is not available or does not exist. Refer to FAQ :ref:`#88 `. -**A** : There are multiple reasons why Model Optimizer does not accept a counts file, including: -the file is not available or does not exist. Refer to FAQ [#89](#question-89). +.. _question-91: -#### Q92. What does the message "For legacy MXNet models Model Optimizer does not support conversion of old MXNet models (trained with 1.0.0 version of MXNet and lower) with custom layers." mean? +Q91. What does the message "Model Optimizer is not able to read counts file .." mean? +##################################################################################################################################################### -**A** : This message means that if you have a model with custom layers and its JSON file has been generated with Apache MXNet version +**A:** There are multiple reasons why Model Optimizer does not accept a counts file, including: +the file is not available or does not exist. Refer to FAQ :ref:`#89 `. + +.. _question-92: + +Q92. What does the message "For legacy MXNet models Model Optimizer does not support conversion of old MXNet models (trained with 1.0.0 version of MXNet and lower) with custom layers." mean? +############################################################################################################################################################################################### + +**A:** This message means that if you have a model with custom layers and its JSON file has been generated with Apache MXNet version lower than 1.0.0, Model Optimizer does not support such topologies. If you want to convert it, you have to rebuild MXNet with unsupported layers or generate a new JSON file with Apache MXNet version 1.0.0 or higher. You also need to implement OpenVINO extension to use custom layers. -For more information, refer to the [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) guide. +For more information, refer to the :doc:`OpenVINO Extensibility Mechanism ` guide. -#### Q93. What does the message "Graph contains a cycle. Can not proceed .." mean? +.. _question-93: -**A** : Model Optimizer supports only straightforward models without cycles. +Q93. What does the message "Graph contains a cycle. Can not proceed .." mean? +##################################################################################################################################################### + +**A:** Model Optimizer supports only straightforward models without cycles. There are multiple ways to avoid cycles: For Tensorflow: -* [Convert models, created with TensorFlow Object Detection API](@ref openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models) + +* :doc:`Convert models, created with TensorFlow Object Detection API ` For all frameworks: -1. [Replace cycle containing Sub-graph in Model Optimizer](@ref openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer) -2. See [OpenVINO Extensibility Mechanism](@ref openvino_docs_Extensibility_UG_Intro) + +1. :doc:`Replace cycle containing Sub-graph in Model Optimizer ` +2. See :doc:`OpenVINO Extensibility Mechanism ` or * Edit the model in its original framework to exclude cycle. -#### Q94. What does the message "Can not transpose attribute '..' with value .. for node '..' .." mean? +.. _question-94: + +Q94. What does the message "Can not transpose attribute '..' with value .. for node '..' .." mean? +##################################################################################################################################################### -**A** : This message means that the model is not supported. It may be caused by using shapes larger than 4-D. +**A:** This message means that the model is not supported. It may be caused by using shapes larger than 4-D. There are two ways to avoid such message: -* [Cut off parts of the model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model). +* :doc:`Cut off parts of the model `. * Edit the network in its original framework to exclude such layers. -#### Q95. What does the message "Expected token ``, has `...`" mean? +.. _question-95: + +Q95. What does the message "Expected token ````, has ``...``" mean? +##################################################################################################################################################### -**A** : This error messages mean that Model Optimizer does not support your Kaldi model, because the Net contains `ParallelComponent` that does not end with the `` tag. +**A:** This error messages mean that Model Optimizer does not support your Kaldi model, because the Net contains ``ParallelComponent`` that does not end with the ```` tag. Make sure that you provide a path to a true Kaldi model and try again. -#### Q96. What does the message "Interp layer shape inference function may be wrong, please, try to update layer shape inference function in the file (extensions/ops/interp.op at the line ...)." mean? +.. _question-96: + +Q96. What does the message "Interp layer shape inference function may be wrong, please, try to update layer shape inference function in the file (extensions/ops/interp.op at the line ...)." mean? +#################################################################################################################################################################################################### + +**A:** There are many flavors of Caffe framework, and most layers in them are implemented identically. +However, there are exceptions. For example, the output value of layer Interp is calculated differently in Deeplab-Caffe and classic Caffe. Therefore, if your model contains layer Interp and the conversion of your model has failed, modify the ``interp_infer`` function in the ``extensions/ops/interp.op`` file according to the comments in the file. + +.. _question-97: -**A** : There are many flavors of Caffe framework, and most layers in them are implemented identically. -However, there are exceptions. For example, the output value of layer Interp is calculated differently in Deeplab-Caffe and classic Caffe. Therefore, if your model contains layer Interp and the conversion of your model has failed, modify the `interp_infer` function in the `extensions/ops/interp.op` file according to the comments in the file. +Q97. What does the message "Mean/scale values should ..." mean? +##################################################################################################################################################### -#### Q97. What does the message "Mean/scale values should ..." mean? +**A:** It means that your mean/scale values have a wrong format. Specify mean/scale values in the form of ``layer_name(val1,val2,val3)``. +You need to specify values for each input of the model. For more information, refer to the :doc:`Converting a Model to Intermediate Representation ` guide. -**A** : It means that your mean/scale values have a wrong format. Specify mean/scale values in the form of `layer_name(val1,val2,val3)`. -You need to specify values for each input of the model. For more information, refer to the [Converting a Model to Intermediate Representation](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. +.. _question-98: -#### Q98. What does the message "Operation _contrib_box_nms is not supported ..." mean? +Q98. What does the message "Operation _contrib_box_nms is not supported ..." mean? +##################################################################################################################################################### -**A** : It means that you are trying to convert a topology contains the `_contrib_box_nms` operation which is not supported directly. However, the sub-graph of operations including `_contrib_box_nms` could be replaced with the DetectionOutput layer if your topology is one of the `gluoncv` topologies. Specify the `--enable_ssd_gluoncv` command-line parameter for Model Optimizer to enable this transformation. +**A:** It means that you are trying to convert a topology contains the ``_contrib_box_nms`` operation which is not supported directly. However, the sub-graph of operations including ``_contrib_box_nms`` could be replaced with the DetectionOutput layer if your topology is one of the ``gluoncv`` topologies. Specify the ``--enable_ssd_gluoncv`` command-line parameter for Model Optimizer to enable this transformation. -#### Q99. What does the message "ModelOptimizer is not able to parse *.caffemodel" mean? +.. _question-99: -**A** : If a `*.caffemodel` file exists and is correct, the error occurred possibly because of the use of Python protobuf implementation. In some cases, error messages may appear during model parsing, for example: "`utf-8` codec can't decode byte 0xe0 in position 4: invalid continuation byte in field: mo_caffe.SpatialTransformerParameter.transform_type". You can either use Python 3.7 or build the `cpp` implementation of `protobuf` yourself for your version of Python. For the complete instructions about building `protobuf` from sources, see the appropriate section in the [Converting Models with Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) guide. +Q99. What does the message "ModelOptimizer is not able to parse *.caffemodel" mean? +##################################################################################################################################################### -#### Q100. What does the message "SyntaxError: 'yield' inside list comprehension" during MxNet model conversion mean? +**A:** If a ``*.caffemodel`` file exists and is correct, the error occurred possibly because of the use of Python protobuf implementation. In some cases, error messages may appear during model parsing, for example: "``utf-8`` codec can't decode byte 0xe0 in position 4: invalid continuation byte in field: mo_caffe.SpatialTransformerParameter.transform_type". You can either use Python 3.7 or build the ``cpp`` implementation of ``protobuf`` yourself for your version of Python. For the complete instructions about building ``protobuf`` from sources, see the appropriate section in the :doc:`Converting Models with Model Optimizer ` guide. -**A** : The issue "SyntaxError: `yield` inside list comprehension" might occur during converting MXNet models (`mobilefacedet-v1-mxnet`, `brain-tumor-segmentation-0001`) on Windows platform with Python 3.8 environment. This issue is caused by the API changes for `yield expression` in Python 3.8. + +.. _question-100: + +Q100. What does the message "SyntaxError: 'yield' inside list comprehension" during MxNet model conversion mean? +##################################################################################################################################################### + +**A:** The issue "SyntaxError: ``yield`` inside list comprehension" might occur during converting MXNet models (``mobilefacedet-v1-mxnet``, ``brain-tumor-segmentation-0001``) on Windows platform with Python 3.8 environment. This issue is caused by the API changes for ``yield expression`` in Python 3.8. The following workarounds are suggested to resolve this issue: 1. Use Python 3.7 to convert MXNet models on Windows -2. Update Apache MXNet by using `pip install mxnet==1.7.0.post2` +2. Update Apache MXNet by using ``pip install mxnet==1.7.0.post2`` Note that it might have conflicts with previously installed PyPI dependencies. -#### Q101. What does the message "The IR preparation was executed by the legacy MO path. ..." mean? +.. _question-101: -**A** : For the models in ONNX format, there are two available paths of IR conversion. +Q101. What does the message "The IR preparation was executed by the legacy MO path. ..." mean? +##################################################################################################################################################### + +**A:** For the models in ONNX format, there are two available paths of IR conversion. The old one is handled by the old Python implementation, while the new one uses new C++ frontends. Starting from the 2022.1 version, the default IR conversion path for ONNX models is processed using the new ONNX frontend. -Certain features, such as `--extensions` and `--transformations_config`, are not yet fully supported on the new frontends. -The new frontends support only paths to shared libraries (.dll and .so) for `--extensions`. They support JSON configurations with defined library fields for `--transformations_config`. -Inputs freezing (enabled by `--freeze_placeholder_with_value` or `--input` arguments) is not supported by the new frontends. -The IR conversion falls back to the old path if a user does not select any expected path of conversion explicitly (with `--use_new_frontend` or `--use_legacy_frontend` MO arguments) and unsupported pre-defined scenario is detected on the new frontend path. +Certain features, such as ``--extensions`` and ``--transformations_config``, are not yet fully supported on the new frontends. +The new frontends support only paths to shared libraries (.dll and .so) for ``--extensions``. They support JSON configurations with defined library fields for ``--transformations_config``. +Inputs freezing (enabled by ``--freeze_placeholder_with_value`` or ``--input`` arguments) is not supported by the new frontends. +The IR conversion falls back to the old path if a user does not select any expected path of conversion explicitly (with ``--use_new_frontend`` or ``--use_legacy_frontend`` MO arguments) and unsupported pre-defined scenario is detected on the new frontend path. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md index ab8a067e3e1a33..2b9f17e35be38e 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md @@ -1,85 +1,99 @@ # Converting a Caffe Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe} -To convert a Caffe model, run Model Optimizer with the path to the input model `.caffemodel` file: +@sphinxdirective + +To convert a Caffe model, run Model Optimizer with the path to the input model ``.caffemodel`` file: + +.. code-block:: cpp + + mo --input_model .caffemodel -```sh - mo --input_model .caffemodel -``` The following list provides the Caffe-specific parameters. -``` -Caffe-specific parameters: - --input_proto INPUT_PROTO, -d INPUT_PROTO - Deploy-ready prototxt file that contains a topology - structure and layer attributes - --caffe_parser_path CAFFE_PARSER_PATH - Path to python Caffe parser generated from caffe.proto - -k K Path to CustomLayersMapping.xml to register custom - layers - --disable_omitting_optional - Disable omitting optional attributes to be used for - custom layers. Use this option if you want to transfer - all attributes of a custom layer to IR. Default - behavior is to transfer the attributes with default - values and the attributes defined by the user to IR. - --enable_flattening_nested_params - Enable flattening optional params to be used for - custom layers. Use this option if you want to transfer - attributes of a custom layer to IR with flattened - nested parameters. Default behavior is to transfer the - attributes without flattening nested parameters. -``` - -### CLI Examples Using Caffe-Specific Parameters - -* Launching Model Optimizer for [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `prototxt` file. - This is needed when the name of the Caffe model and the `.prototxt` file are different or are placed in different directories. Otherwise, it is enough to provide only the path to the input `model.caffemodel` file. - ```sh - mo --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt - ``` -* Launching Model Optimizer for [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `CustomLayersMapping` file. - This is the legacy method of quickly enabling model conversion if your model has custom layers. This requires the Caffe system on the computer. -Example of `CustomLayersMapping.xml` can be found in `/mo/front/caffe/CustomLayersMapping.xml.example`. The optional parameters without default values and not specified by the user in the `.prototxt` file are removed from the Intermediate Representation, and nested parameters are flattened: - ```sh - mo --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params - ``` - This example shows a multi-input model with input layers: `data`, `rois` - ``` - layer { - name: "data" - type: "Input" - top: "data" - input_param { - shape { dim: 1 dim: 3 dim: 224 dim: 224 } - } - } - layer { - name: "rois" - type: "Input" - top: "rois" - input_param { - shape { dim: 1 dim: 5 dim: 1 dim: 1 } - } - } - ``` - -* Launching the Model Optimizer for a multi-input model with two inputs and providing a new shape for each input in the order they are passed to the Model Optimizer. In particular, for data, set the shape to `1,3,227,227`. For rois, set the shape to `1,6,1,1`: - ```sh - mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1] - ``` -## Custom Layer Definition +.. code-block:: cpp + + Caffe-specific parameters: + --input_proto INPUT_PROTO, -d INPUT_PROTO + Deploy-ready prototxt file that contains a topology + structure and layer attributes + --caffe_parser_path CAFFE_PARSER_PATH + Path to python Caffe parser generated from caffe.proto + -k K Path to CustomLayersMapping.xml to register custom + layers + --disable_omitting_optional + Disable omitting optional attributes to be used for + custom layers. Use this option if you want to transfer + all attributes of a custom layer to IR. Default + behavior is to transfer the attributes with default + values and the attributes defined by the user to IR. + --enable_flattening_nested_params + Enable flattening optional params to be used for + custom layers. Use this option if you want to transfer + attributes of a custom layer to IR with flattened + nested parameters. Default behavior is to transfer the + attributes without flattening nested parameters. + + +CLI Examples Using Caffe-Specific Parameters +++++++++++++++++++++++++++++++++++++++++++++ + +* Launching Model Optimizer for `bvlc_alexnet.caffemodel `__ with a specified `prototxt` file. This is needed when the name of the Caffe model and the `.prototxt` file are different or are placed in different directories. Otherwise, it is enough to provide only the path to the input `model.caffemodel` file. + + .. code-block:: cpp + + mo --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt + +* Launching Model Optimizer for `bvlc_alexnet.caffemodel `__ with a specified `CustomLayersMapping` file. This is the legacy method of quickly enabling model conversion if your model has custom layers. This requires the Caffe system on the computer. Example of ``CustomLayersMapping.xml`` can be found in ``/mo/front/caffe/CustomLayersMapping.xml.example``. The optional parameters without default values and not specified by the user in the ``.prototxt`` file are removed from the Intermediate Representation, and nested parameters are flattened: + + .. code-block:: cpp + + mo --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params + + This example shows a multi-input model with input layers: ``data``, ``rois`` + + .. code-block:: cpp + + layer { + name: "data" + type: "Input" + top: "data" + input_param { + shape { dim: 1 dim: 3 dim: 224 dim: 224 } + } + } + layer { + name: "rois" + type: "Input" + top: "rois" + input_param { + shape { dim: 1 dim: 5 dim: 1 dim: 1 } + } + } + +* Launching the Model Optimizer for a multi-input model with two inputs and providing a new shape for each input in the order they are passed to the Model Optimizer. In particular, for data, set the shape to ``1,3,227,227``. For rois, set the shape to ``1,6,1,1``: + + .. code-block:: cpp + + mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1] + +Custom Layer Definition +######################## Internally, when you run Model Optimizer, it loads the model, goes through the topology, and tries to find each layer type in a list of known layers. Custom layers are layers that are not included in the list. If your topology contains such kind of layers, Model Optimizer classifies them as custom. -## Supported Caffe Layers -For the list of supported standard layers, refer to the [Supported Framework Layers](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) page. +Supported Caffe Layers +####################### -## Frequently Asked Questions (FAQ) +For the list of supported standard layers, refer to the :doc:`Supported Framework Layers ` page. -Model Optimizer provides explanatory messages when it is unable to complete conversions due to typographical errors, incorrectly used options, or other issues. A message describes the potential cause of the problem and gives a link to [Model Optimizer FAQ](@ref openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ) which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections to help you understand what went wrong. +Frequently Asked Questions (FAQ) +################################ -## Summary +Model Optimizer provides explanatory messages when it is unable to complete conversions due to typographical errors, incorrectly used options, or other issues. A message describes the potential cause of the problem and gives a link to :doc:`Model Optimizer FAQ ` which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections to help you understand what went wrong. + +Summary +####### In this document, you learned: @@ -87,5 +101,10 @@ In this document, you learned: * Which Caffe models are supported. * How to convert a trained Caffe model by using Model Optimizer with both framework-agnostic and Caffe-specific command-line options. -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific Caffe models. +Additional Resources +#################### + +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific Caffe models. + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md index f5687f60e732b8..7f93a960fed969 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md @@ -1,65 +1,86 @@ # Converting a Kaldi Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi} -> **NOTE**: Model Optimizer supports the [nnet1](http://kaldi-asr.org/doc/dnn1.html) and [nnet2](http://kaldi-asr.org/doc/dnn2.html) formats of Kaldi models. The support of the [nnet3](http://kaldi-asr.org/doc/dnn3.html) format is limited. +@sphinxdirective + +.. note:: + + Model Optimizer supports the `nnet1 `__ and `nnet2 `__ formats of Kaldi models. The support of the `nnet3 `__ format is limited. -To convert a Kaldi model, run Model Optimizer with the path to the input model `.nnet` or `.mdl` file: +To convert a Kaldi model, run Model Optimizer with the path to the input model ``.nnet`` or ``.mdl`` file: -```sh - mo --input_model .nnet -``` +.. code-block:: cpp -## Using Kaldi-Specific Conversion Parameters + mo --input_model .nnet + +Using Kaldi-Specific Conversion Parameters +########################################## The following list provides the Kaldi-specific parameters. -```sh -Kaldi-specific parameters: - --counts COUNTS A file name with full path to the counts file or empty string to utilize count values from the model file - --remove_output_softmax - Removes the Softmax that is the output layer - --remove_memory Remove the Memory layer and add new inputs and outputs instead -``` +.. code-block:: cpp + + Kaldi-specific parameters: + --counts COUNTS A file name with full path to the counts file or empty string to utilize count values from the model file + --remove_output_softmax + Removes the Softmax that is the output layer + --remove_memory Remove the Memory layer and add new inputs and outputs instead -## Examples of CLI Commands +Examples of CLI Commands +######################## -* To launch Model Optimizer for the `wsj_dnn5b_smbr` model with the specified `.nnet` file: - ```sh - mo --input_model wsj_dnn5b_smbr.nnet - ``` +* To launch Model Optimizer for the ``wsj_dnn5b_smbr`` model with the specified ``.nnet`` file: + + .. code-block:: cpp -* To launch Model Optimizer for the `wsj_dnn5b_smbr` model with the existing file that contains counts for the last layer with biases: - ```sh - mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts - ``` + mo --input_model wsj_dnn5b_smbr.nnet + +* To launch Model Optimizer for the ``wsj_dnn5b_smbr`` model with the existing file that contains counts for the last layer with biases: + + .. code-block:: cpp + + mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts + * The Model Optimizer normalizes сounts in the following way: - \f[ - S = \frac{1}{\sum_{j = 0}^{|C|}C_{j}} - \f] - \f[ - C_{i}=log(S*C_{i}) - \f] - where \f$C\f$ - the counts array, \f$C_{i} - i^{th}\f$ element of the counts array, - \f$|C|\f$ - number of elements in the counts array; + + .. math:: + + S = \frac{1}{\sum_{j = 0}^{|C|}C_{j}} + + .. math:: + + C_{i}=log(S\*C_{i}) + + where :math:`C` - the counts array, :math:`C_{i} - i^{th}` element of the counts array, :math:`|C|` - number of elements in the counts array; + * The normalized counts are subtracted from biases of the last or next to last layer (if last layer is SoftMax). + + .. note:: Model Optimizer will show a warning if a model contains values of counts and the `--counts` option is not used. - > **NOTE**: Model Optimizer will show a warning if a model contains values of counts and the `--counts` option is not used. +* If you want to remove the last SoftMax layer in the topology, launch the Model Optimizer with the `--remove_output_softmax` flag: + +.. code-block:: cpp -* If you want to remove the last SoftMax layer in the topology, launch the Model Optimizer with the -`--remove_output_softmax` flag: - ```sh mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --remove_output_softmax - ``` - The Model Optimizer finds the last layer of the topology and removes this layer only if it is a SoftMax layer. +The Model Optimizer finds the last layer of the topology and removes this layer only if it is a SoftMax layer. + +.. note:: Model Optimizer can remove SoftMax layer only if the topology has one output. + +* You can use the *OpenVINO Speech Recognition* sample application for the sample inference of Kaldi models. This sample supports models with only one output. If your model has several outputs, specify the desired one with the ``--output`` option. + +Supported Kaldi Layers +###################### + +For the list of supported standard layers, refer to the :doc:`Supported Framework Layers ` page. + +Additional Resources +#################### + +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific Kaldi models. Here are some examples: - > **NOTE**: Model Optimizer can remove SoftMax layer only if the topology has one output. +* :doc:`Convert Kaldi ASpIRE Chain Time Delay Neural Network (TDNN) Model ` -* You can use the *OpenVINO Speech Recognition* sample application for the sample inference of Kaldi models. This sample supports models with only one output. If your model has several outputs, specify the desired one with the `--output` option. -## Supported Kaldi Layers -For the list of supported standard layers, refer to the [Supported Framework Layers ](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) page. +@endsphinxdirective -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific Kaldi models. Here are some examples: -* [Convert Kaldi ASpIRE Chain Time Delay Neural Network (TDNN) Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model) diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md index 492d5cff8e29f1..43c3888d06074d 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md @@ -1,51 +1,61 @@ # Converting an MXNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet} -To convert an MXNet model, run Model Optimizer with the path to the *`.params`* file of the input model: +@sphinxdirective -```sh - mo --input_model model-file-0000.params -``` +To convert an MXNet model, run Model Optimizer with the path to the ``.params`` file of the input model: + +.. code-block:: sh + + mo --input_model model-file-0000.params + + +Using MXNet-Specific Conversion Parameters +########################################## -## Using MXNet-Specific Conversion Parameters The following list provides the MXNet-specific parameters. -``` -MXNet-specific parameters: - --input_symbol - Symbol file (for example, "model-symbol.json") that contains a topology structure and layer attributes - --nd_prefix_name - Prefix name for args.nd and argx.nd files - --pretrained_model_name - Name of a pre-trained MXNet model without extension and epoch - number. This model will be merged with args.nd and argx.nd - files - --save_params_from_nd - Enable saving built parameters file from .nd files - --legacy_mxnet_model - Enable Apache MXNet loader to make a model compatible with the latest Apache MXNet version. - Use only if your model was trained with Apache MXNet version lower than 1.0.0 - --enable_ssd_gluoncv - Enable transformation for converting the gluoncv ssd topologies. - Use only if your topology is one of ssd gluoncv topologies -``` - -> **NOTE**: By default, Model Optimizer does not use the Apache MXNet loader. It transforms the topology to another format which is compatible with the latest -> version of Apache MXNet. However, the Apache MXNet loader is required for models trained with lower version of Apache MXNet. If your model was trained with an Apache MXNet version lower than 1.0.0, specify the -> `--legacy_mxnet_model` key to enable the Apache MXNet loader. Note that the loader does not support models with custom layers. In this case, you must manually -> recompile Apache MXNet with custom layers and install it in your environment. - -## Custom Layer Definition +.. code-block:: sh + + MXNet-specific parameters: + --input_symbol + Symbol file (for example, "model-symbol.json") that contains a topology structure and layer attributes + --nd_prefix_name + Prefix name for args.nd and argx.nd files + --pretrained_model_name + Name of a pre-trained MXNet model without extension and epoch + number. This model will be merged with args.nd and argx.nd + files + --save_params_from_nd + Enable saving built parameters file from .nd files + --legacy_mxnet_model + Enable Apache MXNet loader to make a model compatible with the latest Apache MXNet version. + Use only if your model was trained with Apache MXNet version lower than 1.0.0 + --enable_ssd_gluoncv + Enable transformation for converting the gluoncv ssd topologies. + Use only if your topology is one of ssd gluoncv topologies + + +.. note:: + + By default, Model Optimizer does not use the Apache MXNet loader. It transforms the topology to another format which is compatible with the latest version of Apache MXNet. However, the Apache MXNet loader is required for models trained with lower version of Apache MXNet. If your model was trained with an Apache MXNet version lower than 1.0.0, specify the ``--legacy_mxnet_model`` key to enable the Apache MXNet loader. Note that the loader does not support models with custom layers. In this case, you must manually recompile Apache MXNet with custom layers and install it in your environment. + +Custom Layer Definition +####################### Internally, when you run Model Optimizer, it loads the model, goes through the topology, and tries to find each layer type in a list of known layers. Custom layers are layers that are not included in the list. If your topology contains such kind of layers, Model Optimizer classifies them as custom. -## Supported MXNet Layers -For the list of supported standard layers, refer to the [Supported Framework Layers](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) page. +Supported MXNet Layers +####################### -## Frequently Asked Questions (FAQ) +For the list of supported standard layers, refer to the :doc:`Supported Framework Layers ` page. -Model Optimizer provides explanatory messages when it is unable to complete conversions due to typographical errors, incorrectly used options, or other issues. A message describes the potential cause of the problem and gives a link to [Model Optimizer FAQ](@ref openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ) which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections to help you understand what went wrong. +Frequently Asked Questions (FAQ) +################################ -## Summary +Model Optimizer provides explanatory messages when it is unable to complete conversions due to typographical errors, incorrectly used options, or other issues. A message describes the potential cause of the problem and gives a link to :doc:`Model Optimizer FAQ ` which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections to help you understand what went wrong. + +Summary +######## In this document, you learned: @@ -53,7 +63,13 @@ In this document, you learned: * Which MXNet models are supported. * How to convert a trained MXNet model by using the Model Optimizer with both framework-agnostic and MXNet-specific command-line options. -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific MXNet models. Here are some examples: -* [Convert MXNet GluonCV Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models) -* [Convert MXNet Style Transfer Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet) +Additional Resources +#################### + +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific MXNet models. Here are some examples: + +* :doc:`Convert MXNet GluonCV Model ` +* :doc:`Convert MXNet Style Transfer Model ` + +@endsphinxdirective + diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md index 5b4c5b6b3396f7..21ebc3b6547f27 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md @@ -1,28 +1,40 @@ # Converting an ONNX Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX} -## Introduction to ONNX -[ONNX](https://github.com/onnx/onnx) is a representation format for deep learning models that allows AI developers to easily transfer models between different frameworks. It is hugely popular among deep learning tools, like PyTorch, Caffe2, Apache MXNet, Microsoft Cognitive Toolkit, and many others. +@sphinxdirective -## Converting an ONNX Model +Introduction to ONNX +#################### -This page provides instructions on how to convert a model from the ONNX format to the OpenVINO IR format using Model Optimizer. To use Model Optimizer, install OpenVINO Development Tools by following the [installation instructions](@ref openvino_docs_install_guides_install_dev_tools). +`ONNX `__ is a representation format for deep learning models that allows AI developers to easily transfer models between different frameworks. It is hugely popular among deep learning tools, like PyTorch, Caffe2, Apache MXNet, Microsoft Cognitive Toolkit, and many others. + +Converting an ONNX Model +######################## + +This page provides instructions on how to convert a model from the ONNX format to the OpenVINO IR format using Model Optimizer. To use Model Optimizer, install OpenVINO Development Tools by following the :doc:`installation instructions `. The Model Optimizer process assumes you have an ONNX model that was directly downloaded from a public repository or converted from any framework that supports exporting to the ONNX format. -To convert an ONNX model, run Model Optimizer with the path to the input model `.onnx` file: +To convert an ONNX model, run Model Optimizer with the path to the input model ``.onnx`` file: + +.. code-block:: sh + + mo --input_model .onnx + +There are no ONNX specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see the *General Conversion Parameters* section in the :doc:`Converting a Model to Intermediate Representation (IR) ` guide. + +Supported ONNX Layers +##################### + +For the list of supported standard layers, refer to the :doc:`Supported Framework Layers ` page. -```sh - mo --input_model .onnx -``` +Additional Resources +#################### -There are no ONNX specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see the *General Conversion Parameters* section in the [Converting a Model to Intermediate Representation (IR)](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific ONNX models. Here are some examples: -## Supported ONNX Layers -For the list of supported standard layers, refer to the [Supported Framework Layers](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) page. +* :doc:`Convert ONNX Faster R-CNN Model ` +* :doc:`Convert ONNX GPT-2 Model ` +* :doc:`Convert ONNX Mask R-CNN Model ` -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific ONNX models. Here are some examples: -* [Convert ONNX Faster R-CNN Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN) -* [Convert ONNX GPT-2 Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2) -* [Convert ONNX Mask R-CNN Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN) +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md index 1388193121a58f..1b6aa7fa824596 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md @@ -1,23 +1,29 @@ # Converting a PaddlePaddle Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle} -To convert a PaddlePaddle model, use the `mo` script and specify the path to the input `.pdmodel` model file: +@sphinxdirective + +To convert a PaddlePaddle model, use the ``mo`` script and specify the path to the input ``.pdmodel`` model file: + +.. code-block:: sh + + mo --input_model .pdmodel -```sh - mo --input_model .pdmodel -``` **For example,** this command converts a yolo v3 PaddlePaddle network to OpenVINO IR network: -```sh - mo --input_model=yolov3.pdmodel --input=image,im_shape,scale_factor --input_shape=[1,3,608,608],[1,2],[1,2] --reverse_input_channels --output=save_infer_model/scale_0.tmp_1,save_infer_model/scale_1.tmp_1 -``` +.. code-block:: sh + + mo --input_model=yolov3.pdmodel --input=image,im_shape,scale_factor --input_shape=[1,3,608,608],[1,2],[1,2] --reverse_input_channels --output=save_infer_model/scale_0.tmp_1,save_infer_model/scale_1.tmp_1 + +Supported PaddlePaddle Layers +############################# + +For the list of supported standard layers, refer to the :doc:`Supported Framework Layers ` page. -## Supported PaddlePaddle Layers -For the list of supported standard layers, refer to the [Supported Framework Layers](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) page. +Officially Supported PaddlePaddle Models +######################################## -## Officially Supported PaddlePaddle Models The following PaddlePaddle models have been officially validated and confirmed to work (as of OpenVINO 2022.1): -@sphinxdirective .. list-table:: :widths: 20 25 55 :header-rows: 1 @@ -67,10 +73,16 @@ The following PaddlePaddle models have been officially validated and confirmed t * - BERT - language representation - Models are exported from `PaddleNLP `_. Refer to `README.md `_. -@endsphinxdirective -## Frequently Asked Questions (FAQ) -When Model Optimizer is unable to run to completion due to typographical errors, incorrectly used options, or other issues, it provides explanatory messages. They describe the potential cause of the problem and give a link to the [Model Optimizer FAQ](@ref openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ), which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in the Model Optimizer Developer Guide to help you understand what went wrong. +Frequently Asked Questions (FAQ) +################################ + +When Model Optimizer is unable to run to completion due to typographical errors, incorrectly used options, or other issues, it provides explanatory messages. They describe the potential cause of the problem and give a link to the :doc:`Model Optimizer FAQ `, which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in the Model Optimizer Developer Guide to help you understand what went wrong. + +Additional Resources +#################### + +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific PaddlePaddle models. + +@endsphinxdirective -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific PaddlePaddle models. diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md index c49ef7f2e82a86..77d31bd1514bd1 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md @@ -1,39 +1,49 @@ # Converting a PyTorch Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch} +@sphinxdirective + The PyTorch framework is supported through export to the ONNX format. In order to optimize and deploy a model that was trained with it: -1. [Export a PyTorch model to ONNX](#export-to-onnx). -2. [Convert the ONNX model](Convert_Model_From_ONNX.md) to produce an optimized [Intermediate Representation](@ref openvino_docs_MO_DG_IR_and_opsets) of the model based on the trained network topology, weights, and biases values. +1. `Export a PyTorch model to ONNX <#exporting-a-pytorch-model-to-onnx-format>`__. +2. :doc:`Convert the ONNX model ` to produce an optimized :doc:`Intermediate Representation ` of the model based on the trained network topology, weights, and biases values. + +Exporting a PyTorch Model to ONNX Format +######################################## -## Exporting a PyTorch Model to ONNX Format -PyTorch models are defined in Python. To export them, use the `torch.onnx.export()` method. The code to +PyTorch models are defined in Python. To export them, use the ``torch.onnx.export()`` method. The code to evaluate or test the model is usually provided with its code and can be used for its initialization and export. The export to ONNX is crucial for this process, but it is covered by PyTorch framework, therefore, It will not be covered here in detail. -For more information, refer to the [Exporting PyTorch models to ONNX format](https://pytorch.org/docs/stable/onnx.html) guide. - -To export a PyTorch model, you need to obtain the model as an instance of `torch.nn.Module` class and call the `export` function. - -```python -import torch - -# Instantiate your model. This is just a regular PyTorch model that will be exported in the following steps. -model = SomeModel() -# Evaluate the model to switch some operations from training mode to inference. -model.eval() -# Create dummy input for the model. It will be used to run the model inside export function. -dummy_input = torch.randn(1, 3, 224, 224) -# Call the export function -torch.onnx.export(model, (dummy_input, ), 'model.onnx') -``` - -## Known Issues - -* As of version 1.8.1, not all PyTorch operations can be exported to ONNX opset 9 which is used by default. -It is recommended to export models to opset 11 or higher when export to default opset 9 is not working. In that case, use `opset_version` -option of the `torch.onnx.export`. For more information about ONNX opset, refer to the [Operator Schemas](https://github.com/onnx/onnx/blob/master/docs/Operators.md) page. - -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific PyTorch models. Here are some examples: -* [Convert PyTorch BERT-NER Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Bert_ner) -* [Convert PyTorch RCAN Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RCAN) -* [Convert PyTorch YOLACT Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT) +For more information, refer to the `Exporting PyTorch models to ONNX format `__ guide. + +To export a PyTorch model, you need to obtain the model as an instance of ``torch.nn.Module`` class and call the ``export`` function. + +.. code-block:: py + + import torch + + # Instantiate your model. This is just a regular PyTorch model that will be exported in the following steps. + model = SomeModel() + # Evaluate the model to switch some operations from training mode to inference. + model.eval() + # Create dummy input for the model. It will be used to run the model inside export function. + dummy_input = torch.randn(1, 3, 224, 224) + # Call the export function + torch.onnx.export(model, (dummy_input, ), 'model.onnx') + + +Known Issues +#################### + +As of version 1.8.1, not all PyTorch operations can be exported to ONNX opset 9 which is used by default. +It is recommended to export models to opset 11 or higher when export to default opset 9 is not working. In that case, use ``opset_version`` option of the ``torch.onnx.export``. For more information about ONNX opset, refer to the `Operator Schemas `__ page. + +Additional Resources +#################### + +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific PyTorch models. Here are some examples: + +* :doc:`Convert PyTorch BERT-NER Model ` +* :doc:`Convert PyTorch RCAN Model ` +* :doc:`Convert PyTorch YOLACT Model ` + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md index fd177ec290c2cb..18c2a42c960620 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md @@ -1,151 +1,186 @@ # Converting a TensorFlow Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow} +@sphinxdirective + This page provides general instructions on how to convert a model from a TensorFlow format to the OpenVINO IR format using Model Optimizer. The instructions are different depending on whether your model was created with TensorFlow v1.X or TensorFlow v2.X. -To use Model Optimizer, install OpenVINO Development Tools by following the [installation instructions](@ref openvino_docs_install_guides_install_dev_tools). +To use Model Optimizer, install OpenVINO Development Tools by following the :doc:`installation instructions `. + +Converting TensorFlow 1 Models +############################### + +Converting Frozen Model Format ++++++++++++++++++++++++++++++++ -## Converting TensorFlow 1 Models +To convert a TensorFlow model, use the ``*mo*`` script to simply convert a model with a path to the input model ``*.pb*`` file: -### Converting Frozen Model Format -To convert a TensorFlow model, use the *`mo`* script to simply convert a model with a path to the input model *`.pb`* file: +.. code-block:: cpp -```sh - mo --input_model .pb -``` + mo --input_model .pb + + +Converting Non-Frozen Model Formats ++++++++++++++++++++++++++++++++++++ -### Converting Non-Frozen Model Formats There are three ways to store non-frozen TensorFlow models and convert them by Model Optimizer: -1. **Checkpoint**. In this case, a model consists of two files: `inference_graph.pb` (or `inference_graph.pbtxt`) and `checkpoint_file.ckpt`. -If you do not have an inference graph file, refer to the [Freezing Custom Models in Python](#freeze-the-tensorflow-model) section. -To convert the model with the inference graph in `.pb` format, run the `mo` script with a path to the checkpoint file: -```sh - mo --input_model .pb --input_checkpoint -``` -To convert the model with the inference graph in `.pbtxt` format, run the `mo` script with a path to the checkpoint file: -```sh - mo --input_model .pbtxt --input_checkpoint --input_model_is_text -``` - -2. **MetaGraph**. In this case, a model consists of three or four files stored in the same directory: `model_name.meta`, `model_name.index`, -`model_name.data-00000-of-00001` (the numbers may vary), and `checkpoint` (optional). -To convert such TensorFlow model, run the `mo` script with a path to the MetaGraph `.meta` file: -```sh - mo --input_meta_graph .meta -``` - -3. **SavedModel format**. In this case, a model consists of a special directory with a `.pb` file -and several subfolders: `variables`, `assets`, and `assets.extra`. For more information about the SavedModel directory, refer to the [README](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/saved_model#components) file in the TensorFlow repository. -To convert such TensorFlow model, run the `mo` script with a path to the SavedModel directory: -```sh - mo --saved_model_dir -``` +1. **Checkpoint**. In this case, a model consists of two files: ``inference_graph.pb`` (or ``inference_graph.pbtxt``) and ``checkpoint_file.ckpt``. +If you do not have an inference graph file, refer to the `Freezing Custom Models in Python <#Freezing-Custom-Models-in-Python>`__ section. +To convert the model with the inference graph in ``.pb`` format, run the `mo` script with a path to the checkpoint file: + +.. code-block:: cpp + + mo --input_model .pb --input_checkpoint + +To convert the model with the inference graph in ``.pbtxt`` format, run the ``mo`` script with a path to the checkpoint file: + +.. code-block:: cpp + + mo --input_model .pbtxt --input_checkpoint --input_model_is_text + + +2. **MetaGraph**. In this case, a model consists of three or four files stored in the same directory: ``model_name.meta``, ``model_name.index``, +``model_name.data-00000-of-00001`` (the numbers may vary), and ``checkpoint`` (optional). +To convert such TensorFlow model, run the `mo` script with a path to the MetaGraph ``.meta`` file: + +.. code-block:: cpp + + mo --input_meta_graph .meta + + +3. **SavedModel format**. In this case, a model consists of a special directory with a ``.pb`` file +and several subfolders: ``variables``, ``assets``, and ``assets.extra``. For more information about the SavedModel directory, refer to the `README `__ file in the TensorFlow repository. +To convert such TensorFlow model, run the ``mo`` script with a path to the SavedModel directory: + +.. code-block:: cpp + + mo --saved_model_dir + You can convert TensorFlow 1.x SavedModel format in the environment that has a 1.x or 2.x version of TensorFlow. However, TensorFlow 2.x SavedModel format strictly requires the 2.x version of TensorFlow. -If a model contains operations currently unsupported by OpenVINO, prune these operations by explicit specification of input nodes using the `--input` option. -To determine custom input nodes, display a graph of the model in TensorBoard. To generate TensorBoard logs of the graph, use the `--tensorboard_logs` option. -TensorFlow 2.x SavedModel format has a specific graph due to eager execution. In case of pruning, find custom input nodes in the `StatefulPartitionedCall/*` subgraph of TensorFlow 2.x SavedModel format. +If a model contains operations currently unsupported by OpenVINO, prune these operations by explicit specification of input nodes using the ``--input`` option. +To determine custom input nodes, display a graph of the model in TensorBoard. To generate TensorBoard logs of the graph, use the ``--tensorboard_logs`` option. +TensorFlow 2.x SavedModel format has a specific graph due to eager execution. In case of pruning, find custom input nodes in the ``StatefulPartitionedCall/*`` subgraph of TensorFlow 2.x SavedModel format. + +Freezing Custom Models in Python +++++++++++++++++++++++++++++++++ -### Freezing Custom Models in Python When a network is defined in Python code, you have to create an inference graph file. Graphs are usually built in a form that allows model training. That means all trainable parameters are represented as variables in the graph. To be able to use such graph with Model Optimizer, it should be frozen and dumped to a file with the following code: -```python -import tensorflow as tf -from tensorflow.python.framework import graph_io -frozen = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph_def, ["name_of_the_output_node"]) -graph_io.write_graph(frozen, './', 'inference_graph.pb', as_text=False) -``` +.. code-block:: python + + import tensorflow as tf + from tensorflow.python.framework import graph_io + frozen = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph_def, ["name_of_the_output_node"]) + graph_io.write_graph(frozen, './', 'inference_graph.pb', as_text=False) Where: -* `sess` is the instance of the TensorFlow Session object where the network topology is defined. -* `["name_of_the_output_node"]` is the list of output node names in the graph; `frozen` graph will - include only those nodes from the original `sess.graph_def` that are directly or indirectly used - to compute given output nodes. The `'name_of_the_output_node'` is an example of a possible output - node name. You should derive the names based on your own graph. -* `./` is the directory where the inference graph file should be generated. -* `inference_graph.pb` is the name of the generated inference graph file. -* `as_text` specifies whether the generated file should be in human readable text format or binary. +* ``sess`` is the instance of the TensorFlow Session object where the network topology is defined. +* ``["name_of_the_output_node"]`` is the list of output node names in the graph; ``frozen`` graph will include only those nodes from the original ``sess.graph_def`` that are directly or indirectly used to compute given output nodes. The ``'name_of_the_output_node'`` is an example of a possible output node name. You should derive the names based on your own graph. +* ``./`` is the directory where the inference graph file should be generated. +* ``inference_graph.pb`` is the name of the generated inference graph file. +* ``as_text`` specifies whether the generated file should be in human readable text format or binary. + +Converting TensorFlow 2 Models +############################### -## Converting TensorFlow 2 Models To convert TensorFlow 2 models, ensure that `openvino-dev[tensorflow2]` is installed via `pip`. TensorFlow 2.X officially supports two model formats: SavedModel and Keras H5 (or HDF5). Below are the instructions on how to convert each of them. -### SavedModel Format -A model in the SavedModel format consists of a directory with a `saved_model.pb` file and two subfolders: `variables` and `assets`. +SavedModel Format ++++++++++++++++++ + +A model in the SavedModel format consists of a directory with a ``saved_model.pb`` file and two subfolders: ``variables`` and ``assets``. To convert such a model, run the `mo` script with a path to the SavedModel directory: -```sh - mo --saved_model_dir -``` +.. code-block:: cpp + + mo --saved_model_dir TensorFlow 2 SavedModel format strictly requires the 2.x version of TensorFlow installed in the environment for conversion to the Intermediate Representation (IR). If a model contains operations currently unsupported by OpenVINO™, -prune these operations by explicit specification of input nodes using the `--input` or `--output` +prune these operations by explicit specification of input nodes using the ``--input`` or ``--output`` options. To determine custom input nodes, visualize a model graph in the TensorBoard. -To generate TensorBoard logs of the graph, use the Model Optimizer `--tensorboard_logs` command-line +To generate TensorBoard logs of the graph, use the Model Optimizer ``--tensorboard_logs`` command-line option. TensorFlow 2 SavedModel format has a specific graph structure due to eager execution. In case of -pruning, find custom input nodes in the `StatefulPartitionedCall/*` subgraph. +pruning, find custom input nodes in the ``StatefulPartitionedCall/*`` subgraph. + +Keras H5 +++++++++ -### Keras H5 If you have a model in the HDF5 format, load the model using TensorFlow 2 and serialize it in the SavedModel format. Here is an example of how to do it: -```python -import tensorflow as tf -model = tf.keras.models.load_model('model.h5') -tf.saved_model.save(model,'model') -``` +.. code-block:: python + + import tensorflow as tf + model = tf.keras.models.load_model('model.h5') + tf.saved_model.save(model,'model') + The Keras H5 model with a custom layer has specifics to be converted into SavedModel format. -For example, the model with a custom layer `CustomLayer` from `custom_layer.py` is converted as follows: +For example, the model with a custom layer ``CustomLayer`` from ``custom_layer.py`` is converted as follows: + +.. code-block:: python + + import tensorflow as tf + from custom_layer import CustomLayer + model = tf.keras.models.load_model('model.h5', custom_objects={'CustomLayer': CustomLayer}) + tf.saved_model.save(model,'model') -```python -import tensorflow as tf -from custom_layer import CustomLayer -model = tf.keras.models.load_model('model.h5', custom_objects={'CustomLayer': CustomLayer}) -tf.saved_model.save(model,'model') -``` Then follow the above instructions for the SavedModel format. -> **NOTE**: Do not use other hacks to resave TensorFlow 2 models into TensorFlow 1 formats. +.. note:: + + Do not use other hacks to resave TensorFlow 2 models into TensorFlow 1 formats. + +Command-Line Interface (CLI) Examples Using TensorFlow-Specific Parameters +########################################################################## -## Command-Line Interface (CLI) Examples Using TensorFlow-Specific Parameters * Launching the Model Optimizer for Inception V1 frozen model when model file is a plain text protobuf: -```sh - mo --input_model inception_v1.pbtxt --input_model_is_text -b 1 -``` +.. code-block:: cpp + + mo --input_model inception_v1.pbtxt --input_model_is_text -b 1 -* Launching the Model Optimizer for Inception V1 frozen model and dump information about the graph to TensorBoard log dir `/tmp/log_dir` -```sh - mo --input_model inception_v1.pb -b 1 --tensorboard_logdir /tmp/log_dir -``` +* Launching the Model Optimizer for Inception V1 frozen model and dump information about the graph to TensorBoard log dir ``/tmp/log_dir`` -* Launching the Model Optimizer for BERT model in the SavedModel format, with three inputs. Specify explicitly the input shapes -where the batch size and the sequence length equal 2 and 30 respectively. +.. code-block:: cpp -```sh -mo --saved_model_dir BERT --input mask,word_ids,type_ids --input_shape [2,30],[2,30],[2,30] -``` + mo --input_model inception_v1.pb -b 1 --tensorboard_logdir /tmp/log_dir -## Supported TensorFlow and TensorFlow 2 Keras Layers -For the list of supported standard layers, refer to the [Supported Framework Layers ](@ref openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers) page. -## Frequently Asked Questions (FAQ) -The Model Optimizer provides explanatory messages if it is unable to run to completion due to typographical errors, incorrectly used options, or other issues. The message describes the potential cause of the problem and gives a link to the [Model Optimizer FAQ](@ref openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ). The FAQ provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in the Model Optimizer Developer Guide to help you understand what went wrong. +* Launching the Model Optimizer for BERT model in the SavedModel format, with three inputs. Specify explicitly the input shapes where the batch size and the sequence length equal 2 and 30 respectively. + +.. code-block:: cpp + + mo --saved_model_dir BERT --input mask,word_ids,type_ids --input_shape [2,30],[2,30],[2,30] + + +Supported TensorFlow and TensorFlow 2 Keras Layers +################################################## + +For the list of supported standard layers, refer to the :doc:`Supported Framework Layers ` page. + +Frequently Asked Questions (FAQ) +################################ + +The Model Optimizer provides explanatory messages if it is unable to run to completion due to typographical errors, incorrectly used options, or other issues. The message describes the potential cause of the problem and gives a link to the :doc:`Model Optimizer FAQ `. The FAQ provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in the Model Optimizer Developer Guide to help you understand what went wrong. + +Summary +####### -## Summary In this document, you learned: * Basic information about how the Model Optimizer works with TensorFlow models. @@ -153,9 +188,13 @@ In this document, you learned: * How to freeze a TensorFlow model. * How to convert a trained TensorFlow model using the Model Optimizer with both framework-agnostic and TensorFlow-specific command-line options. -## Additional Resources -See the [Model Conversion Tutorials](@ref openvino_docs_MO_DG_prepare_model_convert_model_tutorials) page for a set of tutorials providing step-by-step instructions for converting specific TensorFlow models. Here are some examples: -* [Convert TensorFlow EfficientDet Models](@ref openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models) -* [Convert TensorFlow FaceNet Models](@ref openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow) -* [Convert TensorFlow Object Detection API Models](@ref openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models) +Additional Resources +#################### + +See the :doc:`Model Conversion Tutorials ` page for a set of tutorials providing step-by-step instructions for converting specific TensorFlow models. Here are some examples: + +* :doc:`Convert TensorFlow EfficientDet Models ` +* :doc:`Convert TensorFlow FaceNet Models ` +* :doc:`Convert TensorFlow Object Detection API Models ` +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_Tutorials.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_Tutorials.md index a46d1cbcb76962..c647709d218975 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_Tutorials.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_Tutorials.md @@ -35,9 +35,10 @@ openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model -@endsphinxdirective This section provides a set of tutorials that demonstrate conversion methods for specific TensorFlow, ONNX, PyTorch, MXNet, and Kaldi models, that unnecessarily cover your case. -Before studying the tutorials, try to convert the model out-of-the-box by specifying only the `--input_model` parameter in the command line. +Before studying the tutorials, try to convert the model out-of-the-box by specifying only the ``--input_model`` parameter in the command line. + +You will find a collection of :doc:`Python tutorials ` written for running on Jupyter notebooks that provide an introduction to the OpenVINO™ toolkit and explain how to use the Python API and tools for optimized deep learning inference. -You will find a collection of [Python tutorials](../../../tutorials.md) written for running on Jupyter notebooks that provide an introduction to the OpenVINO™ toolkit and explain how to use the Python API and tools for optimized deep learning inference. +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/Converting_Model.md b/docs/MO_DG/prepare_model/convert_model/Converting_Model.md index c829936310da89..3accddd855122e 100644 --- a/docs/MO_DG/prepare_model/convert_model/Converting_Model.md +++ b/docs/MO_DG/prepare_model/convert_model/Converting_Model.md @@ -6,86 +6,93 @@ With Model Optimizer you can increase your model's efficiency by providing an ad .. _when_to_specify_input_shapes: -@endsphinxdirective +Specifying --input_shape Command-line Parameter +############################################### -## Specifying --input_shape Command-line Parameter Model Optimizer supports conversion of models with dynamic input shapes that contain undefined dimensions. However, if the shape of data is not going to change from one inference request to another, it is recommended to set up static shapes (when all dimensions are fully defined) for the inputs. Doing it at this stage, instead of during inference in runtime, can be beneficial in terms of performance and memory consumption. -To set up static shapes, Model Optimizer provides the `--input_shape` parameter. -For more information on input shapes under runtime, refer to the [Changing input shapes](../../../OV_Runtime_UG/ShapeInference.md) guide. -To learn more about dynamic shapes in runtime, refer to the [Dynamic Shapes](../../../OV_Runtime_UG/ov_dynamic_shapes.md) guide. +To set up static shapes, Model Optimizer provides the ``--input_shape`` parameter. +For more information on input shapes under runtime, refer to the :doc:`Changing input shapes ` guide. +To learn more about dynamic shapes in runtime, refer to the :doc:`Dynamic Shapes ` guide. -The OpenVINO Runtime API may present certain limitations in inferring models with undefined dimensions on some hardware. See the [Features support matrix](../../../OV_Runtime_UG/supported_plugins/Device_Plugins.md) for reference. -In this case, the `--input_shape` parameter and the [reshape method](../../../OV_Runtime_UG/ShapeInference.md) can help to resolve undefined dimensions. +The OpenVINO Runtime API may present certain limitations in inferring models with undefined dimensions on some hardware. See the :doc:`Features support matrix ` for reference. +In this case, the ``--input_shape`` parameter and the :doc:`reshape method ` can help to resolve undefined dimensions. -Sometimes, Model Optimizer is unable to convert models out-of-the-box (only the `--input_model` parameter is specified). +Sometimes, Model Optimizer is unable to convert models out-of-the-box (only the ``--input_model`` parameter is specified). Such problem can relate to models with inputs of undefined ranks and a case of cutting off parts of a model. -In this case, input shapes must be specified explicitly with the `--input_shape` parameter. +In this case, input shapes must be specified explicitly with the ``--input_shape`` parameter. For example, run Model Optimizer for the TensorFlow MobileNet model with the single input -and specify the input shape of `[2,300,300,3]`: +and specify the input shape of ``[2,300,300,3]``: -```sh -mo --input_model MobileNet.pb --input_shape [2,300,300,3] -``` +.. code-block:: sh -If a model has multiple inputs, `--input_shape` must be used in conjunction with `--input` parameter. -The `--input` parameter contains a list of input names, for which shapes in the same order are defined via `--input_shape`. -For example, launch Model Optimizer for the ONNX OCR model with a pair of inputs `data` and `seq_len` -and specify shapes `[3,150,200,1]` and `[3]` for them: + mo --input_model MobileNet.pb --input_shape [2,300,300,3] -```sh -mo --input_model ocr.onnx --input data,seq_len --input_shape [3,150,200,1],[3] -``` -Alternatively, specify input shapes, using the `--input` parameter as follows: +If a model has multiple inputs, ``--input_shape`` must be used in conjunction with ``--input`` parameter. +The ``--input`` parameter contains a list of input names, for which shapes in the same order are defined via ``--input_shape``. +For example, launch Model Optimizer for the ONNX OCR model with a pair of inputs ``data`` and ``seq_len`` +and specify shapes ``[3,150,200,1]`` and ``[3]`` for them: -```sh -mo --input_model ocr.onnx --input data[3,150,200,1],seq_len[3] -``` +.. code-block:: sh -The `--input_shape` parameter allows overriding original input shapes to ones compatible with a given model. + mo --input_model ocr.onnx --input data,seq_len --input_shape [3,150,200,1],[3] + + +Alternatively, specify input shapes, using the ``--input`` parameter as follows: + +.. code-block:: sh + + mo --input_model ocr.onnx --input data[3,150,200,1],seq_len[3] + + +The ``--input_shape`` parameter allows overriding original input shapes to ones compatible with a given model. Dynamic shapes, i.e. with dynamic dimensions, can be replaced in the original model with static shapes for the converted model, and vice versa. -The dynamic dimension can be marked in Model Optimizer command-line as `-1`* or *`?`. +The dynamic dimension can be marked in Model Optimizer command-line as ``-1``* or *``?``. For example, launch Model Optimizer for the ONNX OCR model and specify dynamic batch dimension for inputs: -```sh -mo --input_model ocr.onnx --input data,seq_len --input_shape [-1,150,200,1],[-1] -``` +.. code-block:: sh + + mo --input_model ocr.onnx --input data,seq_len --input_shape [-1,150,200,1],[-1] + To optimize memory consumption for models with undefined dimensions in run-time, Model Optimizer provides the capability to define boundaries of dimensions. The boundaries of undefined dimension can be specified with ellipsis. For example, launch Model Optimizer for the ONNX OCR model and specify a boundary for the batch dimension: -```sh -mo --input_model ocr.onnx --input data,seq_len --input_shape [1..3,150,200,1],[1..3] -``` +.. code-block:: sh + + mo --input_model ocr.onnx --input data,seq_len --input_shape [1..3,150,200,1],[1..3] -@sphinxdirective Practically, some models are not ready for input shapes change. In this case, a new input shape cannot be set via Model Optimizer. For more information about shape follow the :doc:`inference troubleshooting ` and :ref:`ways to relax shape inference flow ` guides. -@endsphinxdirective +Specifying --static_shape Command-line Parameter +################################################ - -## Specifying --static_shape Command-line Parameter -Model Optimizer provides the `--static_shape` parameter that allows evaluating shapes of all operations in the model for fixed input shapes +Model Optimizer provides the ``--static_shape`` parameter that allows evaluating shapes of all operations in the model for fixed input shapes and folding shape computing sub-graphs into constants. The resulting IR may be more compact in size and the loading time for such IR may decrease. -However, the resulting IR will not be reshape-able with the help of the [reshape method](../../../OV_Runtime_UG/ShapeInference.md) from OpenVINO Runtime API. -It is worth noting that the `--input_shape` parameter does not affect reshapeability of the model. +However, the resulting IR will not be reshape-able with the help of the :doc:`reshape method ` from OpenVINO Runtime API. +It is worth noting that the ``--input_shape`` parameter does not affect reshapeability of the model. + +For example, launch Model Optimizer for the ONNX OCR model using ``--static_shape``: + +.. code-block:: sh + + mo --input_model ocr.onnx --input data[3,150,200,1],seq_len[3] --static_shape + -For example, launch Model Optimizer for the ONNX OCR model using `--static_shape`: +Additional Resources +#################### -```sh -mo --input_model ocr.onnx --input data[3,150,200,1],seq_len[3] --static_shape -``` +* :doc:`Introduction to converting models with Model Optimizer ` +* :doc:`Cutting Off Parts of a Model ` -## Additional Resources -* [Introduction to converting models with Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) -* [Cutting Off Parts of a Model](Cutting_Model.md) +@endsphinxdirective \ No newline at end of file diff --git a/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md b/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md index 9f3117af1f317d..55cd3679351bfd 100644 --- a/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md +++ b/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md @@ -1,391 +1,462 @@ # Cutting Off Parts of a Model {#openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model} +@sphinxdirective + Sometimes, it is necessary to remove parts of a model when converting it to OpenVINO IR. This chapter describes how to do it, using Model Optimizer command-line options. Model cutting applies mostly to TensorFlow models, which is why TensorFlow will be used in this chapter's examples, but it may be also useful for other frameworks. -## Purpose of Model Cutting +Purpose of Model Cutting +######################## The following examples are the situations when model cutting is useful or even required: -* A model has pre- or post-processing parts that cannot be translated to existing OpenVINO operations. -* A model has a training part that is convenient to be kept in the model but not used during inference. -* A model is too complex be converted at once, because it contains a lot of unsupported operations that cannot be easily implemented as custom layers. -* A problem occurs with model conversion in Model Optimizer or inference in OpenVINO™ Runtime. To identify the issue, limit the conversion scope by iterative search for problematic areas in the model. -* A single custom layer or a combination of custom layers is isolated for debugging purposes. +* A model has pre- or post-processing parts that cannot be translated to existing OpenVINO operations. +* A model has a training part that is convenient to be kept in the model but not used during inference. +* A model is too complex be converted at once, because it contains a lot of unsupported operations that cannot be easily implemented as custom layers. +* A problem occurs with model conversion in Model Optimizer or inference in OpenVINO™ Runtime. To identify the issue, limit the conversion scope by iterative search for problematic areas in the model. +* A single custom layer or a combination of custom layers is isolated for debugging purposes. + +Command-Line Options +#################### -## Command-Line Options +Model Optimizer provides command line options ``--input`` and ``--output`` to specify new entry and exit nodes, while ignoring the rest of the model: -Model Optimizer provides command line options `--input` and `--output` to specify new entry and exit nodes, while ignoring the rest of the model: +* ``--input`` option accepts a comma-separated list of layer names of the input model that should be treated as new entry points to the model. +* ``--output`` option accepts a comma-separated list of layer names of the input model that should be treated as new exit points from the model. -* `--input` option accepts a comma-separated list of layer names of the input model that should be treated as new entry points to the model. -* `--output` option accepts a comma-separated list of layer names of the input model that should be treated as new exit points from the model. +The ``--input`` option is required for cases unrelated to model cutting. For example, when the model contains several inputs and ``--input_shape`` or ``--mean_values`` options are used, the ``--input`` option specifies the order of input nodes for correct mapping between multiple items provided in ``--input_shape`` and ``--mean_values`` and the inputs in the model. -The `--input` option is required for cases unrelated to model cutting. For example, when the model contains several inputs and `--input_shape` or `--mean_values` options are used, the `--input` option specifies the order of input nodes for correct mapping between multiple items provided in `--input_shape` and `--mean_values` and the inputs in the model. +Model cutting is illustrated with the Inception V1 model, found in the ``models/research/slim`` repository. To proceed with this chapter, make sure you do the necessary steps to :doc:`prepare the model for Model Optimizer `. -Model cutting is illustrated with the Inception V1 model, found in the `models/research/slim` repository. To proceed with this chapter, make sure you do the necessary steps to [prepare the model for Model Optimizer](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model). +Default Behavior without --input and --output +############################################# -## Default Behavior without --input and --output +The input model is converted as a whole if neither ``--input`` nor ``--output`` command line options are used. All ``Placeholder`` operations in a TensorFlow graph are automatically identified as entry points. The ``Input`` layer type is generated for each of them. All nodes that have no consumers are automatically identified as exit points. -The input model is converted as a whole if neither `--input` nor `--output` command line options are used. All `Placeholder` operations in a TensorFlow graph are automatically identified as entry points. The `Input` layer type is generated for each of them. All nodes that have no consumers are automatically identified as exit points. +For Inception_V1, there is one ``Placeholder``: input. If the model is viewed in TensorBoard, the input operation is easy to find: -For Inception_V1, there is one `Placeholder`: input. If the model is viewed in TensorBoard, the input operation is easy to find: - -![Placeholder in Inception V1](../../img/inception_v1_std_input.svg) +.. image:: _static/images/inception_v1_std_input.svg + :alt: Placeholder in Inception V1 -`Reshape` is the only output operation, which is enclosed in a nested name scope of `InceptionV1/Logits/Predictions`, under the full name of `InceptionV1/Logits/Predictions/Reshape_1`. +``Reshape`` is the only output operation, which is enclosed in a nested name scope of ``InceptionV1/Logits/Predictions``, under the full name of ``InceptionV1/Logits/Predictions/Reshape_1``. In TensorBoard, along with some of its predecessors, it looks as follows: -![TensorBoard with predecessors](../../img/inception_v1_std_output.svg) +.. image:: _static/images/inception_v1_std_output.svg + :alt: TensorBoard with predecessors Convert this model and put the results in a writable output directory: -```sh -mo --input_model inception_v1.pb -b 1 --output_dir -``` -(The other examples on this page assume that you first go to the `model_optimizer` directory and add the `--output_dir` argument with a directory where you have read/write permissions.) - -The output `.xml` file with an Intermediate Representation contains the `Input` layer among other layers in the model: -```xml - - - - 1 - 3 - 224 - 224 - - - -``` -The `input` layer is converted from the TensorFlow graph `Placeholder` operation `input` and has the same name. - -The `-b` option is used here for conversion to override a possible undefined batch size (coded as -1 in TensorFlow models). If a model was frozen with a defined batch size, you may omit this option in all the examples. - -The last layer in the model is `InceptionV1/Logits/Predictions/Reshape_1`, which matches an output operation in the TensorFlow graph: -```xml - - - - - 1 - 1001 - - - - - 1 - 1001 - - - -``` -Due to automatic identification of inputs and outputs, providing the `--input` and `--output` options to convert the whole model is not required. The following commands are equivalent for the Inception V1 model: -```sh -mo --input_model inception_v1.pb -b 1 --output_dir - -mo --input_model inception_v1.pb -b 1 --input input --output InceptionV1/Logits/Predictions/Reshape_1 --output_dir -``` + +.. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --output_dir + + +(The other examples on this page assume that you first go to the ``model_optimizer`` directory and add the ``--output_dir`` argument with a directory where you have read/write permissions.) + +The output ``.xml`` file with an Intermediate Representation contains the ``Input`` layer among other layers in the model: + +.. code-block:: xml + + + + + 1 + 3 + 224 + 224 + + + + + +The ``input`` layer is converted from the TensorFlow graph ``Placeholder`` operation ``input`` and has the same name. + +The ``-b`` option is used here for conversion to override a possible undefined batch size (coded as -1 in TensorFlow models). If a model was frozen with a defined batch size, you may omit this option in all the examples. + +The last layer in the model is ``InceptionV1/Logits/Predictions/Reshape_1``, which matches an output operation in the TensorFlow graph: + +.. code-block:: xml + + + + + + 1 + 1001 + + + + + 1 + 1001 + + + + + +Due to automatic identification of inputs and outputs, providing the ``--input`` and ``--output`` options to convert the whole model is not required. The following commands are equivalent for the Inception V1 model: + +.. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --output_dir + + mo --input_model inception_v1.pb -b 1 --input input --output InceptionV1/Logits/Predictions/Reshape_1 --output_dir + + The Intermediate Representations are identical for both conversions. The same is true if the model has multiple inputs and/or outputs. -## Model Cutting +Model Cutting +#################### -Now, consider how to cut some parts of the model off. This chapter describes the first convolution block `InceptionV1/InceptionV1/Conv2d_1a_7x7` of the Inception V1 model to illustrate cutting: +Now, consider how to cut some parts of the model off. This chapter describes the first convolution block ``InceptionV1/InceptionV1/Conv2d_1a_7x7`` of the Inception V1 model to illustrate cutting: -![Inception V1 first convolution block](../../img/inception_v1_first_block.svg) +.. image:: _static/images/inception_v1_first_block.svg + :alt: Inception V1 first convolution block -### Cutting at the End +Cutting at the End +++++++++++++++++++++ If you want to cut your model at the end, you have the following options: -1. The following command cuts off the rest of the model after the `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`, making this node the last in the model: - ```sh - mo --input_model inception_v1.pb -b 1 --output=InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir - ``` +1. The following command cuts off the rest of the model after the ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu``, making this node the last in the model: + + .. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --output=InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir + + The resulting Intermediate Representation has three layers: -```xml - - - - - - ... - - - - - - ... - - - ... - - - - - - - - - ... - - - ... - - - - - - - - -``` - As shown in the TensorBoard picture, the original model has more nodes than its Intermediate Representation. Model Optimizer has fused batch normalization `InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm` with convolution `InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution`, which is why it is not present in the final model. This is not an effect of the `--output` option, it is the typical behavior of Model Optimizer for batch normalizations and convolutions. The effect of the `--output` is that the `ReLU` layer becomes the last one in the converted model. - -2. The following command cuts the edge that comes from 0 output port of the `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` and the rest of the model, making this node the last one in the model: - ```sh - mo --input_model inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu:0 --output_dir - ``` + + .. code-block:: xml + + + + + + + ... + + + + + + ... + + + ... + + + + + + + + + ... + + + ... + + + + + + + + + + + As shown in the TensorBoard picture, the original model has more nodes than its Intermediate Representation. Model Optimizer has fused batch normalization ``InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm`` with convolution ``InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution``, which is why it is not present in the final model. This is not an effect of the ``--output`` option, it is the typical behavior of Model Optimizer for batch normalizations and convolutions. The effect of the ``--output`` is that the ``ReLU`` layer becomes the last one in the converted model. + +2. The following command cuts the edge that comes from 0 output port of the ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`` and the rest of the model, making this node the last one in the model: + + .. code-block:: + + mo --input_model inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu:0 --output_dir + + The resulting Intermediate Representation has three layers, which are the same as in the previous case: -```xml - - - - - - ... - - - - - - ... - - - ... - - - - - - - - - ... - - - ... - - - - - - - - -``` + + .. code-block:: xml + + + + + + + ... + + + + + + ... + + + ... + + + + + + + + + ... + + + ... + + + + + + + + + + This type of cutting is useful for cutting multiple output edges. -3. The following command cuts the edge that comes to 0 input port of the `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` and the rest of the model including `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`, deleting this node and making the previous node `InceptionV1/InceptionV1/Conv2d_1a_7x7/Conv2D` the last in the model: - ```sh - mo --input_model inception_v1.pb -b 1 --output=0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir - ``` +3. The following command cuts the edge that comes to 0 input port of the ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`` and the rest of the model including ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu``, deleting this node and making the previous node ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Conv2D`` the last in the model: + + .. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --output=0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir + + The resulting Intermediate Representation has two layers, which are the same as the first two layers in the previous case: -```xml - - - - - - ... - - - - - - ... - - - ... - - - - - - - - - - - -``` - -### Cutting from the Beginning - -If you want to go further and cut the beginning of the model, leaving only the `ReLU` layer, you have the following options: - -1. Use the following command line, where `--input` and `--output` specify the same node in the graph: - ```sh - mo --input_model=inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir - ``` + + .. code-block:: xml + + + + + + + ... + + + + + + ... + + + ... + + + + + + + + + + + + + +Cutting from the Beginning +++++++++++++++++++++++++++ + +If you want to go further and cut the beginning of the model, leaving only the ``ReLU`` layer, you have the following options: + +1. Use the following command line, where ``--input`` and ``--output`` specify the same node in the graph: + + .. code-block:: sh + + mo --input_model=inception_v1.pb -b 1 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir + The resulting Intermediate Representation looks as follows: -```xml - - - - - - ... - - - - - ... - - - ... - - - - - - - -``` - `Input` layer is automatically created to feed the layer that is converted from the node specified in `--input`, which is `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` in this case. Model Optimizer does not replace the `ReLU` node by the `Input` layer. It produces such Intermediate Representation to make the node the first executable node in the final Intermediate Representation. Therefore, Model Optimizer creates enough `Inputs` to feed all input ports of the node that is passed in `--input`.
-Even though `--input_shape` is not specified in the command line, the shapes for layers are inferred from the beginning of the original TensorFlow model to the point, at which the new input is defined. It has the same shape [1,64,112,112] as the model converted as a whole or without cutting off the beginning. - -2. Cut the edge incoming to layer by port number. To specify the incoming port, use the following notation `--input=port:input_node`. -To cut everything before `ReLU` layer, cut the edge incoming to port 0 of `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` node: - ```sh - mo --input_model inception_v1.pb -b 1 --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir - ``` + + .. code-block:: xml + + + + + + + ... + + + + + ... + + + ... + + + + + + + + + + ``Input`` layer is automatically created to feed the layer that is converted from the node specified in ``--input``, which is ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`` in this case. Model Optimizer does not replace the ``ReLU`` node by the ``Input`` layer. It produces such Intermediate Representation to make the node the first executable node in the final Intermediate Representation. Therefore, Model Optimizer creates enough ``Inputs`` to feed all input ports of the node that is passed in ``--input``. + + Even though ``--input_shape`` is not specified in the command line, the shapes for layers are inferred from the beginning of the original TensorFlow model to the point, at which the new input is defined. It has the same shape ``[1,64,112,112]`` as the model converted as a whole or without cutting off the beginning. + +2. Cut the edge incoming to layer by port number. To specify the incoming port, use the following notation ``--input=port:input_node``. To cut everything before ``ReLU`` layer, cut the edge incoming to port 0 of ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`` node: + + .. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir + The resulting Intermediate Representation looks as follows: -```xml - - - - - - ... - - - - - ... - - - ... - - - - - - - -``` - `Input` layer is automatically created to feed the layer that is converted from the node specified in `--input`, which is `InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu` in this case. Model Optimizer does not replace the `ReLU` node by the `Input` layer, it produces such Intermediate Representation to make the node be the first executable node in the final Intermediate Representation. Therefore, Model Optimizer creates enough `Inputs` to feed all input ports of the node that is passed in `--input`.
-Even though `--input_shape` is not specified in the command line, the shapes for layers are inferred from the beginning of the original TensorFlow model to the point, at which the new input is defined. It has the same shape [1,64,112,112] as the model converted as a whole or without cutting off the beginning. - -3. Cut edge outcoming from layer by port number. To specify the outcoming port, use the following notation `--input=input_node:port`. -To cut everything before `ReLU` layer, cut edge from `InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1` node to `ReLU`: - ```sh - mo --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1:0 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir - ``` + + .. code-block:: xml + + + + + + + ... + + + + + ... + + + ... + + + + + + + + + + ``Input`` layer is automatically created to feed the layer that is converted from the node specified in ``--input``, which is ``InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu`` in this case. Model Optimizer does not replace the ``ReLU`` node by the ``Input`` layer, it produces such Intermediate Representation to make the node be the first executable node in the final Intermediate Representation. Therefore, Model Optimizer creates enough ``Inputs`` to feed all input ports of the node that is passed in ``--input``. + + Even though ``--input_shape`` is not specified in the command line, the shapes for layers are inferred from the beginning of the original TensorFlow model to the point, at which the new input is defined. It has the same shape ``[1,64,112,112]`` as the model converted as a whole or without cutting off the beginning. + +3. Cut edge outcoming from layer by port number. To specify the outcoming port, use the following notation ``--input=input_node:port``. To cut everything before ``ReLU`` layer, cut edge from ``InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1`` node to ``ReLU``: + + .. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/BatchNorm/batchnorm/add_1:0 --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir + + The resulting Intermediate Representation looks as follows: -```xml - - - - - - ... - - - - - ... - - - ... - - - - - - - -``` - -## Shape Override for New Inputs - -The input shape can be overridden with `--input_shape`. In this case, the shape is applied to the node referenced in `--input`, not to the original `Placeholder` in the model. For example, the command below -```sh - mo --input_model inception_v1.pb --input_shape=[1,5,10,20] --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir -``` - -gives the following shapes in the `Input` and `ReLU` layers: - -```xml - - - - 1 - 20 - 5 - 10 - - - - - - - 1 - 20 - 5 - 10 - - - - - 1 - 20 - 5 - 10 - - - -``` -An input shape [1,20,5,10] in the final Intermediate Representation differs from the shape [1,5,10,20] specified in the command line, because the original TensorFlow model uses NHWC layout, but the Intermediate Representation uses NCHW layout. Thus, usual NHWC to NCHW layout conversion occurred. - -When `--input_shape` is specified, shape inference inside Model Optimizer is not performed for the nodes in the beginning of the model that are not included in the translated region. It differs from the case when `--input_shape` is not specified as noted in the previous section, where the shape inference is still performed for such nodes to deduce shape for the layers that should fall into the final Intermediate Representation. Therefore, `--input_shape` should be used for a model with a complex graph with loops, which are not supported by Model Optimizer, to exclude such parts from the Model Optimizer shape inference process completely. - -## Inputs with Multiple Input Ports - -There are operations that contain more than one input port. In the example considered here, the convolution `InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution` is such operation. When `--input_shape` is not provided, a new `Input` layer is created for each dynamic input port for the node. If a port is evaluated to a constant blob, this constant remains in the model and a corresponding input layer is not created. TensorFlow convolution used in this model contains two ports: - -* port 0: input tensor for convolution (dynamic) -* port 1: convolution weights (constant) - -Following this behavior, Model Optimizer creates an `Input` layer for port 0 only, leaving port 1 as a constant. Thus, the result of: - -```sh -mo --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --output_dir -``` + + .. code-block:: xml + + + + + + + ... + + + + + ... + + + ... + + layer> + + + + + + + +Shape Override for New Inputs +############################# + +The input shape can be overridden with ``--input_shape``. In this case, the shape is applied to the node referenced in ``--input``, not to the original ``Placeholder`` in the model. For example, the command below: + +.. code-block:: sh + + mo --input_model inception_v1.pb --input_shape=[1,5,10,20] --output InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --input InceptionV1/InceptionV1/Conv2d_1a_7x7/Relu --output_dir + + +gives the following shapes in the ``Input`` and ``ReLU`` layers: + +.. code-block:: xml + + + + + 1 + 20 + 5 + 10 + + + + + + + 1 + 20 + 5 + 10 + + + + + 1 + 20 + 5 + 10 + + + + + +An input shape ``[1,20,5,10]`` in the final Intermediate Representation differs from the shape ``[1,5,10,20]`` specified in the command line, because the original TensorFlow model uses NHWC layout, but the Intermediate Representation uses NCHW layout. Thus, usual NHWC to NCHW layout conversion occurred. + +When ``--input_shape`` is specified, shape inference inside Model Optimizer is not performed for the nodes in the beginning of the model that are not included in the translated region. It differs from the case when ``--input_shape`` is not specified as noted in the previous section, where the shape inference is still performed for such nodes to deduce shape for the layers that should fall into the final Intermediate Representation. Therefore, ``--input_shape`` should be used for a model with a complex graph with loops, which are not supported by Model Optimizer, to exclude such parts from the Model Optimizer shape inference process completely. + +Inputs with Multiple Input Ports +################################ + +There are operations that contain more than one input port. In the example considered here, the convolution ``InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution`` is such operation. When ``--input_shape`` is not provided, a new ``Input`` layer is created for each dynamic input port for the node. If a port is evaluated to a constant blob, this constant remains in the model and a corresponding input layer is not created. TensorFlow convolution used in this model contains two ports: + +* port 0: input tensor for convolution (dynamic) +* port 1: convolution weights (constant) + +Following this behavior, Model Optimizer creates an ``Input`` layer for port 0 only, leaving port 1 as a constant. Thus, the result of: + +.. code-block:: sh + + mo --input_model inception_v1.pb -b 1 --input InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --output_dir + is identical to the result of conversion of the model as a whole, because this convolution is the first executable operation in Inception V1. -Different behavior occurs when `--input_shape` is also used as an attempt to override the input shape: -```sh -mo --input_model inception_v1.pb--input=InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape [1,224,224,3] --output_dir -``` -An error occurs (for more information, see the [Model Optimizer FAQ](@ref FAQ30): -```sh -[ ERROR ] Node InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution has more than 1 input and input shapes were provided. -Try not to provide input shapes or specify input port with PORT:NODE notation, where PORT is an integer. -For more information, see FAQ #30 -``` -When `--input_shape` is specified and the node contains multiple input ports, you need to provide an input port index together with an input node name. The input port index is specified in front of the node name with ':' as a separator (`PORT:NODE`). In this case, the port index 0 of the node `InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution` should be specified as `0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution`. +Different behavior occurs when ``--input_shape`` is also used as an attempt to override the input shape: + +.. code-block:: sh + + mo --input_model inception_v1.pb--input=InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape [1,224,224,3] --output_dir + + +An error occurs (for more information, see the :ref:`Model Optimizer FAQ `): + +.. code-block:: sh + + [ ERROR ] Node InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution has more than 1 input and input shapes were provided. + Try not to provide input shapes or specify input port with PORT:NODE notation, where PORT is an integer. + For more information, see FAQ #30 + +When ``--input_shape`` is specified and the node contains multiple input ports, you need to provide an input port index together with an input node name. The input port index is specified in front of the node name with ``‘:’`` as a separator (``PORT:NODE``). In this case, the port index 0 of the node ``InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution`` should be specified as ``0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution``. The correct command line is: -```sh -mo --input_model inception_v1.pb --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape=[1,224,224,3] --output_dir -``` + +.. code-block:: sh + + mo --input_model inception_v1.pb --input 0:InceptionV1/InceptionV1/Conv2d_1a_7x7/convolution --input_shape=[1,224,224,3] --output_dir + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/kaldi_specific/Aspire_Tdnn_Model.md b/docs/MO_DG/prepare_model/convert_model/kaldi_specific/Aspire_Tdnn_Model.md index f02c5e54bf72be..7292a1ac3089da 100644 --- a/docs/MO_DG/prepare_model/convert_model/kaldi_specific/Aspire_Tdnn_Model.md +++ b/docs/MO_DG/prepare_model/convert_model/kaldi_specific/Aspire_Tdnn_Model.md @@ -1,110 +1,145 @@ # Converting a Kaldi ASpIRE Chain Time Delay Neural Network (TDNN) Model {#openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model} -At the beginning, you should [download a pre-trained model](https://kaldi-asr.org/models/1/0001_aspire_chain_model.tar.gz) +@sphinxdirective + +At the beginning, you should `download a pre-trained model `__ for the ASpIRE Chain Time Delay Neural Network (TDNN) from the Kaldi project official website. -## Converting an ASpIRE Chain TDNN Model to IR +Converting an ASpIRE Chain TDNN Model to IR +########################################### Generate the Intermediate Representation of the model by running Model Optimizer with the following parameters: -```sh - mo --input_model exp/chain/tdnn_7b/final.mdl --output output -``` -The IR will have two inputs: `input` for data, and `ivector` for ivectors. +.. code-block:: sh + + mo --input_model exp/chain/tdnn_7b/final.mdl --output output + -## Example: Running ASpIRE Chain TDNN Model with the Speech Recognition Sample +The IR will have two inputs: ``input`` for data, and ``ivector`` for ivectors. -> **NOTE**: Before you continue with this part of the article, get familiar with the [Speech Recognition sample](../../../../../samples/cpp/speech_sample/README.md). +Example: Running ASpIRE Chain TDNN Model with the Speech Recognition Sample +########################################################################### -In this example, the input data contains one utterance from one speaker. +.. note:: + + Before you continue with this part of the article, get familiar with the + :doc:`Speech Recognition sample `. + +In this example, the input data contains one utterance from one speaker. To run the ASpIRE Chain TDNN Model with Speech Recognition sample, You need to prepare environment. Do it by following the steps below : -1. Download a [Kaldi repository](https://github.com/kaldi-asr/kaldi). -2. Build it by following instructions in `README.md` from the repository. -3. Download the [model archive](https://kaldi-asr.org/models/1/0001_aspire_chain_model.tar.gz) from Kaldi website. -4. Extract the downloaded model archive to the `egs/aspire/s5` folder of the Kaldi repository. + +1. Download a `Kaldi repository `__. +2. Build it by following instructions in ``README.md`` from the repository. +3. Download the `model archive `__ from Kaldi website. +4. Extract the downloaded model archive to the ``egs/aspire/s5`` folder of the Kaldi repository. Once everything has been prepared, you can start a proper run: -1. Prepare the model for decoding. Refer to the `README.txt` file from the downloaded model archive for instructions. -2. Convert data and ivectors to `.ark` format. Refer to the corresponding sections below for instructions. +1. Prepare the model for decoding. Refer to the ``README.txt`` file from the downloaded model archive for instructions. +2. Convert data and ivectors to ``.ark`` format. Refer to the corresponding sections below for instructions. -### Preparing Data +Preparing Data +++++++++++++++++++++ -If you have a `.wav` data file, convert it to the `.ark` format using the following command: -```sh -/src/featbin/compute-mfcc-feats --config=/egs/aspire/s5/conf/mfcc_hires.conf scp:./wav.scp ark,scp:feats.ark,feats.scp -``` -Add the `feats.ark` absolute path to `feats.scp` to avoid errors in later commands. +If you have a ``.wav`` data file, convert it to the ``.ark`` format using the following command: -### Preparing Ivectors +.. code-block:: sh + + /src/featbin/compute-mfcc-feats --config=/egs/aspire/s5/conf/mfcc_hires.conf scp:./wav.scp ark,scp:feats.ark,feats.scp + + +Add the ``feats.ark`` absolute path to ``feats.scp`` to avoid errors in later commands. + +Preparing Ivectors +++++++++++++++++++++ Prepare ivectors for the Speech Recognition sample: -1. Copy the `feats.scp` file to the `egs/aspire/s5/` directory of the built Kaldi repository and navigate there: -```sh -cp feats.scp /egs/aspire/s5/ -cd /egs/aspire/s5/ -``` +1. Copy the ``feats.scp`` file to the ``egs/aspire/s5/`` directory of the built Kaldi repository and navigate there: + + .. code-block:: sh + + cp feats.scp /egs/aspire/s5/ + cd /egs/aspire/s5/ + 2. Extract ivectors from the data: -```sh -./steps/online/nnet2/extract_ivectors_online.sh --nj 1 --ivector_period exp/tdnn_7b_chain_online/ivector_extractor -``` -You can simplify the preparation of ivectors for the Speech Recognition sample. To do it, specify the maximum number of frames in utterances as a parameter for `--ivector_period` -to get only one ivector per utterance. - -To get the maximum number of frames in utterances, use the following command line: -```sh -../../../src/featbin/feat-to-len scp:feats.scp ark,t: | cut -d' ' -f 2 - | sort -rn | head -1 -``` -As a result, you will find the `ivector_online.1.ark` file in ``. - -3. Go to the ``: -```sh -cd -``` - -4. Convert the `ivector_online.1.ark` file to text format, using the `copy-feats` tool. Run the following command: -```sh -/src/featbin/copy-feats --binary=False ark:ivector_online.1.ark ark,t:ivector_online.1.ark.txt -``` - -5. For the Speech Recognition sample, the `.ark` file must contain an ivector -for each frame. Copy the ivector `frame_count` times by running the below script in the Python command prompt: -```python -import subprocess - -subprocess.run(["/src/featbin/feat-to-len", "scp:/egs/aspire/s5/feats.scp", "ark,t:feats_length.txt"]) - -f = open("ivector_online.1.ark.txt", "r") -g = open("ivector_online_ie.ark.txt", "w") -length_file = open("feats_length.txt", "r") -for line in f: - if "[" not in line: - for i in range(frame_count): - line = line.replace("]", " ") - g.write(line) - else: - g.write(line) - frame_count = int(length_file.read().split(" ")[1]) -g.write("]") -f.close() -g.close() -length_file.close() -``` - -6. Create an `.ark` file from `.txt`: -```sh -/src/featbin/copy-feats --binary=True ark,t:ivector_online_ie.ark.txt ark:ivector_online_ie.ark -``` - -### Running the Speech Recognition Sample - -Run the Speech Recognition sample with the created ivector `.ark` file: -```sh -speech_sample -i feats.ark,ivector_online_ie.ark -m final.xml -d CPU -o prediction.ark -cw_l 17 -cw_r 12 -``` + + .. code-block:: sh + + ./steps/online/nnet2/extract_ivectors_online.sh --nj 1 --ivector_period exp/tdnn_7b_chain_online/ivector_extractor + + + You can simplify the preparation of ivectors for the Speech Recognition sample. To do it, specify the maximum number of frames in utterances as a parameter for ``--ivector_period`` to get only one ivector per utterance. + + To get the maximum number of frames in utterances, use the following command line: + + .. code-block:: sh + + ../../../src/featbin/feat-to-len scp:feats.scp ark,t: | cut -d' ' -f 2 - | sort -rn | head -1 + + + As a result, you will find the ``ivector_online.1.ark`` file in ````. + +3. Go to the ````: + + .. code-block:: sh + + cd + + +4. Convert the ``ivector_online.1.ark`` file to text format, using the ``copy-feats`` tool. Run the following command: + + .. code-block:: sh + + /src/featbin/copy-feats --binary=False ark:ivector_online.1.ark ark,t:ivector_online.1.ark.txt + + +5. For the Speech Recognition sample, the ``.ark`` file must contain an ivector +for each frame. Copy the ivector ``frame_count`` times by running the below script in the Python command prompt: + + .. code-block:: python + + import subprocess + + subprocess.run(["/src/featbin/feat-to-len", "scp:/egs/aspire/s5/feats.scp", "ark,t:feats_length.txt"]) + + f = open("ivector_online.1.ark.txt", "r") + g = open("ivector_online_ie.ark.txt", "w") + length_file = open("feats_length.txt", "r") + for line in f: + if "[" not in line: + for i in range(frame_count): + line = line.replace("]", " ") + g.write(line) + else: + g.write(line) + frame_count = int(length_file.read().split(" ")[1]) + g.write("]") + f.close() + g.close() + length_file.close() + + +6. Create an ``.ark`` file from ``.txt``: + + .. code-block:: sh + + /src/featbin/copy-feats --binary=True ark,t:ivector_online_ie.ark.txt ark:ivector_online_ie.ark + + +Running the Speech Recognition Sample ++++++++++++++++++++++++++++++++++++++ + +Run the Speech Recognition sample with the created ivector ``.ark`` file: + +.. code-block:: sh + + speech_sample -i feats.ark,ivector_online_ie.ark -m final.xml -d CPU -o prediction.ark -cw_l 17 -cw_r 12 + Results can be decoded as described in "Use of Sample in Kaldi Speech Recognition Pipeline" -in the [Speech Recognition Sample description](../../../../../samples/cpp/speech_sample/README.md) article. +in the :doc:`Speech Recognition Sample description ` article. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_GluonCV_Models.md b/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_GluonCV_Models.md index 98f2c924b0adbd..823adf3329edd7 100644 --- a/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_GluonCV_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_GluonCV_Models.md @@ -1,26 +1,41 @@ # Converting MXNet GluonCV Models {#openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models} -This article provides the instructions and examples on how to use Model Optimizer to convert [GluonCV SSD and YOLO-v3 models](https://gluon-cv.mxnet.io/model_zoo/detection.html) to IR. - -1. Choose the topology available from the [GluonCV Model Zoo](https://gluon-cv.mxnet.io/model_zoo/detection.html) and export to the MXNet format using the GluonCV API. For example, for the `ssd_512_mobilenet1.0` topology: -```python -from gluoncv import model_zoo, data, utils -from gluoncv.utils import export_block -net = model_zoo.get_model('ssd_512_mobilenet1.0_voc', pretrained=True) -export_block('ssd_512_mobilenet1.0_voc', net, preprocess=True, layout='HWC') -``` -As a result, you will get an MXNet model representation in `ssd_512_mobilenet1.0.params` and `ssd_512_mobilenet1.0.json` files generated in the current directory. -2. Run the Model Optimizer tool, specifying the `--enable_ssd_gluoncv` option. Make sure the `--input_shape` parameter is set to the input shape layout of your model (NHWC or NCHW). The examples below illustrate running the Model Optimizer for the SSD and YOLO-v3 models trained with the NHWC layout and located in the ``: +@sphinxdirective + +This article provides the instructions and examples on how to use Model Optimizer to convert `GluonCV SSD and YOLO-v3 models `__ to IR. + +1. Choose the topology available from the `GluonCV Model Zoo `__ and export to the MXNet format using the GluonCV API. For example, for the ``ssd_512_mobilenet1.0`` topology: + + .. code-block:: python + + from gluoncv import model_zoo, data, utils + from gluoncv.utils import export_block + net = model_zoo.get_model('ssd_512_mobilenet1.0_voc', pretrained=True) + export_block('ssd_512_mobilenet1.0_voc', net, preprocess=True, layout='HWC') + + As a result, you will get an MXNet model representation in ``ssd_512_mobilenet1.0.params`` and ``ssd_512_mobilenet1.0.json`` files generated in the current directory. + +2. Run the Model Optimizer tool, specifying the ``--enable_ssd_gluoncv`` option. Make sure the ``--input_shape`` parameter is set to the input shape layout of your model (NHWC or NCHW). The examples below illustrate running the Model Optimizer for the SSD and YOLO-v3 models trained with the NHWC layout and located in the ````: + * **For GluonCV SSD topologies:** -```sh - mo --input_model /ssd_512_mobilenet1.0.params --enable_ssd_gluoncv --input_shape [1,512,512,3] --input data --output_dir -``` + + .. code-block:: sh + + mo --input_model /ssd_512_mobilenet1.0.params --enable_ssd_gluoncv --input_shape [1,512,512,3] --input data --output_dir + * **For YOLO-v3 topology:** - * To convert the model: - ```sh - mo --input_model /yolo3_mobilenet1.0_voc-0000.params --input_shape [1,255,255,3] --output_dir - ``` - * To convert the model with replacing the subgraph with RegionYolo layers: - ```sh - mo --input_model /models/yolo3_mobilenet1.0_voc-0000.params --input_shape [1,255,255,3] --transformations_config "front/mxnet/yolo_v3_mobilenet1_voc.json" --output_dir - ``` + + * To convert the model: + + .. code-block:: sh + + mo --input_model /yolo3_mobilenet1.0_voc-0000.params --input_shape [1,255,255,3] --output_dir + + * To convert the model with replacing the subgraph with RegionYolo layers: + + .. code-block:: sh + + mo --input_model /models/yolo3_mobilenet1.0_voc-0000.params --input_shape [1,255,255,3] --transformations_config "front/mxnet/yolo_v3_mobilenet1_voc. json" --output_dir + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md b/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md index a87026e109edd8..0163e9e5ee7e8b 100644 --- a/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md +++ b/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md @@ -1,117 +1,171 @@ # Converting an MXNet Style Transfer Model {#openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet} +@sphinxdirective + This article provides instructions on how to generate a model for style transfer, using the public MXNet neural style transfer sample. -**Step 1**: Download or clone the repository [Zhaw's Neural Style Transfer repository](https://github.com/zhaw/neural_style) with an MXNet neural style transfer sample. +**Step 1**: Download or clone the repository `Zhaw's Neural Style Transfer repository `__ with an MXNet neural style transfer sample. **Step 2**: Prepare the environment required to work with the cloned repository: -> **NOTE**: Python-tk installation is needed only for Linux. Python for Windows includes it by default. +.. note:: + + Python-tk installation is needed only for Linux. Python for Windows includes it by default. + + +1. Install packages dependency. + + .. code-block:: sh + + sudo apt-get install python-tk -1. Install packages dependency.
-```sh -sudo apt-get install python-tk -``` 2. Install Python requirements: -```sh -pip3 install --user mxnet -pip3 install --user matplotlib -pip3 install --user scikit-image -``` - -**Step 3**: Download the pretrained [VGG19 model](https://github.com/dmlc/web-data/raw/master/mxnet/neural-style/model/vgg19.params) and save it to the root directory of the cloned repository. The sample expects the model `vgg19.params` file to be in that directory.
- -**Step 4**: Modify source code files of style transfer sample from the cloned repository:
- -1. Go to the `fast_mrf_cnn` subdirectory. -```sh -cd ./fast_mrf_cnn -``` - -2. Open the `symbol.py` file and modify the `decoder_symbol()` function. You should see the following code there: -```py -def decoder_symbol(): - data = mx.sym.Variable('data') - data = mx.sym.Convolution(data=data, num_filter=256, kernel=(3,3), pad=(1,1), stride=(1, 1), name='deco_conv1') -``` -Replace the code above with the following:
-```py -def decoder_symbol_with_vgg(vgg_symbol): - data = mx.sym.Convolution(data=vgg_symbol, num_filter=256, kernel=(3,3), pad=(1,1), stride=(1, 1), name='deco_conv1') -``` - -3. Save and close the `symbol.py` file. - -4. Open and edit the `make_image.py` file. Go to the `__init__()` function in the `Maker` class:
-```py -decoder = symbol.decoder_symbol() -``` -Modfiy it with the following code:
-```py -decoder = symbol.decoder_symbol_with_vgg(vgg_symbol) -``` - -5. To join the pretrained weights with the decoder weights, make the following changes: - After the code lines for loading the decoder weights:
- ```py - args = mx.nd.load('%s_decoder_args.nd'%model_prefix) - auxs = mx.nd.load('%s_decoder_auxs.nd'%model_prefix) - ``` - - Add the following line:
- ```py - arg_dict.update(args) - ``` - -6. Use `arg_dict` instead of `args` as a parameter of the `decoder.bind()` function. Find the line below:
-```py -self.deco_executor = decoder.bind(ctx=mx.gpu(), args=args, aux_states=auxs) -``` -Replace it with the following:
-```py -self.deco_executor = decoder.bind(ctx=mx.cpu(), args=arg_dict, aux_states=auxs) -``` -7. Add the following code to the end of the `generate()` function in the `Maker` class to save the result model as a `.json` file:
-```py -self.vgg_executor._symbol.save('{}-symbol.json'.format('vgg19')) -self.deco_executor._symbol.save('{}-symbol.json'.format('nst_vgg19')) -``` -8. Save and close the `make_image.py` file. - -**Step 5**: Follow the instructions from the `README.md` file in the `fast_mrf_cnn` directory of the cloned repository and run the sample with a decoder model. -For example, use the following code to run the sample with the pretrained decoder weights from the `models` folder and output shape:
-```py -import make_image -maker = make_image.Maker('models/13', (1024, 768)) -maker.generate('output.jpg', '../images/tubingen.jpg') -``` -The `models/13` string in the code above is composed of the following substrings: -* `models/` -- path to the folder that contains `.nd` files with pretrained styles weights. -* `13` -- prefix pointing to the default decoder for the repository, `13_decoder`. - -> **NOTE**: If an error prompts with "No module named `cPickle`", try running the script from Step 5 in Python 2. After that return to Python 3 for the remaining steps. - -Any style can be selected from [collection of pretrained weights](https://pan.baidu.com/s/1skMHqYp). On the Chinese-language page, click the down arrow next to a size in megabytes. Then wait for an overlay box to appear, and click the blue button in it to download. The `generate()` function generates `nst_vgg19-symbol.json` and `vgg19-symbol.json` files for the specified shape. In the code, it is [1024 x 768] for a 4:3 ratio. You can specify another, for example, [224,224] for a square ratio. + + .. code-block:: sh + + pip3 install --user mxnet + pip3 install --user matplotlib + pip3 install --user scikit-image + + +**Step 3**: Download the pre-trained `VGG19 model `__ and save it to the root directory of the cloned repository. The sample expects the model ``vgg19.params`` file to be in that directory. + +**Step 4**: Modify source code files of style transfer sample from the cloned repository: + +1. Go to the ``fast_mrf_cnn`` subdirectory. + + .. code-block:: sh + + cd ./fast_mrf_cnn + + +2. Open the ``symbol.py`` file and modify the ``decoder_symbol()`` function. You should see the following code there: + + .. code-block:: py + + def decoder_symbol(): + data = mx.sym.Variable('data') + data = mx.sym.Convolution(data=data, num_filter=256, kernel=(3,3), pad=(1,1), stride=(1, 1), name='deco_conv1') + + + Replace the code above with the following: + + .. code-block:: py + + def decoder_symbol_with_vgg(vgg_symbol): + data = mx.sym.Convolution(data=vgg_symbol, num_filter=256, kernel=(3,3), pad=(1,1), stride=(1, 1), name='deco_conv1') + + +3. Save and close the ``symbol.py`` file. + +4. Open and edit the ``make_image.py`` file. Go to the ``__init__()`` function in the ``Maker`` class: + + .. code-block:: py + + decoder = symbol.decoder_symbol() + + + Modify it with the following code: + + .. code-block:: py + + decoder = symbol.decoder_symbol_with_vgg(vgg_symbol) + + +5. To join the pre-trained weights with the decoder weights, make the following changes: + After the code lines for loading the decoder weights: + + .. code-block:: py + + args = mx.nd.load('%s_decoder_args.nd'%model_prefix) + auxs = mx.nd.load('%s_decoder_auxs.nd'%model_prefix) + + + Add the following line: + + .. code-block:: py + + arg_dict.update(args) + + +6. Use ``arg_dict`` instead of ``args`` as a parameter of the ``decoder.bind()`` function. Find the line below: + + .. code-block:: py + + self.deco_executor = decoder.bind(ctx=mx.gpu(), args=args, aux_states=auxs) + + + Replace it with the following: + + .. code-block:: py + + self.deco_executor = decoder.bind(ctx=mx.cpu(), args=arg_dict, aux_states=auxs) + + +7. Add the following code to the end of the ``generate()`` function in the ``Maker`` class to save the result model as a ``.json`` file: + + .. code-block:: py + + self.vgg_executor._symbol.save('{}-symbol.json'.format('vgg19')) + self.deco_executor._symbol.save('{}-symbol.json'.format('nst_vgg19')) + + +8. Save and close the ``make_image.py`` file. + +**Step 5**: Follow the instructions from the ``README.md`` file in the ``fast_mrf_cnn`` directory of the cloned repository and run the sample with a decoder model. +For example, use the following code to run the sample with the pre-trained decoder weights from the ``models`` folder and output shape: + +.. code-block:: py + + import make_image + maker = make_image.Maker('models/13', (1024, 768)) + maker.generate('output.jpg', '../images/tubingen.jpg') + + +The ``models/13`` string in the code above is composed of the following substrings: + +* ``models/`` -- path to the folder that contains ``.nd`` files with pre-trained styles weights. +* ``13`` -- prefix pointing to the default decoder for the repository, ``13_decoder``. + +.. note:: + + If an error prompts with "No module named ``cPickle``", try running the script from Step 5 in Python 2. After that return to Python 3 for the remaining steps. + +Any style can be selected from `collection of pre-trained weights `__. On the Chinese-language page, click the down arrow next to a size in megabytes. Then wait for an overlay box to appear, and click the blue button in it to download. The ``generate()`` function generates ``nst_vgg19-symbol.json`` and ``vgg19-symbol.json`` files for the specified shape. In the code, it is ``[1024 x 768]`` for a 4:3 ratio. You can specify another, for example, ``[224,224]`` for a square ratio. **Step 6**: Run the Model Optimizer to generate an Intermediate Representation (IR): -1. Create a new directory. For example:
-```sh -mkdir nst_model -``` -2. Copy the initial and generated model files to the created directory. For example, to copy the pretrained decoder weights from the `models` folder to the `nst_model` directory, run the following commands:
-```sh -cp nst_vgg19-symbol.json nst_model -cp vgg19-symbol.json nst_model -cp ../vgg19.params nst_model/vgg19-0000.params -cp models/13_decoder_args.nd nst_model -cp models/13_decoder_auxs.nd nst_model -``` -> **NOTE**: Make sure that all the `.params` and `.json` files are in the same directory as the `.nd` files. Otherwise, the conversion process fails. - -3. Run the Model Optimizer for Apache MXNet. Use the `--nd_prefix_name` option to specify the decoder prefix and `--input_shape` to specify input shapes in [N,C,W,H] order. For example:
-```sh -mo --input_symbol /nst_vgg19-symbol.json --framework mxnet --output_dir --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model /vgg19-0000.params -``` -4. The IR is generated (`.bin`, `.xml` and `.mapping` files) in the specified output directory, and ready to be consumed by the OpenVINO Runtime. +1. Create a new directory. For example: + + .. code-block:: sh + + mkdir nst_model + + +2. Copy the initial and generated model files to the created directory. For example, to copy the pre-trained decoder weights from the ``models`` folder to the ``nst_model`` directory, run the following commands: + + .. code-block:: sh + + cp nst_vgg19-symbol.json nst_model + cp vgg19-symbol.json nst_model + cp ../vgg19.params nst_model/vgg19-0000.params + cp models/13_decoder_args.nd nst_model + cp models/13_decoder_auxs.nd nst_model + + + .. note:: + + Make sure that all the ``.params`` and ``.json`` files are in the same directory as the ``.nd`` files. Otherwise, the conversion process fails. + + +3. Run the Model Optimizer for Apache MXNet. Use the ``--nd_prefix_name`` option to specify the decoder prefix and ``--input_shape`` to specify input shapes in ``[N,C,W,H]`` order. For example: + + .. code-block:: sh + + mo --input_symbol /nst_vgg19-symbol.json --framework mxnet --output_dir --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model /vgg19-0000.params + + +4. The IR is generated (``.bin``, ``.xml`` and ``.mapping`` files) in the specified output directory, and ready to be consumed by the OpenVINO Runtime. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Faster_RCNN.md b/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Faster_RCNN.md index e16c86c8ed2671..ccebf3b7178b4a 100644 --- a/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Faster_RCNN.md +++ b/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Faster_RCNN.md @@ -1,23 +1,29 @@ # Converting an ONNX Faster R-CNN Model {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN} -The instructions below are applicable **only** to the Faster R-CNN model converted to the ONNX file format from the [maskrcnn-benchmark model](https://github.com/facebookresearch/maskrcnn-benchmark): +@sphinxdirective -1. Download the pretrained model file from [onnx/models](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/faster-rcnn): - * (commit-SHA: 8883e49e68de7b43e263d56b9ed156dfa1e03117). +The instructions below are applicable **only** to the Faster R-CNN model converted to the ONNX file format from the `maskrcnn-benchmark model `__: + +1. Download the pretrained model file from `onnx/models `__ (commit-SHA: 8883e49e68de7b43e263d56b9ed156dfa1e03117). 2. Generate the Intermediate Representation of the model, by changing your current working directory to the Model Optimizer installation directory, and running Model Optimizer with the following parameters: -```sh - mo \ ---input_model FasterRCNN-10.onnx \ ---input_shape [1,3,800,800] \ ---input 0:2 \ ---mean_values [102.9801,115.9465,122.7717] \ ---transformations_config front/onnx/faster_rcnn.json -``` - -Be aware that the height and width specified with the `input_shape` command line parameter could be different. For more information about supported input image dimensions and required pre- and post-processing steps, refer to the [Faster R-CNN article](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/faster-rcnn). - -3. Interpret the outputs of the generated IR: class indices, probabilities and box coordinates. Below are the outputs from the "DetectionOutput" layer: - * class indices. - * probabilities. - * box coordinates. + +.. code-block:: sh + + mo \ + --input_model FasterRCNN-10.onnx \ + --input_shape [1,3,800,800] \ + --input 0:2 \ + --mean_values [102.9801,115.9465,122.7717] \ + --transformations_config front/onnx/faster_rcnn.json + + +Be aware that the height and width specified with the ``input_shape`` command line parameter could be different. For more information about supported input image dimensions and required pre- and post-processing steps, refer to the `Faster R-CNN article `__. + +3. Interpret the outputs of the generated IR: class indices, probabilities and box coordinates. Below are the outputs from the ``DetectionOutput`` layer: + + * class indices + * probabilities + * box coordinates + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_GPT2.md b/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_GPT2.md index cdd2b41b7de961..c694ad431a47a8 100644 --- a/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_GPT2.md +++ b/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_GPT2.md @@ -1,17 +1,25 @@ # Converting an ONNX GPT-2 Model {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2} -[Public pretrained GPT-2 model](https://github.com/onnx/models/tree/master/text/machine_comprehension/gpt-2) is a large +@sphinxdirective + +`Public pre-trained GPT-2 model `__ is a large transformer-based language model with a simple objective: predict the next word, given all of the previous words within some text. -## Downloading the Pre-Trained Base GPT-2 Model +Downloading the Pre-Trained Base GPT-2 Model +############################################ -To download the model, go to [this model](https://github.com/onnx/models/blob/master/text/machine_comprehension/gpt-2/model/gpt2-10.onnx), and press **Download**. +To download the model, go to `this model `__, and press **Download**. -To download the model and sample test data, go to [this model](https://github.com/onnx/models/blob/master/text/machine_comprehension/gpt-2/model/gpt2-10.tar.gz), and press **Download**. +To download the model and sample test data, go to `this model `__, and press **Download**. -## Converting an ONNX GPT-2 Model to IR +Converting an ONNX GPT-2 Model to IR +#################################### Generate the Intermediate Representation of the model GPT-2 by running Model Optimizer with the following parameters: -```sh -mo --input_model gpt2-10.onnx --input_shape [X,Y,Z] --output_dir -``` + +.. code-block:: sh + + mo --input_model gpt2-10.onnx --input_shape [X,Y,Z] --output_dir + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Mask_RCNN.md b/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Mask_RCNN.md index fed92fabe20bf7..63f64bb39824d3 100644 --- a/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Mask_RCNN.md +++ b/docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Mask_RCNN.md @@ -1,26 +1,32 @@ # Converting an ONNX Mask R-CNN Model {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN} -The instructions below are applicable **only** to the Mask R-CNN model converted to the ONNX file format from the [maskrcnn-benchmark model](https://github.com/facebookresearch/maskrcnn-benchmark). +@sphinxdirective -1. Download the pretrained model file from [onnx/models](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/mask-rcnn): - * commit-SHA: 8883e49e68de7b43e263d56b9ed156dfa1e03117. +The instructions below are applicable **only** to the Mask R-CNN model converted to the ONNX file format from the `maskrcnn-benchmark model `__. + +1. Download the pretrained model file from `onnx/models `__ (commit-SHA: 8883e49e68de7b43e263d56b9ed156dfa1e03117). 2. Generate the Intermediate Representation of the model by changing your current working directory to the Model Optimizer installation directory and running Model Optimizer with the following parameters: -```sh - mo \ ---input_model mask_rcnn_R_50_FPN_1x.onnx \ ---input "0:2" \ ---input_shape [1,3,800,800] \ ---mean_values [102.9801,115.9465,122.7717] \ ---transformations_config front/onnx/mask_rcnn.json -``` - -Be aware that the height and width specified with the `input_shape` command line parameter could be different. For more information about supported input image dimensions and required pre- and post-processing steps, refer to the [documentation](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/mask-rcnn). - -3. Interpret the outputs of the generated IR file: masks, class indices, probabilities and box coordinates. - * masks. - * class indices. - * probabilities. - * box coordinates. - -The first one is a layer with the name `6849/sink_port_0`, and rest are outputs from the `DetectionOutput` layer. + +.. code-block:: sh + + mo \ + --input_model mask_rcnn_R_50_FPN_1x.onnx \ + --input "0:2" \ + --input_shape [1,3,800,800] \ + --mean_values [102.9801,115.9465,122.7717] \ + --transformations_config front/onnx/mask_rcnn.json + + +Be aware that the height and width specified with the ``input_shape`` command line parameter could be different. For more information about supported input image dimensions and required pre- and post-processing steps, refer to the `documentation `__. + +3. Interpret the outputs of the generated IR file: masks, class indices, probabilities and box coordinates: + + * masks + * class indices + * probabilities + * box coordinates + +The first one is a layer with the name ``6849/sink_port_0``, and rest are outputs from the ``DetectionOutput`` layer. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Bert_ner.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Bert_ner.md index 846ffebdefa928..962165553398fa 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Bert_ner.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Bert_ner.md @@ -1,58 +1,66 @@ # Converting a PyTorch BERT-NER Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Bert_ner} +@sphinxdirective + The goal of this article is to present a step-by-step guide on how to convert PyTorch BERT-NER model to OpenVINO IR. First, you need to download the model and convert it to ONNX. -## Downloading and Converting the Model to ONNX +Downloading and Converting the Model to ONNX +############################################ To download a pretrained model or train the model yourself, refer -to the [instructions](https://github.com/kamalkraj/BERT-NER/blob/dev/README.md) in the -BERT-NER model repository. The model with configuration files is stored in the `out_base` directory. +to the `instructions `__ in the +BERT-NER model repository. The model with configuration files is stored in the ``out_base`` directory. To convert the model to ONNX format, create and run the following script in the root directory of the model repository. If you download the pretrained model, you need -to download [bert.py](https://github.com/kamalkraj/BERT-NER/blob/dev/bert.py) to run the script. -The instructions were tested with the commit-SHA: `e5be564156f194f1becb0d82aeaf6e762d9eb9ed`. - -```python -import torch - -from bert import Ner - -ner = Ner("out_base") - -input_ids, input_mask, segment_ids, valid_positions = ner.preprocess('Steve went to Paris') -input_ids = torch.tensor([input_ids], dtype=torch.long, device=ner.device) -input_mask = torch.tensor([input_mask], dtype=torch.long, device=ner.device) -segment_ids = torch.tensor([segment_ids], dtype=torch.long, device=ner.device) -valid_ids = torch.tensor([valid_positions], dtype=torch.long, device=ner.device) - -ner_model, tknizr, model_config = ner.load_model("out_base") - -with torch.no_grad(): - logits = ner_model(input_ids, segment_ids, input_mask, valid_ids) -torch.onnx.export(ner_model, - (input_ids, segment_ids, input_mask, valid_ids), - "bert-ner.onnx", - input_names=['input_ids', 'segment_ids', 'input_mask', 'valid_ids'], - output_names=['output'], - dynamic_axes={ - "input_ids": {0: "batch_size"}, - "segment_ids": {0: "batch_size"}, - "input_mask": {0: "batch_size"}, - "valid_ids": {0: "batch_size"}, - "output": {0: "output"} - }, - opset_version=11, - ) -``` - -The script generates ONNX model file `bert-ner.onnx`. - -## Converting an ONNX BERT-NER model to IR - -```bash -mo --input_model bert-ner.onnx --input "input_mask[1,128],segment_ids[1,128],input_ids[1,128]" -``` - -where `1` is `batch_size` and `128` is `sequence_length`. +to download `bert.py `__ to run the script. +The instructions were tested with the commit-SHA: ``e5be564156f194f1becb0d82aeaf6e762d9eb9ed``. + +.. code-block:: python + + import torch + + from bert import Ner + + ner = Ner("out_base") + + input_ids, input_mask, segment_ids, valid_positions = ner.preprocess('Steve went to Paris') + input_ids = torch.tensor([input_ids], dtype=torch.long, device=ner.device) + input_mask = torch.tensor([input_mask], dtype=torch.long, device=ner.device) + segment_ids = torch.tensor([segment_ids], dtype=torch.long, device=ner.device) + valid_ids = torch.tensor([valid_positions], dtype=torch.long, device=ner.device) + + ner_model, tknizr, model_config = ner.load_model("out_base") + + with torch.no_grad(): + logits = ner_model(input_ids, segment_ids, input_mask, valid_ids) + torch.onnx.export(ner_model, + (input_ids, segment_ids, input_mask, valid_ids), + "bert-ner.onnx", + input_names=['input_ids', 'segment_ids', 'input_mask', 'valid_ids'], + output_names=['output'], + dynamic_axes={ + "input_ids": {0: "batch_size"}, + "segment_ids": {0: "batch_size"}, + "input_mask": {0: "batch_size"}, + "valid_ids": {0: "batch_size"}, + "output": {0: "output"} + }, + opset_version=11, + ) + + +The script generates ONNX model file ``bert-ner.onnx``. + +Converting an ONNX BERT-NER model to IR +####################################### + +.. code-block:: bash + + mo --input_model bert-ner.onnx --input "input_mask[1,128],segment_ids[1,128],input_ids[1,128]" + + +where ``1`` is ``batch_size`` and ``128`` is ``sequence_length``. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Cascade_RCNN_res101.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Cascade_RCNN_res101.md index de8063f8499e35..bb0649bf172b11 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Cascade_RCNN_res101.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Cascade_RCNN_res101.md @@ -1,30 +1,41 @@ # Converting a PyTorch Cascade RCNN R-101 Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Cascade_RCNN_res101} +@sphinxdirective + The goal of this article is to present a step-by-step guide on how to convert a PyTorch Cascade RCNN R-101 model to OpenVINO IR. First, you need to download the model and convert it to ONNX. -## Downloading and Converting Model to ONNX +Downloading and Converting Model to ONNX +######################################## + +* Clone the `repository `__ : + +.. code-block:: sh + + git clone https://github.com/open-mmlab/mmdetection + cd mmdetection + + +.. note:: + + To set up an environment, refer to the `instructions `__. + +* Download the pre-trained `model `__. The model is also available `here `__. + +* To convert the model to ONNX format, use this `script `__. -* Clone the [repository](https://github.com/open-mmlab/mmdetection): + .. code-block:: sh -```bash -git clone https://github.com/open-mmlab/mmdetection -cd mmdetection -``` + python3 tools/deployment/pytorch2onnx.py configs/cascade_rcnn/cascade_rcnn_r101_fpn_1x_coco.py cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth --output-file cascade_rcnn_r101_fpn_1x_coco.onnx -> **NOTE**: To set up an environment, refer to the [instructions](https://github.com/open-mmlab/mmdetection/blob/master/docs/en/get_started.md#installation). -* Download the pretrained [model](https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_rcnn_r101_fpn_1x_coco/cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth). The model is also available [here](https://github.com/open-mmlab/mmdetection/blob/master/configs/cascade_rcnn/README.md). +The script generates ONNX model file ``cascade_rcnn_r101_fpn_1x_coco.onnx`` in the directory ``tools/deployment/``. If required, specify the model name or output directory, using ``--output-file /.onnx``. -* To convert the model to ONNX format, use this [script](https://github.com/open-mmlab/mmdetection/blob/master/tools/deployment/pytorch2onnx.py). +Converting an ONNX Cascade RCNN R-101 Model to OpenVINO IR +########################################################## -```bash -python3 tools/deployment/pytorch2onnx.py configs/cascade_rcnn/cascade_rcnn_r101_fpn_1x_coco.py cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth --output-file cascade_rcnn_r101_fpn_1x_coco.onnx -``` +.. code-block:: sh -The script generates ONNX model file `cascade_rcnn_r101_fpn_1x_coco.onnx` in the directory `tools/deployment/`. If required, specify the model name or output directory, using `--output-file /.onnx`. + mo --input_model cascade_rcnn_r101_fpn_1x_coco.onnx --mean_values [123.675,116.28,103.53] --scale_values [58.395,57.12,57.375] -## Converting an ONNX Cascade RCNN R-101 Model to OpenVINO IR -```bash -mo --input_model cascade_rcnn_r101_fpn_1x_coco.onnx --mean_values [123.675,116.28,103.53] --scale_values [58.395,57.12,57.375] -``` +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_F3Net.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_F3Net.md index c301558689c8b2..d8be169689c822 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_F3Net.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_F3Net.md @@ -1,33 +1,45 @@ # Converting a PyTorch F3Net Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_F3Net} -[F3Net](https://github.com/weijun88/F3Net): Fusion, Feedback and Focus for Salient Object Detection +@sphinxdirective -## Cloning the F3Net Repository +`F3Net `__ : Fusion, Feedback and Focus for Salient Object Detection + +Cloning the F3Net Repository +############################ To clone the repository, run the following command: -```sh -git clone http://github.com/weijun88/F3Net.git -``` +.. code-block:: sh + + git clone http://github.com/weijun88/F3Net.git -## Downloading and Converting the Model to ONNX + +Downloading and Converting the Model to ONNX +############################################ To download the pretrained model or train the model yourself, refer to the -[instructions](https://github.com/weijun88/F3Net/blob/master/README.md) in the F3Net model repository. First, convert the model to ONNX format. Create and run the following Python script in the `src` directory of the model repository: -```python -import torch -from dataset import Config -from net import F3Net - -cfg = Config(mode='test', snapshot=) -net = F3Net(cfg) -image = torch.zeros([1, 3, 352, 352]) -torch.onnx.export(net, image, 'f3net.onnx', export_params=True, do_constant_folding=True, opset_version=11) -``` -The script generates the ONNX model file `f3net.onnx`. The model conversion was tested with the commit-SHA: `eecace3adf1e8946b571a4f4397681252f9dc1b8`. - -## Converting an ONNX F3Net Model to IR - -```sh -mo --input_model /f3net.onnx -``` +`instructions `__ in the F3Net model repository. First, convert the model to ONNX format. Create and run the following Python script in the ``src`` directory of the model repository: + +.. code-block:: python + + import torch + from dataset import Config + from net import F3Net + + cfg = Config(mode='test', snapshot=) + net = F3Net(cfg) + image = torch.zeros([1, 3, 352, 352]) + torch.onnx.export(net, image, 'f3net.onnx', export_params=True, do_constant_folding=True, opset_version=11) + + +The script generates the ONNX model file ``f3net.onnx``. The model conversion was tested with the commit-SHA: ``eecace3adf1e8946b571a4f4397681252f9dc1b8``. + +Converting an ONNX F3Net Model to IR +#################################### + +.. code-block:: sh + + mo --input_model /f3net.onnx + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_QuartzNet.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_QuartzNet.md index d7131df957bbb8..451431b735870a 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_QuartzNet.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_QuartzNet.md @@ -1,40 +1,51 @@ # Converting a PyTorch QuartzNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_QuartzNet} -[NeMo project](https://github.com/NVIDIA/NeMo) provides the QuartzNet model. +@sphinxdirective -## Downloading the Pretrained QuartzNet Model +`NeMo project `__ provides the QuartzNet model. -To download the pretrained model, refer to the [NeMo Speech Models Catalog](https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels). +Downloading the Pre-trained QuartzNet Model +########################################### + +To download the pre-trained model, refer to the `NeMo Speech Models Catalog `__. Here are the instructions on how to obtain QuartzNet in ONNX format. -1. Install the NeMo toolkit, using the [instructions](https://github.com/NVIDIA/NeMo/tree/main#installation). +1. Install the NeMo toolkit, using the `instructions `__. 2. Run the following code: -```python -import nemo -import nemo.collections.asr as nemo_asr +.. code-block:: python + + import nemo + import nemo.collections.asr as nemo_asr + + quartznet = nemo_asr.models.EncDecCTCModel.from_pretrained(model_name="QuartzNet15x5Base-En") + # Export QuartzNet model to ONNX format + quartznet.decoder.export('decoder_qn.onnx') + quartznet.encoder.export('encoder_qn.onnx') + quartznet.export('qn.onnx') -quartznet = nemo_asr.models.EncDecCTCModel.from_pretrained(model_name="QuartzNet15x5Base-En") -# Export QuartzNet model to ONNX format -quartznet.decoder.export('decoder_qn.onnx') -quartznet.encoder.export('encoder_qn.onnx') -quartznet.export('qn.onnx') -``` -This code produces 3 ONNX model files: `encoder_qn.onnx`, `decoder_qn.onnx`, `qn.onnx`. -They are `decoder`, `encoder`, and a combined `decoder(encoder(x))` models, respectively. +This code produces 3 ONNX model files: ``encoder_qn.onnx``, ``decoder_qn.onnx``, ``qn.onnx``. +They are ``decoder``, ``encoder``, and a combined ``decoder(encoder(x))`` models, respectively. -## Converting an ONNX QuartzNet model to IR +Converting an ONNX QuartzNet model to IR +######################################## If using a combined model: -```sh -mo --input_model /qt.onnx --input_shape [B,64,X] -``` + +.. code-block:: sh + + mo --input_model /qt.onnx --input_shape [B,64,X] + If using separate models: -```sh -mo --input_model /encoder_qt.onnx --input_shape [B,64,X] -mo --input_model /decoder_qt.onnx --input_shape [B,1024,Y] -``` -Where shape is determined by the audio file Mel-Spectrogram length: B - batch dimension, X - dimension based on the input length, Y - determined by encoder output, usually `X / 2`. +.. code-block:: sh + + mo --input_model /encoder_qt.onnx --input_shape [B,64,X] + mo --input_model /decoder_qt.onnx --input_shape [B,1024,Y] + + +Where shape is determined by the audio file Mel-Spectrogram length: ``B`` - batch dimension, ``X`` - dimension based on the input length, ``Y`` - determined by encoder output, usually ``X / 2``. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RCAN.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RCAN.md index 3cea756301ca6a..3e89de179d4c86 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RCAN.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RCAN.md @@ -1,30 +1,39 @@ # Converting a PyTorch RCAN Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RCAN} -[RCAN](https://github.com/yulunzhang/RCAN): Image Super-Resolution Using Very Deep Residual Channel Attention Networks +@sphinxdirective -## Downloading and Converting the Model to ONNX +`RCAN `__ : Image Super-Resolution Using Very Deep Residual Channel Attention Networks -To download the pretrained model or train the model yourself, refer to the [instruction](https://github.com/yulunzhang/RCAN/blob/master/README.md) in the RCAN model repository. First, convert the model to ONNX format. Create and run the script with the following content in the root +Downloading and Converting the Model to ONNX +############################################ + +To download the pre-trained model or train the model yourself, refer to the `instruction `__ in the RCAN model repository. First, convert the model to ONNX format. Create and run the script with the following content in the root directory of the model repository: -```python -from argparse import Namespace +.. code-block:: python + + from argparse import Namespace + + import torch + + from RCAN_TestCode.code.model.rcan import RCAN + + config = Namespace(n_feats=64, n_resblocks=4, n_resgroups=2, reduction=16, scale=[2], data_train='DIV2K', res_scale=1, + n_colors=3, rgb_range=255) + net = RCAN(config) + net.eval() + dummy_input = torch.randn(1, 3, 360, 640) + torch.onnx.export(net, dummy_input, 'RCAN.onnx') + + +The script generates the ONNX model file ``RCAN.onnx``. More information about model parameters (``n_resblocks``, ``n_resgroups``, and others) and their different values can be found in the model repository. The model conversion was tested with the commit-SHA: ``3339ebc59519c3bb2b5719b87dd36515ec7f3ba7``. -import torch +Converting an ONNX RCAN Model to IR +################################### -from RCAN_TestCode.code.model.rcan import RCAN +.. code-block:: sh -config = Namespace(n_feats=64, n_resblocks=4, n_resgroups=2, reduction=16, scale=[2], data_train='DIV2K', res_scale=1, - n_colors=3, rgb_range=255) -net = RCAN(config) -net.eval() -dummy_input = torch.randn(1, 3, 360, 640) -torch.onnx.export(net, dummy_input, 'RCAN.onnx') -``` -The script generates the ONNX model file `RCAN.onnx`. More information about model parameters (`n_resblocks`, `n_resgroups`, and others) and their different values can be found in the model repository. The model conversion was tested with the commit-SHA: `3339ebc59519c3bb2b5719b87dd36515ec7f3ba7`. + mo --input_model RCAN.onnx -## Converting an ONNX RCAN Model to IR -```sh -mo --input_model RCAN.onnx -``` +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RNNT.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RNNT.md index b3c50255533c9b..822df82fff5c08 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RNNT.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RNNT.md @@ -1,106 +1,127 @@ # Converting a PyTorch RNN-T Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RNNT} -This guide covers conversion of RNN-T model from [MLCommons](https://github.com/mlcommons) repository. Follow +@sphinxdirective + +This guide covers conversion of RNN-T model from `MLCommons `__ repository. Follow the instructions below to export a PyTorch model into ONNX, before converting it to IR: **Step 1**. Clone RNN-T PyTorch implementation from MLCommons repository (revision r1.0). Make a shallow clone to pull only RNN-T model without full repository. If you already have a full repository, skip this and go to **Step 2**: -```bash -git clone -b r1.0 -n https://github.com/mlcommons/inference rnnt_for_openvino --depth 1 -cd rnnt_for_openvino -git checkout HEAD speech_recognition/rnnt -``` + +.. code-block:: sh + + git clone -b r1.0 -n https://github.com/mlcommons/inference rnnt_for_openvino --depth 1 + cd rnnt_for_openvino + git checkout HEAD speech_recognition/rnnt + **Step 2**. If you already have a full clone of MLCommons inference repository, create a folder for pretrained PyTorch model, where conversion into IR will take place. You will also need to specify the path to your full clone at **Step 5**. Skip this step if you have a shallow clone. -```bash -mkdir rnnt_for_openvino -cd rnnt_for_openvino -``` +.. code-block:: sh + + mkdir rnnt_for_openvino + cd rnnt_for_openvino + + +**Step 3**. Download pre-trained weights for PyTorch implementation from `here `__. +For UNIX-like systems, you can use ``wget``: + +.. code-block:: sh + + wget https://zenodo.org/record/3662521/files/DistributedDataParallel_1576581068.9962234-epoch-100.pt + -**Step 3**. Download pretrained weights for PyTorch implementation from [here](https://zenodo.org/record/3662521#.YG21DugzZaQ). -For UNIX-like systems, you can use `wget`: -```bash -wget https://zenodo.org/record/3662521/files/DistributedDataParallel_1576581068.9962234-epoch-100.pt -``` -The link was taken from `setup.sh` in the `speech_recoginitin/rnnt` subfolder. You will get exactly the same weights as -if you were following the [guide](https://github.com/mlcommons/inference/tree/master/speech_recognition/rnnt). +The link was taken from ``setup.sh`` in the ``speech_recoginitin/rnnt`` subfolder. You will get exactly the same weights as +if you were following the `guide `__. **Step 4**. Install required Python packages: -```bash -pip3 install torch toml -``` + +.. code-block:: sh + + pip3 install torch toml + **Step 5**. Export RNN-T model into ONNX, using the script below. Copy the code below into a file named -`export_rnnt_to_onnx.py` and run it in the current directory `rnnt_for_openvino`: - -> **NOTE**: If you already have a full clone of MLCommons inference repository, you need to -> specify the `mlcommons_inference_path` variable. - -```python -import toml -import torch -import sys - - -def load_and_migrate_checkpoint(ckpt_path): - checkpoint = torch.load(ckpt_path, map_location="cpu") - migrated_state_dict = {} - for key, value in checkpoint['state_dict'].items(): - key = key.replace("joint_net", "joint.net") - migrated_state_dict[key] = value - del migrated_state_dict["audio_preprocessor.featurizer.fb"] - del migrated_state_dict["audio_preprocessor.featurizer.window"] - return migrated_state_dict - - -mlcommons_inference_path = './' # specify relative path for MLCommons inferene -checkpoint_path = 'DistributedDataParallel_1576581068.9962234-epoch-100.pt' -config_toml = 'speech_recognition/rnnt/pytorch/configs/rnnt.toml' -config = toml.load(config_toml) -rnnt_vocab = config['labels']['labels'] -sys.path.insert(0, mlcommons_inference_path + 'speech_recognition/rnnt/pytorch') - -from model_separable_rnnt import RNNT - -model = RNNT(config['rnnt'], len(rnnt_vocab) + 1, feature_config=config['input_eval']) -model.load_state_dict(load_and_migrate_checkpoint(checkpoint_path)) - -seq_length, batch_size, feature_length = 157, 1, 240 -inp = torch.randn([seq_length, batch_size, feature_length]) -feature_length = torch.LongTensor([seq_length]) -x_padded, x_lens = model.encoder(inp, feature_length) -torch.onnx.export(model.encoder, (inp, feature_length), "rnnt_encoder.onnx", opset_version=12, - input_names=['input', 'feature_length'], output_names=['x_padded', 'x_lens'], - dynamic_axes={'input': {0: 'seq_len', 1: 'batch'}}) - -symbol = torch.LongTensor([[20]]) -hidden = torch.randn([2, batch_size, 320]), torch.randn([2, batch_size, 320]) -g, hidden = model.prediction.forward(symbol, hidden) -torch.onnx.export(model.prediction, (symbol, hidden), "rnnt_prediction.onnx", opset_version=12, - input_names=['symbol', 'hidden_in_1', 'hidden_in_2'], - output_names=['g', 'hidden_out_1', 'hidden_out_2'], - dynamic_axes={'symbol': {0: 'batch'}, 'hidden_in_1': {1: 'batch'}, 'hidden_in_2': {1: 'batch'}}) - -f = torch.randn([batch_size, 1, 1024]) -model.joint.forward(f, g) -torch.onnx.export(model.joint, (f, g), "rnnt_joint.onnx", opset_version=12, - input_names=['0', '1'], output_names=['result'], dynamic_axes={'0': {0: 'batch'}, '1': {0: 'batch'}}) -``` - -```bash -python3 export_rnnt_to_onnx.py -``` - -After completing this step, the files `rnnt_encoder.onnx`, `rnnt_prediction.onnx`, and `rnnt_joint.onnx` will be saved in the current directory. +``export_rnnt_to_onnx.py`` and run it in the current directory ``rnnt_for_openvino``: + +.. note:: + + If you already have a full clone of MLCommons inference repository, you need + to specify the ``mlcommons_inference_path`` variable. + +.. code-block:: python + + import toml + import torch + import sys + + + def load_and_migrate_checkpoint(ckpt_path): + checkpoint = torch.load(ckpt_path, map_location="cpu") + migrated_state_dict = {} + for key, value in checkpoint['state_dict'].items(): + key = key.replace("joint_net", "joint.net") + migrated_state_dict[key] = value + del migrated_state_dict["audio_preprocessor.featurizer.fb"] + del migrated_state_dict["audio_preprocessor.featurizer.window"] + return migrated_state_dict + + + mlcommons_inference_path = './' # specify relative path for MLCommons inferene + checkpoint_path = 'DistributedDataParallel_1576581068.9962234-epoch-100.pt' + config_toml = 'speech_recognition/rnnt/pytorch/configs/rnnt.toml' + config = toml.load(config_toml) + rnnt_vocab = config['labels']['labels'] + sys.path.insert(0, mlcommons_inference_path + 'speech_recognition/rnnt/pytorch') + + from model_separable_rnnt import RNNT + + model = RNNT(config['rnnt'], len(rnnt_vocab) + 1, feature_config=config['input_eval']) + model.load_state_dict(load_and_migrate_checkpoint(checkpoint_path)) + + seq_length, batch_size, feature_length = 157, 1, 240 + inp = torch.randn([seq_length, batch_size, feature_length]) + feature_length = torch.LongTensor([seq_length]) + x_padded, x_lens = model.encoder(inp, feature_length) + torch.onnx.export(model.encoder, (inp, feature_length), "rnnt_encoder.onnx", opset_version=12, + input_names=['input', 'feature_length'], output_names=['x_padded', 'x_lens'], + dynamic_axes={'input': {0: 'seq_len', 1: 'batch'}}) + + symbol = torch.LongTensor([[20]]) + hidden = torch.randn([2, batch_size, 320]), torch.randn([2, batch_size, 320]) + g, hidden = model.prediction.forward(symbol, hidden) + torch.onnx.export(model.prediction, (symbol, hidden), "rnnt_prediction.onnx", opset_version=12, + input_names=['symbol', 'hidden_in_1', 'hidden_in_2'], + output_names=['g', 'hidden_out_1', 'hidden_out_2'], + dynamic_axes={'symbol': {0: 'batch'}, 'hidden_in_1': {1: 'batch'}, 'hidden_in_2': {1: 'batch'}}) + + f = torch.randn([batch_size, 1, 1024]) + model.joint.forward(f, g) + torch.onnx.export(model.joint, (f, g), "rnnt_joint.onnx", opset_version=12, + input_names=['0', '1'], output_names=['result'], dynamic_axes={'0': {0: 'batch'}, '1': {0: 'batch'}}) + + +.. code-block:: sh + + python3 export_rnnt_to_onnx.py + + +After completing this step, the files ``rnnt_encoder.onnx``, ``rnnt_prediction.onnx``, and ``rnnt_joint.onnx`` will be saved in the current directory. **Step 6**. Run the conversion commands: -```sh -mo --input_model rnnt_encoder.onnx --input "input[157,1,240],feature_length->157" -mo --input_model rnnt_prediction.onnx --input "symbol[1,1],hidden_in_1[2,1,320],hidden_in_2[2,1,320]" -mo --input_model rnnt_joint.onnx --input "0[1,1,1024],1[1,1,320]" -``` -> **NOTE**: The hardcoded value for sequence length = 157 was taken from the MLCommons, but conversion to IR preserves network [reshapeability](@ref openvino_docs_OV_UG_ShapeInference). Therefore, input shapes can be changed manually to any value during either conversion or inference. +.. code-block:: sh + + mo --input_model rnnt_encoder.onnx --input "input[157,1,240],feature_length->157" + mo --input_model rnnt_prediction.onnx --input "symbol[1,1],hidden_in_1[2,1,320],hidden_in_2[2,1,320]" + mo --input_model rnnt_joint.onnx --input "0[1,1,1024],1[1,1,320]" + + +.. note:: + + The hardcoded value for sequence length = 157 was taken from the MLCommons, but conversion to IR preserves network :doc:`reshapeability `. Therefore, input shapes can be changed manually to any value during either conversion or inference. + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_YOLACT.md b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_YOLACT.md index e1526519172794..9ea6aa47ec21c6 100644 --- a/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_YOLACT.md +++ b/docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_YOLACT.md @@ -1,187 +1,212 @@ # Converting a PyTorch YOLACT Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT} +@sphinxdirective + You Only Look At CoefficienTs (YOLACT) is a simple, fully convolutional model for real-time instance segmentation. -The PyTorch implementation is publicly available in [this GitHub repository](https://github.com/dbolya/yolact). +The PyTorch implementation is publicly available in `this GitHub repository `__. The YOLACT++ model is not supported, because it uses deformable convolutional layers that cannot be represented in ONNX format. -## Creating a Patch File +.. _patch-file-yolact: + +Creating a Patch File +##################### Before converting the model, create a patch file for the repository. The patch modifies the framework code by adding a special command-line argument to the framework options. The argument enables inference graph dumping: -1. Go to a writable directory and create a `YOLACT_onnx_export.patch` file. +1. Go to a writable directory and create a ``YOLACT_onnx_export.patch`` file. 2. Copy the following diff code to the file: -```git -From 76deb67d4f09f29feda1a633358caa18335d9e9f Mon Sep 17 00:00:00 2001 -From: "OpenVINO" -Date: Fri, 12 Mar 2021 00:27:35 +0300 -Subject: [PATCH] Add export to ONNX - ---- - eval.py | 5 ++++- - utils/augmentations.py | 7 +++++-- - yolact.py | 29 +++++++++++++++++++---------- - 3 files changed, 28 insertions(+), 13 deletions(-) - -diff --git a/eval.py b/eval.py -index 547bc0a..bde0680 100644 ---- a/eval.py -+++ b/eval.py -@@ -593,9 +593,12 @@ def badhash(x): - return x - - def evalimage(net:Yolact, path:str, save_path:str=None): -- frame = torch.from_numpy(cv2.imread(path)).cuda().float() -+ frame = torch.from_numpy(cv2.imread(path)).float() -+ if torch.cuda.is_available(): -+ frame = frame.cuda() - batch = FastBaseTransform()(frame.unsqueeze(0)) - preds = net(batch) -+ torch.onnx.export(net, batch, "yolact.onnx", opset_version=11) - - img_numpy = prep_display(preds, frame, None, None, undo_transform=False) - -diff --git a/utils/augmentations.py b/utils/augmentations.py -index cc7a73a..2420603 100644 ---- a/utils/augmentations.py -+++ b/utils/augmentations.py -@@ -623,8 +623,11 @@ class FastBaseTransform(torch.nn.Module): - def __init__(self): - super().__init__() - -- self.mean = torch.Tensor(MEANS).float().cuda()[None, :, None, None] -- self.std = torch.Tensor( STD ).float().cuda()[None, :, None, None] -+ self.mean = torch.Tensor(MEANS).float()[None, :, None, None] -+ self.std = torch.Tensor( STD ).float()[None, :, None, None] -+ if torch.cuda.is_available(): -+ self.mean.cuda() -+ self.std.cuda() - self.transform = cfg.backbone.transform - - def forward(self, img): -diff --git a/yolact.py b/yolact.py -index d83703b..f8c787c 100644 ---- a/yolact.py -+++ b/yolact.py -@@ -17,19 +17,22 @@ import torch.backends.cudnn as cudnn - from utils import timer - from utils.functions import MovingAverage, make_net - --# This is required for Pytorch 1.0.1 on Windows to initialize Cuda on some driver versions. --# See the bug report here: https://github.com/pytorch/pytorch/issues/17108 --torch.cuda.current_device() -- --# As of March 10, 2019, Pytorch DataParallel still doesn't support JIT Script Modules --use_jit = torch.cuda.device_count() <= 1 --if not use_jit: -- print('Multiple GPUs detected! Turning off JIT.') -+use_jit = False - - ScriptModuleWrapper = torch.jit.ScriptModule if use_jit else nn.Module - script_method_wrapper = torch.jit.script_method if use_jit else lambda fn, _rcn=None: fn - - -+def decode(loc, priors): -+ variances = [0.1, 0.2] -+ boxes = torch.cat((priors[:, :2] + loc[:, :, :2] * variances[0] * priors[:, 2:], priors[:, 2:] * torch.exp(loc[:, :, 2:] * variances[1])), 2) -+ -+ boxes_result1 = boxes[:, :, :2] - boxes[:, :, 2:] / 2 -+ boxes_result2 = boxes[:, :, 2:] + boxes_result1 -+ boxes_result = torch.cat((boxes_result1, boxes_result2), 2) -+ -+ return boxes_result -+ - - class Concat(nn.Module): - def __init__(self, nets, extra_params): -@@ -476,7 +479,10 @@ class Yolact(nn.Module): - - def load_weights(self, path): - """ Loads weights from a compressed save file. """ -- state_dict = torch.load(path) -+ if torch.cuda.is_available(): -+ state_dict = torch.load(path) -+ else: -+ state_dict = torch.load(path, map_location=torch.device('cpu')) - - # For backward compatability, remove these (the new variable is called layers) - for key in list(state_dict.keys()): -@@ -673,8 +679,11 @@ class Yolact(nn.Module): - else: - pred_outs['conf'] = F.softmax(pred_outs['conf'], -1) - -- return self.detect(pred_outs, self) -+ pred_outs['boxes'] = decode(pred_outs['loc'], pred_outs['priors']) # decode output boxes - -+ pred_outs.pop('priors') # remove unused in postprocessing layers -+ pred_outs.pop('loc') # remove unused in postprocessing layers -+ return pred_outs - - - --- -``` + + .. code-block:: console + + From 76deb67d4f09f29feda1a633358caa18335d9e9f Mon Sep 17 00:00:00 2001 + From: "OpenVINO" + Date: Fri, 12 Mar 2021 00:27:35 +0300 + Subject: [PATCH] Add export to ONNX + + --- + eval.py | 5 ++++- + utils/augmentations.py | 7 +++++-- + yolact.py | 29 +++++++++++++++++++---------- + 3 files changed, 28 insertions(+), 13 deletions(-) + + diff --git a/eval.py b/eval.py + index 547bc0a..bde0680 100644 + --- a/eval.py + +++ b/eval.py + @@ -593,9 +593,12 @@ def badhash(x): + return x + + def evalimage(net:Yolact, path:str, save_path:str=None): + - frame = torch.from_numpy(cv2.imread(path)).cuda().float() + + frame = torch.from_numpy(cv2.imread(path)).float() + + if torch.cuda.is_available(): + + frame = frame.cuda() + batch = FastBaseTransform()(frame.unsqueeze(0)) + preds = net(batch) + + torch.onnx.export(net, batch, "yolact.onnx", opset_version=11) + + img_numpy = prep_display(preds, frame, None, None, undo_transform=False) + + diff --git a/utils/augmentations.py b/utils/augmentations.py + index cc7a73a..2420603 100644 + --- a/utils/augmentations.py + +++ b/utils/augmentations.py + @@ -623,8 +623,11 @@ class FastBaseTransform(torch.nn.Module): + def __init__(self): + super().__init__() + + - self.mean = torch.Tensor(MEANS).float().cuda()[None, :, None, None] + - self.std = torch.Tensor( STD ).float().cuda()[None, :, None, None] + + self.mean = torch.Tensor(MEANS).float()[None, :, None, None] + + self.std = torch.Tensor( STD ).float()[None, :, None, None] + + if torch.cuda.is_available(): + + self.mean.cuda() + + self.std.cuda() + self.transform = cfg.backbone.transform + + def forward(self, img): + diff --git a/yolact.py b/yolact.py + index d83703b..f8c787c 100644 + --- a/yolact.py + +++ b/yolact.py + @@ -17,19 +17,22 @@ import torch.backends.cudnn as cudnn + from utils import timer + from utils.functions import MovingAverage, make_net + + -# This is required for Pytorch 1.0.1 on Windows to initialize Cuda on some driver versions. + -# See the bug report here: https://github.com/pytorch/pytorch/issues/17108 + -torch.cuda.current_device() + - + -# As of March 10, 2019, Pytorch DataParallel still doesn't support JIT Script Modules + -use_jit = torch.cuda.device_count() <= 1 + -if not use_jit: + - print('Multiple GPUs detected! Turning off JIT.') + +use_jit = False + + ScriptModuleWrapper = torch.jit.ScriptModule if use_jit else nn.Module + script_method_wrapper = torch.jit.script_method if use_jit else lambda fn, _rcn=None: fn + + + +def decode(loc, priors): + + variances = [0.1, 0.2] + + boxes = torch.cat((priors[:, :2] + loc[:, :, :2] * variances[0] * priors[:, 2:], priors[:, 2:] * torch.exp(loc[:, :, 2:] * variances[1])), 2) + + + + boxes_result1 = boxes[:, :, :2] - boxes[:, :, 2:] / 2 + + boxes_result2 = boxes[:, :, 2:] + boxes_result1 + + boxes_result = torch.cat((boxes_result1, boxes_result2), 2) + + + + return boxes_result + + + + class Concat(nn.Module): + def __init__(self, nets, extra_params): + @@ -476,7 +479,10 @@ class Yolact(nn.Module): + + def load_weights(self, path): + """ Loads weights from a compressed save file. """ + - state_dict = torch.load(path) + + if torch.cuda.is_available(): + + state_dict = torch.load(path) + + else: + + state_dict = torch.load(path, map_location=torch.device('cpu')) + + # For backward compatability, remove these (the new variable is called layers) + for key in list(state_dict.keys()): + @@ -673,8 +679,11 @@ class Yolact(nn.Module): + else: + pred_outs['conf'] = F.softmax(pred_outs['conf'], -1) + + - return self.detect(pred_outs, self) + + pred_outs['boxes'] = decode(pred_outs['loc'], pred_outs['priors']) # decode output boxes + + + pred_outs.pop('priors') # remove unused in postprocessing layers + + pred_outs.pop('loc') # remove unused in postprocessing layers + + return pred_outs + + + + -- + + 3. Save and close the file. -## Converting a YOLACT Model to the OpenVINO IR format +Converting a YOLACT Model to the OpenVINO IR format +################################################### **Step 1**. Clone the GitHub repository and check out the commit: 1. Clone the YOLACT repository: -```sh -git clone https://github.com/dbolya/yolact -``` + + .. code-block:: sh + + git clone https://github.com/dbolya/yolact + + 2. Check out the necessary commit: -```sh -git checkout 57b8f2d95e62e2e649b382f516ab41f949b57239 -``` -3. Set up the environment as described in `README.md`. + .. code-block:: sh + + git checkout 57b8f2d95e62e2e649b382f516ab41f949b57239 + -**Step 2**. Download a pretrained model from the list attached in the `Evaluation` section of `README.md` document, for example `yolact_base_54_800000.pth`. +3. Set up the environment as described in ``README.md``. + +**Step 2**. Download a pre-trained model from the list attached in the ``Evaluation`` section of ``README.md`` document, for example ``yolact_base_54_800000.pth``. **Step 3**. Export the model to ONNX format. -1. Apply the `YOLACT_onnx_export.patch` patch to the repository. Refer to the Create a Patch File instructions if you do not have it: -```sh -git apply /path/to/patch/YOLACT_onnx_export.patch -``` +1. Apply the `YOLACT_onnx_export.patch` patch to the repository. Refer to the :ref:`Create a Patch File ` instructions if you do not have it: + + .. code-block:: sh + + git apply /path/to/patch/YOLACT_onnx_export.patch + 2. Evaluate the YOLACT model to export it to ONNX format: -```sh -python3 eval.py \ - --trained_model=/path/to/yolact_base_54_800000.pth \ - --score_threshold=0.3 \ - --top_k=10 \ - --image=/path/to/image.jpg \ - --cuda=False -``` + .. code-block:: sh + + python3 eval.py \ + --trained_model=/path/to/yolact_base_54_800000.pth \ + --score_threshold=0.3 \ + --top_k=10 \ + --image=/path/to/image.jpg \ + --cuda=False -3. The script may fail, but you should get `yolact.onnx` file. + +3. The script may fail, but you should get ``yolact.onnx`` file. **Step 4**. Convert the model to the IR: -```sh -mo --input_model /path/to/yolact.onnx -``` + +.. code-block:: sh + + mo --input_model /path/to/yolact.onnx + **Step 4**. Embed input preprocessing into the IR: To get performance gain by offloading to the OpenVINO application of mean/scale values and RGB->BGR conversion, use the following options of the Model Optimizer (MO): * If the backbone of the model is Resnet50-FPN or Resnet101-FPN, use the following MO command line: -```sh -mo \ - --input_model /path/to/yolact.onnx \ - --reverse_input_channels \ - --mean_values "[123.68, 116.78, 103.94]" \ - --scale_values "[58.40, 57.12, 57.38]" -``` + + .. code-block:: sh + + mo \ + --input_model /path/to/yolact.onnx \ + --reverse_input_channels \ + --mean_values "[123.68, 116.78, 103.94]" \ + --scale_values "[58.40, 57.12, 57.38]" + * If the backbone of the model is Darknet53-FPN, use the following MO command line: -```sh -mo \ - --input_model /path/to/yolact.onnx \ - --reverse_input_channels \ - --scale 255 -``` + + .. code-block:: sh + + mo \ + --input_model /path/to/yolact.onnx \ + --reverse_input_channels \ + --scale 255 + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/supported_model_formats.md b/docs/MO_DG/prepare_model/convert_model/supported_model_formats.md index a6b9804b5e8235..05879c7f94c55b 100644 --- a/docs/MO_DG/prepare_model/convert_model/supported_model_formats.md +++ b/docs/MO_DG/prepare_model/convert_model/supported_model_formats.md @@ -15,22 +15,22 @@ openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi openvino_docs_MO_DG_prepare_model_convert_model_tutorials -@endsphinxdirective - **OpenVINO IR (Intermediate Representation)** - the proprietary format of OpenVINO™, benefiting from the full extent of its features. -**ONNX, PaddlePaddle, TensorFlow** - formats supported directly, which means they can be used with OpenVINO Runtime without any prior conversion. For a guide on how to run inference on ONNX, PaddlePaddle, or TensorFlow, see how to [Integrate OpenVINO™ with Your Application](../../../OV_Runtime_UG/integrate_with_your_application.md). +**ONNX, PaddlePaddle, TensorFlow** - formats supported directly, which means they can be used with OpenVINO Runtime without any prior conversion. For a guide on how to run inference on ONNX, PaddlePaddle, or TensorFlow, see how to :doc:`Integrate OpenVINO™ with Your Application `. **MXNet, Caffe, Kaldi** - formats supported indirectly, which means they need to be converted to OpenVINO IR before running inference. The conversion is done with Model Optimizer and in some cases may involve intermediate steps. Refer to the following articles for details on conversion for different formats and models: -* [How to convert ONNX](./Convert_Model_From_ONNX.md) -* [How to convert PaddlePaddle](./Convert_Model_From_Paddle.md) -* [How to convert TensorFlow](./Convert_Model_From_TensorFlow.md) -* [How to convert MXNet](./Convert_Model_From_MxNet.md) -* [How to convert Caffe](./Convert_Model_From_Caffe.md) -* [How to convert Kaldi](./Convert_Model_From_Kaldi.md) +* :doc:`How to convert ONNX ` +* :doc:`How to convert PaddlePaddle ` +* :doc:`How to convert TensorFlow ` +* :doc:`How to convert MXNet ` +* :doc:`How to convert Caffe ` +* :doc:`How to convert Kaldi ` -* [Conversion examples for specific models](./Convert_Model_Tutorials.md) +* :doc:`Conversion examples for specific models ` + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_AttentionOCR_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_AttentionOCR_From_Tensorflow.md index ced208fe3f3231..6465ddc2abb2ba 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_AttentionOCR_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_AttentionOCR_From_Tensorflow.md @@ -1,35 +1,49 @@ # Converting a TensorFlow Attention OCR Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_AttentionOCR_From_Tensorflow} -This tutorial explains how to convert the Attention OCR (AOCR) model from the [TensorFlow Attention OCR repository](https://github.com/emedvedev/attention-ocr) to the Intermediate Representation (IR). +@sphinxdirective -## Extracting a Model from `aocr` Library +This tutorial explains how to convert the Attention OCR (AOCR) model from the `TensorFlow Attention OCR repository `__ to the Intermediate Representation (IR). + +Extracting a Model from ``aocr`` Library +######################################## + +To get an AOCR model, download ``aocr`` Python library: + +.. code-block:: sh + + pip install git+https://github.com/emedvedev/attention-ocr.git@master#egg=aocr -To get an AOCR model, download `aocr` Python library: -``` -pip install git+https://github.com/emedvedev/attention-ocr.git@master#egg=aocr -``` This library contains a pretrained model and allows training and running AOCR, using the command line. After installation of `aocr`, extract the model: -``` -aocr export --format=frozengraph model/path/ -``` -Once extracted, the model can be found in `model/path/` folder. -## Converting the TensorFlow AOCR Model to IR +.. code-block:: sh + + aocr export --format=frozengraph model/path/ + +Once extracted, the model can be found in ``model/path/`` folder. + +Converting the TensorFlow AOCR Model to IR +########################################## The original AOCR model includes the preprocessing data, which contains: + * Decoding input data to binary format where input data is an image represented as a string. * Resizing binary image to working resolution. -The resized image is sent to the convolution neural network (CNN). Because Model Optimizer does not support image decoding, the preprocessing part of the model should be cut off, using the `--input` command-line parameter. -```sh -mo \ ---input_model=model/path/frozen_graph.pb \ ---input="map/TensorArrayStack/TensorArrayGatherV3:0[1,32,86,1]" \ ---output "transpose_1,transpose_2" \ ---output_dir path/to/ir/ -``` +The resized image is sent to the convolution neural network (CNN). Because Model Optimizer does not support image decoding, the preprocessing part of the model should be cut off, using the ``--input`` command-line parameter. + +.. code-block:: sh + + mo \ + --input_model=model/path/frozen_graph.pb \ + --input="map/TensorArrayStack/TensorArrayGatherV3:0[1,32,86,1]" \ + --output "transpose_1,transpose_2" \ + --output_dir path/to/ir/ + Where: -* `map/TensorArrayStack/TensorArrayGatherV3:0[1 32 86 1]` - name of node producing tensor after preprocessing. -* `transpose_1` - name of the node producing tensor with predicted characters. -* `transpose_2` - name of the node producing tensor with predicted characters probabilities. + +* ``map/TensorArrayStack/TensorArrayGatherV3:0[1 32 86 1]`` - name of node producing tensor after preprocessing. +* ``transpose_1`` - name of the node producing tensor with predicted characters. +* ``transpose_2`` - name of the node producing tensor with predicted characters probabilities. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_BERT_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_BERT_From_Tensorflow.md index b0ef95f1dd57c3..6d56a3c1d18047 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_BERT_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_BERT_From_Tensorflow.md @@ -1,79 +1,109 @@ # Converting a TensorFlow BERT Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_BERT_From_Tensorflow} +@sphinxdirective + Pretrained models for BERT (Bidirectional Encoder Representations from Transformers) are -[publicly available](https://github.com/google-research/bert). +`publicly available `__. + +.. _supported_models: + +Supported Models +################ -## Supported Models +The following models from the pretrained `BERT model list `__ are currently supported: -The following models from the pretrained [BERT model list](https://github.com/google-research/bert#pre-trained-models) are currently supported: +* ``BERT-Base, Cased`` +* ``BERT-Base, Uncased`` +* ``BERT-Base, Multilingual Cased`` +* ``BERT-Base, Multilingual Uncased`` +* ``BERT-Base, Chinese`` +* ``BERT-Large, Cased`` +* ``BERT-Large, Uncased`` -* `BERT-Base, Cased` -* `BERT-Base, Uncased` -* `BERT-Base, Multilingual Cased` -* `BERT-Base, Multilingual Uncased` -* `BERT-Base, Chinese` -* `BERT-Large, Cased` -* `BERT-Large, Uncased` +Downloading the Pretrained BERT Model +##################################### -## Downloading the Pretrained BERT Model +Download and unzip an archive with the `BERT-Base, Multilingual Uncased Model `__. -Download and unzip an archive with the [BERT-Base, Multilingual Uncased Model](https://storage.googleapis.com/bert_models/2018_11_03/multilingual_L-12_H-768_A-12.zip). +After the archive is unzipped, the directory ``uncased_L-12_H-768_A-12`` is created and contains the following files: -After the archive is unzipped, the directory `uncased_L-12_H-768_A-12` is created and contains the following files: -* `bert_config.json` -* `bert_model.ckpt.data-00000-of-00001` -* `bert_model.ckpt.index` -* `bert_model.ckpt.meta` -* `vocab.txt` +* ``bert_config.json`` +* ``bert_model.ckpt.data-00000-of-00001`` +* ``bert_model.ckpt.index`` +* ``bert_model.ckpt.meta`` +* ``vocab.txt`` -Pretrained model meta-graph files are `bert_model.ckpt.*`. +Pretrained model meta-graph files are ``bert_model.ckpt.*``. -## Converting a TensorFlow BERT Model to IR +Converting a TensorFlow BERT Model to IR +######################################### To generate the BERT Intermediate Representation (IR) of the model, run Model Optimizer with the following parameters: -```sh - mo \ ---input_meta_graph uncased_L-12_H-768_A-12/bert_model.ckpt.meta \ ---output bert/pooler/dense/Tanh \ ---input Placeholder{i32},Placeholder_1{i32},Placeholder_2{i32} -``` + +.. code-block:: sh + + mo \ + --input_meta_graph uncased_L-12_H-768_A-12/bert_model.ckpt.meta \ + --output bert/pooler/dense/Tanh \ + --input Placeholder{i32},Placeholder_1{i32},Placeholder_2{i32} + Pretrained models are not suitable for batch reshaping out-of-the-box because of multiple hardcoded shapes in the model. -# Converting a Reshapable TensorFlow BERT Model to OpenVINO IR +Converting a Reshapable TensorFlow BERT Model to OpenVINO IR +============================================================= Follow these steps to make a pretrained TensorFlow BERT model reshapable over batch dimension: -1. Download a pretrained BERT model you want to use from the Supported Models list + +1. Download a pretrained BERT model you want to use from the `Supported Models list <#supported_models>`__. + 2. Clone google-research/bert git repository: -```sh -https://github.com/google-research/bert.git -``` -3. Go to the root directory of the cloned repository:
-```sh -cd bert -``` -4. (Optional) Checkout to the commit that the conversion was tested on:
-```sh -git checkout eedf5716c -``` + +.. code-block:: sh + + https://github.com/google-research/bert.git + +3. Go to the root directory of the cloned repository: + +.. code-block:: sh + + cd bert + +4. (Optional) Checkout to the commit that the conversion was tested on: + +.. code-block:: sh + + git checkout eedf5716c + 5. Download script to load GLUE data: - * For UNIX-like systems, run the following command: -```sh -wget https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py -``` - * For Windows systems:
- Download the [Python script](https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py) to the current working directory. + + * For UNIX-like systems, run the following command: + + .. code-block:: sh + + wget https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py + + * For Windows systems: + + Download the `Python script `__ to the current working directory. + 6. Download GLUE data by running: -```sh -python3 download_glue_data.py --tasks MRPC -``` -7. Open the file `modeling.py` in the text editor and delete lines 923-924. They should look like this: -```python + +.. code-block:: sh + + python3 download_glue_data.py --tasks MRPC + +7. Open the file ``modeling.py`` in the text editor and delete lines 923-924. They should look like this: + +.. code-block:: python + if not non_static_indexes: return shape -``` -8. Open the file `run_classifier.py` and insert the following code after the line 645: -```python + +8. Open the file ``run_classifier.py`` and insert the following code after the line 645: + +.. code-block:: python + import os, sys import tensorflow as tf from tensorflow.python.framework import graph_io @@ -86,34 +116,42 @@ python3 download_glue_data.py --tasks MRPC graph_io.write_graph(frozen, './', 'inference_graph.pb', as_text=False) print('BERT frozen model path {}'.format(os.path.join(os.path.dirname(__file__), 'inference_graph.pb'))) sys.exit(0) -``` + Lines before the inserted code should look like this: -```python + +.. code-block:: python + (total_loss, per_example_loss, logits, probabilities) = create_model( bert_config, is_training, input_ids, input_mask, segment_ids, label_ids, num_labels, use_one_hot_embeddings) -``` -9. Set environment variables `BERT_BASE_DIR`, `BERT_REPO_DIR` and run the script `run_classifier.py` to create `inference_graph.pb` file in the root of the cloned BERT repository. -```sh -export BERT_BASE_DIR=/path/to/bert/uncased_L-12_H-768_A-12 -export BERT_REPO_DIR=/current/working/directory - -python3 run_classifier.py \ - --task_name=MRPC \ - --do_eval=true \ - --data_dir=$BERT_REPO_DIR/glue_data/MRPC \ - --vocab_file=$BERT_BASE_DIR/vocab.txt \ - --bert_config_file=$BERT_BASE_DIR/bert_config.json \ - --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ - --output_dir=./ -``` + + +9. Set environment variables ``BERT_BASE_DIR``, ``BERT_REPO_DIR`` and run the script ``run_classifier.py`` to create ``inference_graph.pb`` file in the root of the cloned BERT repository. + +.. code-block:: sh + + export BERT_BASE_DIR=/path/to/bert/uncased_L-12_H-768_A-12 + export BERT_REPO_DIR=/current/working/directory + + python3 run_classifier.py \ + --task_name=MRPC \ + --do_eval=true \ + --data_dir=$BERT_REPO_DIR/glue_data/MRPC \ + --vocab_file=$BERT_BASE_DIR/vocab.txt \ + --bert_config_file=$BERT_BASE_DIR/bert_config.json \ + --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ + --output_dir=./ Run Model Optimizer with the following command line parameters to generate reshape-able BERT Intermediate Representation (IR): -```sh - mo \ ---input_model inference_graph.pb \ ---input "IteratorGetNext:0{i32}[1,128],IteratorGetNext:1{i32}[1,128],IteratorGetNext:4{i32}[1,128]" -``` -For other applicable parameters, refer to the [Convert Model from TensorFlow](../Convert_Model_From_TensorFlow.md) guide. - -For more information about reshape abilities, refer to the [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md) guide. + +.. code-block:: sh + + mo \ + --input_model inference_graph.pb \ + --input "IteratorGetNext:0{i32}[1,128],IteratorGetNext:1{i32}[1,128],IteratorGetNext:4{i32}[1,128]" + +For other applicable parameters, refer to the :doc:`Convert Model from TensorFlow ` guide. + +For more information about reshape abilities, refer to the :doc:`Using Shape Inference ` guide. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_CRNN_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_CRNN_From_Tensorflow.md index 68a399c67922ed..4eee022ef71c74 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_CRNN_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_CRNN_From_Tensorflow.md @@ -1,52 +1,77 @@ # Converting a TensorFlow CRNN Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_CRNN_From_Tensorflow} +@sphinxdirective + This tutorial explains how to convert a CRNN model to OpenVINO™ Intermediate Representation (IR). There are several public versions of TensorFlow CRNN model implementation available on GitHub. This tutorial explains how to convert the model from -the [CRNN Tensorflow](https://github.com/MaybeShewill-CV/CRNN_Tensorflow) repository to IR, and is validated with Python 3.7, TensorFlow 1.15.0, and protobuf 3.19.0. +the `CRNN Tensorflow `__ repository to IR, and is validated with Python 3.7, TensorFlow 1.15.0, and protobuf 3.19.0. If you have another implementation of CRNN model, it can be converted to OpenVINO IR in a similar way. You need to get inference graph and run Model Optimizer on it. **To convert the model to IR:** **Step 1.** Clone this GitHub repository and check out the commit: - 1. Clone the repository: -```sh -git clone https://github.com/MaybeShewill-CV/CRNN_Tensorflow.git -``` - 2. Go to the `CRNN_Tensorflow` directory of the cloned repository: -```sh -cd path/to/CRNN_Tensorflow -``` - 3. Check out the necessary commit: -```sh -git checkout 64f1f1867bffaacfeacc7a80eebf5834a5726122 -``` + +1. Clone the repository: + +.. code-block:: sh + + git clone https://github.com/MaybeShewill-CV/CRNN_Tensorflow.git + +2. Go to the ``CRNN_Tensorflow`` directory of the cloned repository: + +.. code-block:: sh + + cd path/to/CRNN_Tensorflow + +3. Check out the necessary commit: + +.. code-block:: sh + + git checkout 64f1f1867bffaacfeacc7a80eebf5834a5726122 + **Step 2.** Train the model using the framework or the pretrained checkpoint provided in this repository. + **Step 3.** Create an inference graph: - 1. Add the `CRNN_Tensorflow` folder to `PYTHONPATH`. - * For Linux: -```sh -export PYTHONPATH="${PYTHONPATH}:/path/to/CRNN_Tensorflow/" -``` - * For Windows, add `/path/to/CRNN_Tensorflow/` to the `PYTHONPATH` environment variable in settings. - 2. Edit the `tools/demo_shadownet.py` script. After `saver.restore(sess=sess, save_path=weights_path)` line, add the following code: -```python -from tensorflow.python.framework import graph_io -frozen = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['shadow/LSTMLayers/transpose_time_major']) -graph_io.write_graph(frozen, '.', 'frozen_graph.pb', as_text=False) -``` - 3. Run the demo with the following command: -```sh -python tools/demo_shadownet.py --image_path data/test_images/test_01.jpg --weights_path model/shadownet/shadownet_2017-10-17-11-47-46.ckpt-199999 -``` - If you want to use your checkpoint, replace the path in the `--weights_path` parameter with a path to your checkpoint. - 4. In the `CRNN_Tensorflow` directory, you will find the inference CRNN graph `frozen_graph.pb`. You can use this graph with OpenVINO - to convert the model to IR and then run inference. + +1. Add the ``CRNN_Tensorflow`` folder to ``PYTHONPATH``. + + * For Linux: + + .. code-block:: sh + + export PYTHONPATH="${PYTHONPATH}:/path/to/CRNN_Tensorflow/" + + + * For Windows, add ``/path/to/CRNN_Tensorflow/`` to the ``PYTHONPATH`` environment variable in settings. + + + +2. Edit the ``tools/demo_shadownet.py`` script. After ``saver.restore(sess=sess, save_path=weights_path)`` line, add the following code: + +.. code-block:: python + + from tensorflow.python.framework import graph_io + frozen = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['shadow/LSTMLayers/transpose_time_major']) + graph_io.write_graph(frozen, '.', 'frozen_graph.pb', as_text=False) + +3. Run the demo with the following command: + +.. code-block:: sh + + python tools/demo_shadownet.py --image_path data/test_images/test_01.jpg --weights_path model/shadownet/shadownet_2017-10-17-11-47-46.ckpt-199999 + + +If you want to use your checkpoint, replace the path in the ``--weights_path`` parameter with a path to your checkpoint. + +4. In the ``CRNN_Tensorflow`` directory, you will find the inference CRNN graph ``frozen_graph.pb``. You can use this graph with OpenVINO to convert the model to IR and then run inference. **Step 4.** Convert the model to IR: -```sh -mo --input_model path/to/your/CRNN_Tensorflow/frozen_graph.pb -``` +.. code-block:: sh + + mo --input_model path/to/your/CRNN_Tensorflow/frozen_graph.pb + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_DeepSpeech_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_DeepSpeech_From_Tensorflow.md index 24bc0b35d34ff9..f024ec5465ec38 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_DeepSpeech_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_DeepSpeech_From_Tensorflow.md @@ -1,81 +1,99 @@ # Converting a TensorFlow DeepSpeech Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_DeepSpeech_From_Tensorflow} -[DeepSpeech project](https://github.com/mozilla/DeepSpeech) provides an engine to train speech-to-text models. +@sphinxdirective -## Downloading the Pretrained DeepSpeech Model +`DeepSpeech project `__ provides an engine to train speech-to-text models. + +Downloading the Pretrained DeepSpeech Model +########################################### Create a directory where model and metagraph with pretrained weights will be stored: -``` -mkdir deepspeech -cd deepspeech -``` -[Pretrained English speech-to-text model](https://github.com/mozilla/DeepSpeech/releases/tag/v0.8.2) is publicly available. + +.. code-block:: sh + + mkdir deepspeech + cd deepspeech + +`Pre-trained English speech-to-text model `__ is publicly available. To download the model, follow the instruction below: * For UNIX-like systems, run the following command: -``` -wget -O - https://github.com/mozilla/DeepSpeech/archive/v0.8.2.tar.gz | tar xvfz - -wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.8.2/deepspeech-0.8.2-checkpoint.tar.gz | tar xvfz - -``` + +.. code-block:: sh + + wget -O - https://github.com/mozilla/DeepSpeech/archive/v0.8.2.tar.gz | tar xvfz - + wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.8.2/deepspeech-0.8.2-checkpoint.tar.gz | tar xvfz - + * For Windows systems: - 1. Download [the archive with the model](https://github.com/mozilla/DeepSpeech/archive/v0.8.2.tar.gz). - 2. Download the [TensorFlow MetaGraph with pretrained weights](https://github.com/mozilla/DeepSpeech/releases/download/v0.8.2/deepspeech-0.8.2-checkpoint.tar.gz). + + 1. Download `the archive with the model `__. + 2. Download the `TensorFlow MetaGraph with pre-trained weights `__. 3. Unpack it with a file archiver application. -## Freezing the Model into a *.pb File +Freezing the Model into a *.pb File +################################### After unpacking the archives above, you have to freeze the model. This requires TensorFlow version 1, which is not available under Python 3.8, so you need Python 3.7 or lower. Before freezing, deploy a virtual environment and install the required packages: -``` -virtualenv --python=python3.7 venv-deep-speech -source venv-deep-speech/bin/activate -cd DeepSpeech-0.8.2 -pip3 install -e . -``` + +.. code-block:: sh + + virtualenv --python=python3.7 venv-deep-speech + source venv-deep-speech/bin/activate + cd DeepSpeech-0.8.2 + pip3 install -e . + Freeze the model with the following command: -``` -python3 DeepSpeech.py --checkpoint_dir ../deepspeech-0.8.2-checkpoint --export_dir ../ -``` -After that, you will get the pretrained frozen model file `output_graph.pb` in the directory `deepspeech` created at + +.. code-block:: sh + + python3 DeepSpeech.py --checkpoint_dir ../deepspeech-0.8.2-checkpoint --export_dir ../ + +After that, you will get the pretrained frozen model file ``output_graph.pb`` in the directory ``deepspeech`` created at the beginning. The model contains the preprocessing and main parts. The first preprocessing part performs conversion of input spectrogram into a form useful for speech recognition (mel). This part of the model is not convertible into -the IR because it contains unsupported operations `AudioSpectrogram` and `Mfcc`. +the IR because it contains unsupported operations ``AudioSpectrogram`` and ``Mfcc``. The main and most computationally expensive part of the model converts the preprocessed audio into text. There are two specificities with the supported part of the model. The first is that the model contains an input with sequence length. So the model can be converted with a fixed input length shape, thus the model is not reshapable. -Refer to the [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md) guide. +Refer to the :doc:`Using Shape Inference ` guide. -The second is that the frozen model still has two variables: `previous_state_c` and `previous_state_h`, figure +The second is that the frozen model still has two variables: ``previous_state_c`` and ``previous_state_h``, figure with the frozen *.pb model is below. It means that the model keeps training these variables at each inference. -![DeepSpeech model view](../../../img/DeepSpeech-0.8.2.png) +.. image:: ./_static/images/DeepSpeech-0.8.2.png -At the first inference, the variables are initialized with zero tensors. After execution, the results of the `BlockLSTM' +At the first inference, the variables are initialized with zero tensors. After execution, the results of the ``BlockLSTM`` are assigned to cell state and hidden state, which are these two variables. -## Converting the Main Part of DeepSpeech Model into OpenVINO IR +Converting the Main Part of DeepSpeech Model into OpenVINO IR +############################################################# -Model Optimizer assumes that the output model is for inference only. That is why you should cut `previous_state_c` -and `previous_state_h` variables off and resolve keeping cell and hidden states on the application level. +Model Optimizer assumes that the output model is for inference only. That is why you should cut ``previous_state_c`` and ``previous_state_h`` variables off and resolve keeping cell and hidden states on the application level. There are certain limitations for the model conversion: -- Time length (`time_len`) and sequence length (`seq_len`) are equal. -- Original model cannot be reshaped, so you should keep original shapes. + +* Time length (``time_len``) and sequence length (``seq_len``) are equal. +* Original model cannot be reshaped, so you should keep original shapes. To generate the IR, run Model Optimizer with the following parameters: -```sh -mo \ ---input_model output_graph.pb \ ---input "input_lengths->[16],input_node[1,16,19,26],previous_state_h[1,2048],previous_state_c[1,2048]" \ ---output "cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/GatherNd_1,cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/GatherNd,logits" -``` + +.. code-block:: sh + + mo \ + --input_model output_graph.pb \ + --input "input_lengths->[16],input_node[1,16,19,26],previous_state_h[1,2048],previous_state_c[1,2048]" \ + --output "cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/GatherNd_1,cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/GatherNd,logits" + Where: -* `input_lengths->[16]` Replaces the input node with name "input_lengths" with a constant tensor of shape [1] with a - single integer value of 16. This means that the model now can consume input sequences of length 16 only. -* `input_node[1 16 19 26],previous_state_h[1 2048],previous_state_c[1 2048]` replaces the variables with a placeholder. -* `--output ".../GatherNd_1,.../GatherNd,logits" ` output node names. + +* ``input_lengths->[16]`` Replaces the input node with name "input_lengths" with a constant tensor of shape [1] with a single integer value of 16. This means that the model now can consume input sequences of length 16 only. +* ``input_node[1 16 19 26],previous_state_h[1 2048],previous_state_c[1 2048]`` replaces the variables with a placeholder. +* ``--output ".../GatherNd_1,.../GatherNd,logits"`` output node names. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_EfficientDet_Models.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_EfficientDet_Models.md index 7c28307fa4e4f9..fca2f79570e651 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_EfficientDet_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_EfficientDet_Models.md @@ -1,93 +1,74 @@ # Converting TensorFlow EfficientDet Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models} +@sphinxdirective + This tutorial explains how to convert EfficientDet public object detection models to the Intermediate Representation (IR). -## Converting EfficientDet Model to the IR +.. _efficientdet-to-ir: + +Converting EfficientDet Model to the IR +####################################### There are several public versions of EfficientDet model implementation available on GitHub. This tutorial explains how to -convert models from the [repository](https://github.com/google/automl/tree/master/efficientdet) - (commit 96e1fee) to the OpenVINO format. - -### Getting a Frozen TensorFlow Model - -Follow the instructions below to get frozen TensorFlow EfficientDet model. EfficientDet-D4 model is an example: - -1. Clone the repository:
-```sh -git clone https://github.com/google/automl -cd automl/efficientdet -``` -2. (Optional) Checkout to the commit that the conversion was tested on:
-```sh -git checkout 96e1fee -``` -3. Install required dependencies:
-```sh -python3 -m pip install --upgrade pip -python3 -m pip install -r requirements.txt -python3 -m pip install --upgrade tensorflow-model-optimization -``` -4. Download and extract the model checkpoint [efficientdet-d4.tar.gz](https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco2/efficientdet-d4.tar.gz) -referenced in the **"Pretrained EfficientDet Checkpoints"** section of the model repository:
-```sh -wget https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco2/efficientdet-d4.tar.gz -tar zxvf efficientdet-d4.tar.gz -``` -5. Freeze the model:
-```sh - mo --runmode=saved_model --model_name=efficientdet-d4 --ckpt_path=efficientdet-d4 --saved_model_dir=savedmodeldir -``` -As a result, the frozen model file `savedmodeldir/efficientdet-d4_frozen.pb` will be generated. - -> **NOTE**: For custom trained models, specify `--hparams` flag to `config.yaml` which was used during training. - -> **NOTE**: If you see an error *AttributeError: module 'tensorflow_core.python.keras.api._v2.keras.initializers' has no attribute 'variance_scaling'*, apply the fix from the [patch](https://github.com/google/automl/pull/846). - -### Converting an EfficientDet TensorFlow Model to the IR - -To generate the IR of the EfficientDet TensorFlow model, run:
-```sh -mo \ ---input_model savedmodeldir/efficientdet-d4_frozen.pb \ ---transformations_config front/tf/automl_efficientdet.json \ ---input_shape [1,$IMAGE_SIZE,$IMAGE_SIZE,3] \ ---reverse_input_channels -``` - -Where `$IMAGE_SIZE` is the size that the input image of the original TensorFlow model will be resized to. Different -EfficientDet models were trained with different input image sizes. To determine the right one, refer to the `efficientdet_model_param_dict` -dictionary in the [hparams_config.py](https://github.com/google/automl/blob/96e1fee/efficientdet/hparams_config.py#L304) file. -The attribute `image_size` specifies the shape to be defined for the model conversion. - -The `transformations_config` command line parameter specifies the configuration json file containing hints -for the Model Optimizer on how to convert the model and trigger transformations implemented in the -`/openvino/tools/mo/front/tf/AutomlEfficientDet.py`. The json file contains some parameters which must be changed if you -train the model yourself and modified the `hparams_config` file or the parameters are different from the ones used for EfficientDet-D4. -The attribute names are self-explanatory or match the name in the `hparams_config` file. - -> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the [Converting a Model to Intermediate Representation (IR)](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. +convert models from the `repository `__ (commit 96e1fee) to the OpenVINO format. + +Download and extract the model checkpoint `efficientdet-d4.tar.gz `__ +referenced in the **"Pretrained EfficientDet Checkpoints"** section of the model repository: + +.. code-block:: sh + + wget https://storage.googleapis.com/cloud-tpu-checkpoints/efficientdet/coco2/efficientdet-d4.tar.gz + tar zxvf efficientdet-d4.tar.gz + +Converting an EfficientDet TensorFlow Model to the IR ++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +To generate the IR of the EfficientDet TensorFlow model, run: + +.. code-block:: sh + + mo \ + --input_meta_graph efficientdet-d4/model.meta \ + --input_shape [1,$IMAGE_SIZE,$IMAGE_SIZE,3] \ + --reverse_input_channels + + +Where ``$IMAGE_SIZE`` is the size that the input image of the original TensorFlow model will be resized to. Different +EfficientDet models were trained with different input image sizes. To determine the right one, refer to the ``efficientdet_model_param_dict`` +dictionary in the `hparams_config.py `__ file. +The attribute ``image_size`` specifies the shape to be defined for the model conversion. + +.. note:: + + The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the ``RGB<->BGR`` conversion specifying the command-line parameter: ``--reverse_input_channels``. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the :doc:`Converting a Model to Intermediate Representation (IR) ` guide. OpenVINO toolkit provides samples that can be used to infer EfficientDet model. -For more information, refer to the [Open Model Zoo Demos](@ref omz_demos). - -## Interpreting Results of the TensorFlow Model and the IR - -The TensorFlow model produces as output a list of 7-element tuples: `[image_id, y_min, x_min, y_max, x_max, confidence, class_id]`, where: -* `image_id` -- image batch index. -* `y_min` -- absolute `y` coordinate of the lower left corner of the detected object. -* `x_min` -- absolute `x` coordinate of the lower left corner of the detected object. -* `y_max` -- absolute `y` coordinate of the upper right corner of the detected object. -* `x_max` -- absolute `x` coordinate of the upper right corner of the detected object. -* `confidence` -- is the confidence of the detected object. -* `class_id` -- is the id of the detected object class counted from 1. - -The output of the IR is a list of 7-element tuples: `[image_id, class_id, confidence, x_min, y_min, x_max, y_max]`, where: -* `image_id` -- image batch index. -* `class_id` -- is the id of the detected object class counted from 0. -* `confidence` -- is the confidence of the detected object. -* `x_min` -- normalized `x` coordinate of the lower left corner of the detected object. -* `y_min` -- normalized `y` coordinate of the lower left corner of the detected object. -* `x_max` -- normalized `x` coordinate of the upper right corner of the detected object. -* `y_max` -- normalized `y` coordinate of the upper right corner of the detected object. - -The first element with `image_id = -1` means end of data. +For more information, refer to the :doc:`Open Model Zoo Demos `. + +Interpreting Results of the TensorFlow Model and the IR +####################################################### + +The TensorFlow model produces as output a list of 7-element tuples: ``[image_id, y_min, x_min, y_max, x_max, confidence, class_id]``, where: + +* ``image_id`` -- image batch index. +* ``y_min`` -- absolute ``y`` coordinate of the lower left corner of the detected object. +* ``x_min`` -- absolute ``x`` coordinate of the lower left corner of the detected object. +* ``y_max`` -- absolute ``y`` coordinate of the upper right corner of the detected object. +* ``x_max`` -- absolute ``x`` coordinate of the upper right corner of the detected object. +* ``confidence`` -- the confidence of the detected object. +* ``class_id`` -- the id of the detected object class counted from 1. + +The output of the IR is a list of 7-element tuples: ``[image_id, class_id, confidence, x_min, y_min, x_max, y_max]``, where: + +* ``image_id`` -- image batch index. +* ``class_id`` -- the id of the detected object class counted from 0. +* ``confidence`` -- the confidence of the detected object. +* ``x_min`` -- normalized ``x`` coordinate of the lower left corner of the detected object. +* ``y_min`` -- normalized ``y`` coordinate of the lower left corner of the detected object. +* ``x_max`` -- normalized ``x`` coordinate of the upper right corner of the detected object. +* ``y_max`` -- normalized ``y`` coordinate of the upper right corner of the detected object. + +The first element with ``image_id = -1`` means end of data. + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_FaceNet_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_FaceNet_From_Tensorflow.md index 5e987b1bcd1a79..aa7b3056c8bb1e 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_FaceNet_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_FaceNet_From_Tensorflow.md @@ -1,28 +1,33 @@ # Converting TensorFlow FaceNet Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow} -[Public pretrained FaceNet models](https://github.com/davidsandberg/facenet#pre-trained-models) contain both training +@sphinxdirective + +`Public pre-trained FaceNet models `__ contain both training and inference part of graph. Switch between this two states is manageable with placeholder value. Intermediate Representation (IR) models are intended for inference, which means that train part is redundant. -There are two inputs in this network: boolean `phase_train` which manages state of the graph (train/infer) and -`batch_size` which is a part of batch joining pattern. - +There are two inputs in this network: boolean ``phase_train`` which manages state of the graph (train/infer) and +``batch_size`` which is a part of batch joining pattern. -![FaceNet model view](../../../img/FaceNet.svg) +.. image:: ./_static/images/FaceNet.svg -## Converting a TensorFlow FaceNet Model to the IR +Converting a TensorFlow FaceNet Model to the IR +############################################### To generate a FaceNet OpenVINO model, feed a TensorFlow FaceNet model to Model Optimizer with the following parameters: -```sh - mo ---input_model path_to_model/model_name.pb \ ---freeze_placeholder_with_value "phase_train->False" -``` - -The batch joining pattern transforms to a placeholder with the model default shape if `--input_shape` or `--batch`*/*`-b` are not -provided. Otherwise, the placeholder shape has custom parameters. - -* `--freeze_placeholder_with_value "phase_train->False"` to switch graph to inference mode -* `--batch`*/*`-b` is applicable to override original network batch -* `--input_shape` is applicable with or without `--input` + +.. code-block:: sh + + mo + --input_model path_to_model/model_name.pb \ + --freeze_placeholder_with_value "phase_train->False" + + +The batch joining pattern transforms to a placeholder with the model default shape if ``--input_shape`` or ``--batch`*/*`-b`` are not provided. Otherwise, the placeholder shape has custom parameters. + +* ``--freeze_placeholder_with_value "phase_train->False"`` to switch graph to inference mode +* ``--batch`*/*`-b`` is applicable to override original network batch +* ``--input_shape`` is applicable with or without ``--input`` * other options are applicable + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md index 6f984f53b1dbc5..0cb950cf94a97d 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md @@ -1,276 +1,304 @@ # Converting a TensorFlow GNMT Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_GNMT_From_Tensorflow} +@sphinxdirective + This tutorial explains how to convert Google Neural Machine Translation (GNMT) model to the Intermediate Representation (IR). -There are several public versions of TensorFlow GNMT model implementation available on GitHub. This tutorial explains how to convert the GNMT model from the [TensorFlow Neural Machine Translation (NMT) repository](https://github.com/tensorflow/nmt) to the IR. +There are several public versions of TensorFlow GNMT model implementation available on GitHub. This tutorial explains how to convert the GNMT model from the `TensorFlow Neural Machine Translation (NMT) repository `__ to the IR. -## Creating a Patch File +Creating a Patch File +##################### Before converting the model, you need to create a patch file for the repository. The patch modifies the framework code by adding a special command-line argument to the framework options that enables inference graph dumping: -1. Go to a writable directory and create a `GNMT_inference.patch` file. +1. Go to a writable directory and create a ``GNMT_inference.patch`` file. 2. Copy the following diff code to the file: -```git -diff --git a/nmt/inference.py b/nmt/inference.py -index 2cbef07..e185490 100644 ---- a/nmt/inference.py -+++ b/nmt/inference.py -@@ -17,9 +17,11 @@ - from __future__ import print_function - - import codecs -+import os - import time - - import tensorflow as tf -+from tensorflow.python.framework import graph_io - - from . import attention_model - from . import gnmt_model -@@ -105,6 +107,29 @@ def start_sess_and_load_model(infer_model, ckpt_path): - return sess, loaded_infer_model - - -+def inference_dump_graph(ckpt_path, path_to_dump, hparams, scope=None): -+ model_creator = get_model_creator(hparams) -+ infer_model = model_helper.create_infer_model(model_creator, hparams, scope) -+ sess = tf.Session( -+ graph=infer_model.graph, config=utils.get_config_proto()) -+ with infer_model.graph.as_default(): -+ loaded_infer_model = model_helper.load_model( -+ infer_model.model, ckpt_path, sess, "infer") -+ utils.print_out("Dumping inference graph to {}".format(path_to_dump)) -+ loaded_infer_model.saver.save( -+ sess, -+ os.path.join(path_to_dump + 'inference_GNMT_graph') -+ ) -+ utils.print_out("Dumping done!") -+ -+ output_node_name = 'index_to_string_Lookup' -+ utils.print_out("Freezing GNMT graph with output node {}...".format(output_node_name)) -+ frozen = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, -+ [output_node_name]) -+ graph_io.write_graph(frozen, '.', os.path.join(path_to_dump, 'frozen_GNMT_inference_graph.pb'), as_text=False) -+ utils.print_out("Freezing done. Freezed model frozen_GNMT_inference_graph.pb saved to {}".format(path_to_dump)) -+ -+ - def inference(ckpt_path, - inference_input_file, - inference_output_file, -diff --git a/nmt/nmt.py b/nmt/nmt.py -index f5823d8..a733748 100644 ---- a/nmt/nmt.py -+++ b/nmt/nmt.py -@@ -310,6 +310,13 @@ def add_arguments(parser): - parser.add_argument("--num_intra_threads", type=int, default=0, - help="number of intra_op_parallelism_threads") - -+ # Special argument for inference model dumping without inference -+ parser.add_argument("--dump_inference_model", type="bool", nargs="?", -+ const=True, default=False, -+ help="Argument for dump inference graph for specified trained ckpt") -+ -+ parser.add_argument("--path_to_dump", type=str, default="", -+ help="Path to dump inference graph.") - - def create_hparams(flags): - """Create training hparams.""" -@@ -396,6 +403,9 @@ def create_hparams(flags): - language_model=flags.language_model, - num_intra_threads=flags.num_intra_threads, - num_inter_threads=flags.num_inter_threads, -+ -+ dump_inference_model=flags.dump_inference_model, -+ path_to_dump=flags.path_to_dump, - ) - - -@@ -613,7 +623,7 @@ def create_or_load_hparams( - return hparams - - --def run_main(flags, default_hparams, train_fn, inference_fn, target_session=""): -+def run_main(flags, default_hparams, train_fn, inference_fn, inference_dump, target_session=""): - """Run main.""" - # Job - jobid = flags.jobid -@@ -653,8 +663,26 @@ def run_main(flags, default_hparams, train_fn, inference_fn, target_session=""): - out_dir, default_hparams, flags.hparams_path, - save_hparams=(jobid == 0)) - -- ## Train / Decode -- if flags.inference_input_file: -+ # Dumping inference model -+ if flags.dump_inference_model: -+ # Inference indices -+ hparams.inference_indices = None -+ if flags.inference_list: -+ (hparams.inference_indices) = ( -+ [int(token) for token in flags.inference_list.split(",")]) -+ -+ # Ckpt -+ ckpt = flags.ckpt -+ if not ckpt: -+ ckpt = tf.train.latest_checkpoint(out_dir) -+ -+ # Path to dump graph -+ assert flags.path_to_dump != "", "Please, specify path_to_dump model." -+ path_to_dump = flags.path_to_dump -+ if not tf.gfile.Exists(path_to_dump): tf.gfile.MakeDirs(path_to_dump) -+ -+ inference_dump(ckpt, path_to_dump, hparams) -+ elif flags.inference_input_file: - # Inference output directory - trans_file = flags.inference_output_file - assert trans_file -@@ -693,7 +721,8 @@ def main(unused_argv): - default_hparams = create_hparams(FLAGS) - train_fn = train.train - inference_fn = inference.inference -- run_main(FLAGS, default_hparams, train_fn, inference_fn) -+ inference_dump = inference.inference_dump_graph -+ run_main(FLAGS, default_hparams, train_fn, inference_fn, inference_dump) - - - if __name__ == "__main__": - -``` + +.. code-block:: cpp + + diff --git a/nmt/inference.py b/nmt/inference.py + index 2cbef07..e185490 100644 + --- a/nmt/inference.py + +++ b/nmt/inference.py + @@ -17,9 +17,11 @@ + from __future__ import print_function + + import codecs + +import os + import time + + import tensorflow as tf + +from tensorflow.python.framework import graph_io + + from . import attention_model + from . import gnmt_model + @@ -105,6 +107,29 @@ def start_sess_and_load_model(infer_model, ckpt_path): + return sess, loaded_infer_model + + + +def inference_dump_graph(ckpt_path, path_to_dump, hparams, scope=None): + + model_creator = get_model_creator(hparams) + + infer_model = model_helper.create_infer_model(model_creator, hparams, scope) + + sess = tf.Session( + + graph=infer_model.graph, config=utils.get_config_proto()) + + with infer_model.graph.as_default(): + + loaded_infer_model = model_helper.load_model( + + infer_model.model, ckpt_path, sess, "infer") + + utils.print_out("Dumping inference graph to {}".format(path_to_dump)) + + loaded_infer_model.saver.save( + + sess, + + os.path.join(path_to_dump + 'inference_GNMT_graph') + + ) + + utils.print_out("Dumping done!") + + + + output_node_name = 'index_to_string_Lookup' + + utils.print_out("Freezing GNMT graph with output node {}...".format(output_node_name)) + + frozen = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, + + [output_node_name]) + + graph_io.write_graph(frozen, '.', os.path.join(path_to_dump, 'frozen_GNMT_inference_graph.pb'), as_text=False) + + utils.print_out("Freezing done. Freezed model frozen_GNMT_inference_graph.pb saved to {}".format(path_to_dump)) + + + + + def inference(ckpt_path, + inference_input_file, + inference_output_file, + diff --git a/nmt/nmt.py b/nmt/nmt.py + index f5823d8..a733748 100644 + --- a/nmt/nmt.py + +++ b/nmt/nmt.py + @@ -310,6 +310,13 @@ def add_arguments(parser): + parser.add_argument("--num_intra_threads", type=int, default=0, + help="number of intra_op_parallelism_threads") + + + # Special argument for inference model dumping without inference + + parser.add_argument("--dump_inference_model", type="bool", nargs="?", + + const=True, default=False, + + help="Argument for dump inference graph for specified trained ckpt") + + + + parser.add_argument("--path_to_dump", type=str, default="", + + help="Path to dump inference graph.") + + def create_hparams(flags): + """Create training hparams.""" + @@ -396,6 +403,9 @@ def create_hparams(flags): + language_model=flags.language_model, + num_intra_threads=flags.num_intra_threads, + num_inter_threads=flags.num_inter_threads, + + + + dump_inference_model=flags.dump_inference_model, + + path_to_dump=flags.path_to_dump, + ) + + + @@ -613,7 +623,7 @@ def create_or_load_hparams( + return hparams + + + -def run_main(flags, default_hparams, train_fn, inference_fn, target_session=""): + +def run_main(flags, default_hparams, train_fn, inference_fn, inference_dump, target_session=""): + """Run main.""" + # Job + jobid = flags.jobid + @@ -653,8 +663,26 @@ def run_main(flags, default_hparams, train_fn, inference_fn, target_session=""): + out_dir, default_hparams, flags.hparams_path, + save_hparams=(jobid == 0)) + + - ## Train / Decode + - if flags.inference_input_file: + + # Dumping inference model + + if flags.dump_inference_model: + + # Inference indices + + hparams.inference_indices = None + + if flags.inference_list: + + (hparams.inference_indices) = ( + + [int(token) for token in flags.inference_list.split(",")]) + + + + # Ckpt + + ckpt = flags.ckpt + + if not ckpt: + + ckpt = tf.train.latest_checkpoint(out_dir) + + + + # Path to dump graph + + assert flags.path_to_dump != "", "Please, specify path_to_dump model." + + path_to_dump = flags.path_to_dump + + if not tf.gfile.Exists(path_to_dump): tf.gfile.MakeDirs(path_to_dump) + + + + inference_dump(ckpt, path_to_dump, hparams) + + elif flags.inference_input_file: + # Inference output directory + trans_file = flags.inference_output_file + assert trans_file + @@ -693,7 +721,8 @@ def main(unused_argv): + default_hparams = create_hparams(FLAGS) + train_fn = train.train + inference_fn = inference.inference + - run_main(FLAGS, default_hparams, train_fn, inference_fn) + + inference_dump = inference.inference_dump_graph + + run_main(FLAGS, default_hparams, train_fn, inference_fn, inference_dump) + + + if __name__ == "__main__": + + 3. Save and close the file. -## Converting a GNMT Model to the IR +Converting a GNMT Model to the IR +################################# -> **NOTE**: Use TensorFlow version 1.13 or lower. +.. note:: Use TensorFlow version 1.13 or lower. **Step 1**. Clone the GitHub repository and check out the commit: 1. Clone the NMT reposirory: -```sh -git clone https://github.com/tensorflow/nmt.git -``` + +.. code-block:: sh + + git clone https://github.com/tensorflow/nmt.git + 2. Check out the necessary commit: -```sh -git checkout b278487980832417ad8ac701c672b5c3dc7fa553 -``` + +.. code-block:: sh + + git checkout b278487980832417ad8ac701c672b5c3dc7fa553 + **Step 2**. Get a trained model. You have two options: -* Train the model with the GNMT `wmt16_gnmt_4_layer.json` or `wmt16_gnmt_8_layer.json` configuration file using the NMT framework. +* Train the model with the GNMT ``wmt16_gnmt_4_layer.json`` or ``wmt16_gnmt_8_layer.json`` configuration file using the NMT framework. * *Do not use the pre-trained checkpoints provided in the NMT repository, as they are outdated and can be incompatible with the current repository version.* -This tutorial assumes the use of the trained GNMT model from `wmt16_gnmt_4_layer.json` config, German to English translation. +This tutorial assumes the use of the trained GNMT model from ``wmt16_gnmt_4_layer.json`` config, German to English translation. **Step 3**. Create an inference graph: The OpenVINO assumes that a model is used for inference only. Hence, before converting the model into the IR, you need to transform the training graph into the inference graph. For the GNMT model, the training graph and the inference graph have different decoders: the training graph uses a greedy search decoding algorithm, while the inference graph uses a beam search decoding algorithm. -1. Apply the `GNMT_inference.patch` patch to the repository. Refer to the Create a Patch File instructions if you do not have it: -```sh - git apply /path/to/patch/GNMT_inference.patch -``` +1. Apply the ``GNMT_inference.patch`` patch to the repository. `Create a Patch File <#Creating-a-Patch-File>`__ instructions if you do not have it: + +.. code-block:: sh + + git apply /path/to/patch/GNMT_inference.patch + 2. Run the NMT framework to dump the inference model: -```sh -python -m nmt.nmt - --src=de - --tgt=en - --ckpt=/path/to/ckpt/translate.ckpt - --hparams_path=/path/to/repository/nmt/nmt/standard_hparams/wmt16_gnmt_4_layer.json - --vocab_prefix=/path/to/vocab/vocab.bpe.32000 - --out_dir="" - --dump_inference_model - --infer_mode beam_search - --path_to_dump /path/to/dump/model/ -``` - -If you use different checkpoints, use the corresponding values for the `src`,`tgt`,`ckpt`,`hparams_path`, and `vocab_prefix` parameters. -Inference checkpoint `inference_GNMT_graph` and frozen inference graph `frozen_GNMT_inference_graph.pb` will appear in the `/path/to/dump/model/` folder. - -To generate `vocab.bpe.32000`, execute the `nmt/scripts/wmt16_en_de.sh` script. If you face an issue of a size mismatch between the checkpoint graph's embedding layer and vocabulary (both src and target), make sure you add the following code to the `nmt.py` file to the `extend_hparams` function after the line 508 (after initialization of the `src_vocab_size` and `tgt_vocab_size` variables): -```py -src_vocab_size -= 1 -tgt_vocab_size -= 1 -``` +.. code-block:: sh + + python -m nmt.nmt + --src=de + --tgt=en + --ckpt=/path/to/ckpt/translate.ckpt + --hparams_path=/path/to/repository/nmt/nmt/standard_hparams/wmt16_gnmt_4_layer.json + --vocab_prefix=/path/to/vocab/vocab.bpe.32000 + --out_dir="" + --dump_inference_model + --infer_mode beam_search + --path_to_dump /path/to/dump/model/ + + +If you use different checkpoints, use the corresponding values for the ``src``, ``tgt``, ``ckpt``, ``hparams_path``, and ``vocab_prefix`` parameters. +Inference checkpoint ``inference_GNMT_graph`` and frozen inference graph ``frozen_GNMT_inference_graph.pb`` will appear in the ``/path/to/dump/model/`` folder. + +To generate ``vocab.bpe.32000``, execute the ``nmt/scripts/wmt16_en_de.sh`` script. If you face an issue of a size mismatch between the checkpoint graph's embedding layer and vocabulary (both src and target), make sure you add the following code to the ``nmt.py`` file to the ``extend_hparams`` function after the line 508 (after initialization of the ``src_vocab_size`` and ``tgt_vocab_size`` variables): + +.. code-block:: py + + src_vocab_size -= 1 + tgt_vocab_size -= 1 + **Step 4**. Convert the model to the IR: -```sh -mo ---input_model /path/to/dump/model/frozen_GNMT_inference_graph.pb ---input "IteratorGetNext:1{i32}[1],IteratorGetNext:0{i32}[1,50],dynamic_seq2seq/hash_table_Lookup_1:0[1]->[2],dynamic_seq2seq/hash_table_Lookup:0[1]->[1]" ---output dynamic_seq2seq/decoder/decoder/GatherTree ---output_dir /path/to/output/IR/ -``` +.. code-block:: sh -Input and output cutting with the `--input` and `--output` options is required since OpenVINO™ does not support `IteratorGetNext` and `LookupTableFindV2` operations. + mo + --input_model /path/to/dump/model/frozen_GNMT_inference_graph.pb + --input "IteratorGetNext:1{i32}[1],IteratorGetNext:0{i32}[1,50],dynamic_seq2seq/hash_table_Lookup_1:0[1]->[2],dynamic_seq2seq/hash_table_Lookup:0[1]->[1]" + --output dynamic_seq2seq/decoder/decoder/GatherTree + --output_dir /path/to/output/IR/ + + +Input and output cutting with the ``--input`` and ``--output`` options is required since OpenVINO™ does not support ``IteratorGetNext`` and ``LookupTableFindV2`` operations. Input cutting: -* `IteratorGetNext` operation iterates over a dataset. It is cut by output ports: port 0 contains data tensor with shape `[batch_size, max_sequence_length]`, port 1 contains `sequence_length` for every batch with shape `[batch_size]`. +* ``IteratorGetNext`` operation iterates over a dataset. It is cut by output ports: port 0 contains data tensor with shape ``[batch_size, max_sequence_length]``, port 1 contains ``sequence_length`` for every batch with shape ``[batch_size]``. -* `LookupTableFindV2` operations (`dynamic_seq2seq/hash_table_Lookup_1` and `dynamic_seq2seq/hash_table_Lookup` nodes in the graph) are cut with constant values). +* ``LookupTableFindV2`` operations (``dynamic_seq2seq/hash_table_Lookup_1`` and ``dynamic_seq2seq/hash_table_Lookup`` nodes in the graph) are cut with constant values). Output cutting: -* `LookupTableFindV2` operation is cut from the output and the `dynamic_seq2seq/decoder/decoder/GatherTree` node is treated as a new exit point. +* ``LookupTableFindV2`` operation is cut from the output and the ``dynamic_seq2seq/decoder/decoder/GatherTree`` node is treated as a new exit point. -For more information about model cutting, refer to the [Cutting Off Parts of a Model](@ref openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model) guide. +For more information about model cutting, refer to the :doc:`Cutting Off Parts of a Model ` guide. -## Using a GNMT Model +Using a GNMT Model +################## -> **NOTE**: This step assumes you have converted a model to the Intermediate Representation. +.. note:: + + This step assumes you have converted a model to the Intermediate Representation. Inputs of the model: -* `IteratorGetNext/placeholder_out_port_0` input with shape `[batch_size, max_sequence_length]` contains `batch_size` decoded input sentences. - Every sentence is decoded the same way as indices of sentence elements in vocabulary and padded with index of `eos` (end of sentence symbol). If the length of the sentence is less than `max_sequence_length`, remaining elements are filled with index of `eos` token. -* `IteratorGetNext/placeholder_out_port_1` input with shape `[batch_size]` contains sequence lengths for every sentence from the first input. - For example, if `max_sequence_length = 50`, `batch_size = 1` and the sentence has only 30 elements, then the input tensor for `IteratorGetNext/placeholder_out_port_1` should be `[30]`. +* ``IteratorGetNext/placeholder_out_port_0`` input with shape ``[batch_size, max_sequence_length]`` contains ``batch_size`` decoded input sentences. Every sentence is decoded the same way as indices of sentence elements in vocabulary and padded with index of ``eos`` (end of sentence symbol). If the length of the sentence is less than ``max_sequence_length``, remaining elements are filled with index of ``eos`` token. + +* ``IteratorGetNext/placeholder_out_port_1`` input with shape ``[batch_size]`` contains sequence lengths for every sentence from the first input. For example, if ``max_sequence_length = 50``, ``batch_size = 1`` and the sentence has only 30 elements, then the input tensor for ``IteratorGetNext/placeholder_out_port_1`` should be ``[30]``. Outputs of the model: -* `dynamic_seq2seq/decoder/decoder/GatherTree` tensor with shape `[max_sequence_length * 2, batch, beam_size]`, - that contains `beam_size` best translations for every sentence from input (also decoded as indices of words in +* ``dynamic_seq2seq/decoder/decoder/GatherTree`` tensor with shape ``[max_sequence_length * 2, batch, beam_size]``, + that contains ``beam_size`` best translations for every sentence from input (also decoded as indices of words in vocabulary). -> **NOTE**: The shape of this tensor in TensorFlow can be different: instead of `max_sequence_length * 2`, it can be any value less than that, because OpenVINO does not support dynamic shapes of outputs, while TensorFlow can stop decoding iterations when `eos` symbol is generated. -#### Running GNMT IR +.. note:: + The shape of this tensor in TensorFlow can be different: instead of ``max_sequence_length * 2``, it can be any value less than that, because OpenVINO does not support dynamic shapes of outputs, while TensorFlow can stop decoding iterations when ``eos`` symbol is generated. + +Running GNMT IR +--------------- 1. With benchmark app: -```sh -benchmark_app -m -d CPU -``` + +.. code-block:: sh + + benchmark_app -m -d CPU + 2. With OpenVINO Runtime Python API: -> **NOTE**: Before running the example, insert a path to your GNMT `.xml` and `.bin` files into `MODEL_PATH` and `WEIGHTS_PATH`, and fill `input_data_tensor` and `seq_lengths` tensors according to your input data. +.. note:: + + Before running the example, insert a path to your GNMT ``.xml`` and ``.bin`` files into ``MODEL_PATH`` and ``WEIGHTS_PATH``, and fill ``input_data_tensor`` and ``seq_lengths`` tensors according to your input data. + +.. code-block:: py + + from openvino.inference_engine import IENetwork, IECore + + MODEL_PATH = '/path/to/IR/frozen_GNMT_inference_graph.xml' + WEIGHTS_PATH = '/path/to/IR/frozen_GNMT_inference_graph.bin' -```python -from openvino.inference_engine import IENetwork, IECore + # Creating network + net = IENetwork( + model=MODEL_PATH, + weights=WEIGHTS_PATH) -MODEL_PATH = '/path/to/IR/frozen_GNMT_inference_graph.xml' -WEIGHTS_PATH = '/path/to/IR/frozen_GNMT_inference_graph.bin' + # Creating input data + input_data = {'IteratorGetNext/placeholder_out_port_0': input_data_tensor, + 'IteratorGetNext/placeholder_out_port_1': seq_lengths} -# Creating network -net = IENetwork( - model=MODEL_PATH, - weights=WEIGHTS_PATH) + # Creating plugin and loading extensions + ie = IECore() + ie.add_extension(extension_path="libcpu_extension.so", device_name="CPU") -# Creating input data -input_data = {'IteratorGetNext/placeholder_out_port_0': input_data_tensor, - 'IteratorGetNext/placeholder_out_port_1': seq_lengths} + # Loading network + exec_net = ie.load_network(network=net, device_name="CPU") -# Creating plugin and loading extensions -ie = IECore() -ie.add_extension(extension_path="libcpu_extension.so", device_name="CPU") + # Run inference + result_ie = exec_net.infer(input_data) -# Loading network -exec_net = ie.load_network(network=net, device_name="CPU") -# Run inference -result_ie = exec_net.infer(input_data) -``` +For more information about Python API, refer to the `OpenVINO Runtime Python API `__ guide. -For more information about Python API, refer to the [OpenVINO Runtime Python API](https://docs.openvino.ai/latest/api/api_reference.html) guide. +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_NCF_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_NCF_From_Tensorflow.md index 5c7820f4cba04e..fb9951ddfad3a7 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_NCF_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_NCF_From_Tensorflow.md @@ -1,49 +1,58 @@ # Converting a TensorFlow Neural Collaborative Filtering Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_NCF_From_Tensorflow} +@sphinxdirective + This tutorial explains how to convert Neural Collaborative Filtering (NCF) model to the OpenVINO Intermediate Representation. -[Public TensorFlow NCF model](https://github.com/tensorflow/models/tree/master/official/recommendation) does not contain pre-trained weights. To convert this model to the IR: - 1. Use [the instructions](https://github.com/tensorflow/models/tree/master/official/recommendation#train-and-evaluate-model) from this repository to train the model. - 2. Freeze the inference graph you get in the previous step in `model_dir`, following -the instructions from the **Freezing Custom Models in Python** section of the -[Converting a TensorFlow Model](../Convert_Model_From_TensorFlow.md) guide. +`Public TensorFlow NCF model `__ does not contain pre-trained weights. To convert this model to the IR: + +1. Use `the instructions `__ from this repository to train the model. + +2. Freeze the inference graph you get in the previous step in ``model_dir``, following the instructions from the **Freezing Custom Models in Python** section of the :doc:`Converting a TensorFlow Model ` guide. + Run the following commands: -```python -import tensorflow as tf -from tensorflow.python.framework import graph_io - -sess = tf.compat.v1.Session() -saver = tf.compat.v1.train.import_meta_graph("/path/to/model/model.meta") -saver.restore(sess, tf.train.latest_checkpoint('/path/to/model/')) - -frozen = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph_def, \ - ["rating/BiasAdd"]) -graph_io.write_graph(frozen, './', 'inference_graph.pb', as_text=False) -``` -where `rating/BiasAdd` is an output node. - - 3. Convert the model to the OpenVINO format. If you look at your frozen model, you can see that -it has one input that is split into four `ResourceGather` layers. (Click image to zoom in.) - -![NCF model beginning](../../../img/NCF_start.svg) - - However, as the Model Optimizer does not support such data feeding, you should skip it. Cut -the edges incoming in `ResourceGather` port 1: -```sh - mo --input_model inference_graph.pb \ ---input 1:embedding/embedding_lookup,1:embedding_1/embedding_lookup, \ -1:embedding_2/embedding_lookup,1:embedding_3/embedding_lookup \ ---input_shape [256],[256],[256],[256] \ ---output_dir -``` -In the `input_shape` parameter, 256 specifies the `batch_size` for your model. + +.. code-block:: py + + import tensorflow as tf + from tensorflow.python.framework import graph_io + + sess = tf.compat.v1.Session() + saver = tf.compat.v1.train.import_meta_graph("/path/to/model/model.meta") + saver.restore(sess, tf.train.latest_checkpoint('/path/to/model/')) + + frozen = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph_def, \ + ["rating/BiasAdd"]) + graph_io.write_graph(frozen, './', 'inference_graph.pb', as_text=False) + +where ``rating/BiasAdd`` is an output node. + +3. Convert the model to the OpenVINO format. If you look at your frozen model, you can see that +it has one input that is split into four ``ResourceGather`` layers. (Click image to zoom in.) + +.. image:: ./_static/images/NCF_start.svg + +However, as the Model Optimizer does not support such data feeding, you should skip it. Cut +the edges incoming in ``ResourceGather`` port 1: + +.. code-block:: shell + + mo --input_model inference_graph.pb \ + --input 1:embedding/embedding_lookup,1:embedding_1/embedding_lookup, \ + 1:embedding_2/embedding_lookup,1:embedding_3/embedding_lookup \ + --input_shape [256],[256],[256],[256] \ + --output_dir + +In the ``input_shape`` parameter, 256 specifies the ``batch_size`` for your model. Alternatively, you can do steps 2 and 3 in one command line: -```sh - mo --input_meta_graph /path/to/model/model.meta \ ---input 1:embedding/embedding_lookup,1:embedding_1/embedding_lookup, \ -1:embedding_2/embedding_lookup,1:embedding_3/embedding_lookup \ ---input_shape [256],[256],[256],[256] --output rating/BiasAdd \ ---output_dir -``` +.. code-block:: shell + + mo --input_meta_graph /path/to/model/model.meta \ + --input 1:embedding/embedding_lookup,1:embedding_1/embedding_lookup, \ + 1:embedding_2/embedding_lookup,1:embedding_3/embedding_lookup \ + --input_shape [256],[256],[256],[256] --output rating/BiasAdd \ + --output_dir + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md index 87b2dd159ebf01..0a40001afeb6d5 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md @@ -1,18 +1,19 @@ # Converting TensorFlow Object Detection API Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models} -**NOTES**: +@sphinxdirective -* Starting with the 2022.1 release, Model Optimizer can convert the TensorFlow Object Detection API Faster and Mask RCNNs topologies differently. By default, Model Optimizer adds operation "Proposal" to the generated IR. This operation needs an additional input to the model with name "image_info" which should be fed with several values describing the preprocessing applied to the input image (refer to the [Proposal](@ref openvino_docs_ops_detection_Proposal_4) operation specification for more information). However, this input is redundant for the models trained and inferred with equal size images. Model Optimizer can generate IR for such models and insert operation [DetectionOutput](@ref openvino_docs_ops_detection_DetectionOutput_1) instead of `Proposal`. The `DetectionOutput` operation does not require additional model input "image_info". Moreover, for some models the produced inference results are closer to the original TensorFlow model. In order to trigger new behavior, the attribute "operation_to_add" in the corresponding JSON transformation configuration file should be set to value "DetectionOutput" instead of default one "Proposal". -* Starting with the 2021.1 release, Model Optimizer converts the TensorFlow Object Detection API SSDs, Faster and Mask RCNNs topologies keeping shape-calculating sub-graphs by default, so topologies can be re-shaped in the OpenVINO Runtime using dedicated reshape API. Refer to the [Using Shape Inference](@ref openvino_docs_OV_UG_ShapeInference) guide for more information on how to use this feature. It is possible to change the both spatial dimensions of the input image and batch size. -* To generate IRs for TF 1 SSD topologies, Model Optimizer creates a number of `PriorBoxClustered` operations instead of a constant node with prior boxes calculated for the particular input image size. This change allows you to reshape the topology in the OpenVINO Runtime using dedicated API. The reshaping is supported for all SSD topologies except FPNs, which contain hardcoded shapes for some operations preventing from changing topology input shape. +* Starting with the 2022.1 release, Model Optimizer can convert the TensorFlow Object Detection API Faster and Mask RCNNs topologies differently. By default, Model Optimizer adds operation "Proposal" to the generated IR. This operation needs an additional input to the model with name "image_info" which should be fed with several values describing the preprocessing applied to the input image (refer to the :doc:`Proposal ` operation specification for more information). However, this input is redundant for the models trained and inferred with equal size images. Model Optimizer can generate IR for such models and insert operation :doc:`DetectionOutput ` instead of ``Proposal``. The `DetectionOutput` operation does not require additional model input "image_info". Moreover, for some models the produced inference results are closer to the original TensorFlow model. In order to trigger new behavior, the attribute "operation_to_add" in the corresponding JSON transformation configuration file should be set to value "DetectionOutput" instead of default one "Proposal". +* Starting with the 2021.1 release, Model Optimizer converts the TensorFlow Object Detection API SSDs, Faster and Mask RCNNs topologies keeping shape-calculating sub-graphs by default, so topologies can be re-shaped in the OpenVINO Runtime using dedicated reshape API. Refer to the :doc:`Using Shape Inference ` guide for more information on how to use this feature. It is possible to change the both spatial dimensions of the input image and batch size. +* To generate IRs for TF 1 SSD topologies, Model Optimizer creates a number of ``PriorBoxClustered`` operations instead of a constant node with prior boxes calculated for the particular input image size. This change allows you to reshape the topology in the OpenVINO Runtime using dedicated API. The reshaping is supported for all SSD topologies except FPNs, which contain hardcoded shapes for some operations preventing from changing topology input shape. -## Converting a Model +Converting a Model +################## -You can download TensorFlow Object Detection API models from the TensorFlow 1 Detection Model Zoo or TensorFlow 2 Detection Model Zoo. +You can download TensorFlow Object Detection API models from the `TensorFlow 1 Detection Model Zoo `__ or `TensorFlow 2 Detection Model Zoo `__. -> **NOTE**: Before converting, make sure you have configured Model Optimizer. For configuration steps, refer to the [Configuring Model Optimizer](../../../Deep_Learning_Model_Optimizer_DevGuide.md). +.. note:: -@sphinxdirective + Before converting, make sure you have configured Model Optimizer. For configuration steps, refer to the :doc:`Configuring Model Optimizer `. To convert a TensorFlow Object Detection API model, run the ``mo`` command with the following required parameters: @@ -48,105 +49,119 @@ To convert a TensorFlow Object Detection API model, run the ``mo`` command with * ``rfcn_support_api_v1.14.json`` - for RFCN topology from the models zoo frozen with TensorFlow version 1.14.0 or higher * ``--tensorflow_object_detection_api_pipeline_config `` - A special configuration file that describes the topology hyper-parameters and structure of the TensorFlow Object Detection API model. For the models downloaded from the TensorFlow Object Detection API zoo, the configuration file is named ``pipeline.config``. If you plan to train a model yourself, you can find templates for these files in the `models repository `__. -* ``--input_shape`` (optional) - A custom input image shape. For more information how the ``--input_shape`` parameter is handled for the TensorFlow Object Detection API models, refer to the :ref:`Custom Input Shape ` guide. +* ``--input_shape`` (optional) - A custom input image shape. For more information how the ``--input_shape`` parameter is handled for the TensorFlow Object Detection API models, refer to the `Custom Input Shape <#Custom-Input-Shape>`__ guide. -@endsphinxdirective +.. note:: + The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the ``RGB<->BGR`` conversion specifying the command-line parameter: ``--reverse_input_channels``. Otherwise, inference results may be incorrect. If you convert a TensorFlow Object Detection API model to use with the OpenVINO sample applications, you must specify the ``--reverse_input_channels`` parameter. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the :doc:`Converting a Model to Intermediate Representation (IR) ` guide. -> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. If you convert a TensorFlow Object Detection API model to use with the OpenVINO sample applications, you must specify the `--reverse_input_channels` parameter. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the [Converting a Model to Intermediate Representation (IR)](../Converting_Model.md) guide. +Additionally to the mandatory parameters listed above you can use optional conversion parameters if needed. A full list of parameters is available in the :doc:`Converting a TensorFlow Model ` guide. -Additionally to the mandatory parameters listed above you can use optional conversion parameters if needed. A full list of parameters is available in the [Converting a TensorFlow Model](../Convert_Model_From_TensorFlow.md) guide. +For example, if you downloaded the pre-trained `SSD InceptionV2 topology `__ and extracted archive to the directory ``/tmp/ssd_inception_v2_coco_2018_01_28``, the sample command line to convert the model looks as follows: -For example, if you downloaded the pre-trained [SSD InceptionV2 topology](http://download.tensorflow.org/models/object_detection/ssd_inception_v2_coco_2018_01_28.tar.gz) and extracted archive to the directory `/tmp/ssd_inception_v2_coco_2018_01_28`, the sample command line to convert the model looks as follows: +.. code-block:: sh -``` -mo --input_model=/tmp/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb --transformations_config front/tf/ssd_v2_support.json --tensorflow_object_detection_api_pipeline_config /tmp/ssd_inception_v2_coco_2018_01_28/pipeline.config --reverse_input_channels -``` + mo --input_model=/tmp/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb --transformations_config front/tf/ssd_v2_support.json --tensorflow_object_detection_api_pipeline_config /tmp/ssd_inception_v2_coco_2018_01_28/pipeline.config --reverse_input_channels -## OpenVINO™ Toolkit Samples and Open Model Zoo Demos + +OpenVINO™ Toolkit Samples and Open Model Zoo Demos +################################################## OpenVINO comes with a number of samples to demonstrate use of OpenVINO Runtime API. Additionally, Open Model Zoo provides set of demo applications to show implementation of close to real life applications, based on deep learning in various tasks, including Image Classification, Visual Object Detection, Text Recognition, Speech Recognition, Natural Language Processing and others. Refer to the links below for more details. -* [OpenVINO Samples](@ref openvino_docs_OV_UG_Samples_Overview) -* [Open Model Zoo Demos](@ref omz_demos) +* :doc:`OpenVINO Samples ` +* :doc:`Open Model Zoo Demos ` -## Feeding Input Images to the Samples +Feeding Input Images to the Samples +################################### There are several important notes about feeding input images to the samples: -1. OpenVINO samples stretch input image to the size of the input operation without preserving aspect ratio. This behavior is usually correct for most topologies (including SSDs), but incorrect for other models like Faster R-CNN, Mask R-CNN and R-FCN. These models usually use keeps aspect ratio resizer. The type of preprocessing is defined in the pipeline configuration file in the section `image_resizer`. If keeping aspect ratio is used, then it is necessary to resize image before passing it to the sample and optionally pad the resized image with 0s (if the attribute "pad_to_max_dimension" in the pipeline.config is equal to "true"). +1. OpenVINO samples stretch input image to the size of the input operation without preserving aspect ratio. This behavior is usually correct for most topologies (including SSDs), but incorrect for other models like Faster R-CNN, Mask R-CNN and R-FCN. These models usually use keeps aspect ratio resizer. The type of preprocessing is defined in the pipeline configuration file in the section ``image_resizer``. If keeping aspect ratio is used, then it is necessary to resize image before passing it to the sample and optionally pad the resized image with 0s (if the attribute "pad_to_max_dimension" in the pipeline.config is equal to "true"). -2. TensorFlow implementation of image resize may be different from the one implemented in the sample. Even reading input image from compressed format (like `.jpg`) could give different results in the sample and TensorFlow. If it is necessary to compare accuracy between the TensorFlow and the OpenVINO, it is recommended to pass pre-resized input image in a non-compressed format (like `.bmp`). +2. TensorFlow implementation of image resize may be different from the one implemented in the sample. Even reading input image from compressed format (like ``.jpg``) could give different results in the sample and TensorFlow. If it is necessary to compare accuracy between the TensorFlow and the OpenVINO, it is recommended to pass pre-resized input image in a non-compressed format (like ``.bmp``). -3. If you want to infer the model with the OpenVINO samples, convert the model specifying the `--reverse_input_channels` command line parameter. The samples load images in BGR channels order, while TensorFlow models were trained with images in RGB order. When the `--reverse_input_channels` command line parameter is specified, Model Optimizer performs first convolution or other channel dependent operation weights modification so the output will be like the image is passed with RGB channels order. +3. If you want to infer the model with the OpenVINO samples, convert the model specifying the ``--reverse_input_channels`` command line parameter. The samples load images in BGR channels order, while TensorFlow models were trained with images in RGB order. When the ``--reverse_input_channels`` command line parameter is specified, Model Optimizer performs first convolution or other channel dependent operation weights modification so the output will be like the image is passed with RGB channels order. 4. Read carefully the messages printed by Model Optimizer during a model conversion. They contain important instructions on how to prepare input data before running the inference and how to interpret the output. -@sphinxdirective +Custom Input Shape +################## -.. _custom-input-shape: +Model Optimizer handles the command line parameter ``--input_shape`` for TensorFlow Object Detection API models in a special way depending on the image resizer type defined in the ``pipeline.config`` file. TensorFlow Object Detection API generates different ``Preprocessor`` sub-graph based on the image resizer type. Model Optimizer supports two types of image resizer: -@endsphinxdirective +* ``fixed_shape_resizer`` --- *Stretches* input image to the specific height and width. The ``pipeline.config`` snippet below shows a ``fixed_shape_resizer`` sample definition: +.. code-block:: sh -## Custom Input Shape -Model Optimizer handles the command line parameter `--input_shape` for TensorFlow Object Detection API models in a special way depending on the image resizer type defined in the `pipeline.config` file. TensorFlow Object Detection API generates different `Preprocessor` sub-graph based on the image resizer type. Model Optimizer supports two types of image resizer: -* `fixed_shape_resizer` --- *Stretches* input image to the specific height and width. The `pipeline.config` snippet below shows a `fixed_shape_resizer` sample definition: -``` -image_resizer { - fixed_shape_resizer { - height: 300 - width: 300 + image_resizer { + fixed_shape_resizer { + height: 300 + width: 300 + } } -} -``` -* `keep_aspect_ratio_resizer` --- Resizes the input image *keeping aspect ratio* to satisfy the minimum and maximum size constraints. The `pipeline.config` snippet below shows a `keep_aspect_ratio_resizer` sample definition: -``` -image_resizer { - keep_aspect_ratio_resizer { - min_dimension: 600 - max_dimension: 1024 + +* ``keep_aspect_ratio_resizer`` --- Resizes the input image *keeping aspect ratio* to satisfy the minimum and maximum size constraints. The ``pipeline.config`` snippet below shows a ``keep_aspect_ratio_resizer`` sample definition: + +.. code-block:: sh + + image_resizer { + keep_aspect_ratio_resizer { + min_dimension: 600 + max_dimension: 1024 + } } -} -``` + If an additional parameter "pad_to_max_dimension" is equal to "true", then the resized image will be padded with 0s to the square image of size "max_dimension". -### Fixed Shape Resizer Replacement -* If the `--input_shape` command line parameter is not specified, Model Optimizer generates an input operation with the height and width as defined in the `pipeline.config`. +Fixed Shape Resizer Replacement ++++++++++++++++++++++++++++++++ -* If the `--input_shape [1, H, W, 3]` command line parameter is specified, Model Optimizer sets the input operation height to `H` and width to `W` and convert the model. However, the conversion may fail because of the following reasons: - * The model is not reshape-able, meaning that it's not possible to change the size of the model input image. For example, SSD FPN models have `Reshape` operations with hard-coded output shapes, but the input size to these `Reshape` instances depends on the input image size. In this case, Model Optimizer shows an error during the shape inference phase. Run Model Optimizer with `--log_level DEBUG` to see the inferred operations output shapes to see the mismatch. - * Custom input shape is too small. For example, if you specify `--input_shape [1,100,100,3]` to convert a SSD Inception V2 model, one of convolution or pooling nodes decreases input tensor spatial dimensions to non-positive values. In this case, Model Optimizer shows error message like this: '[ ERROR ] Shape [ 1 -1 -1 256] is not fully defined for output X of "node_name".' +* If the ``--input_shape`` command line parameter is not specified, Model Optimizer generates an input operation with the height and width as defined in the ``pipeline.config``. +* If the ``--input_shape [1, H, W, 3]`` command line parameter is specified, Model Optimizer sets the input operation height to ``H`` and width to ``W`` and convert the model. However, the conversion may fail because of the following reasons: -### Keeping Aspect Ratio Resizer Replacement -* If the `--input_shape` command line parameter is not specified, Model Optimizer generates an input operation with both height and width equal to the value of parameter `min_dimension` in the `keep_aspect_ratio_resizer`. + * The model is not reshape-able, meaning that it's not possible to change the size of the model input image. For example, SSD FPN models have ``Reshape`` operations with hard-coded output shapes, but the input size to these ``Reshape`` instances depends on the input image size. In this case, Model Optimizer shows an error during the shape inference phase. Run Model Optimizer with ``--log_level DEBUG`` to see the inferred operations output shapes to see the mismatch. + * Custom input shape is too small. For example, if you specify ``--input_shape [1,100,100,3]`` to convert a SSD Inception V2 model, one of convolution or pooling nodes decreases input tensor spatial dimensions to non-positive values. In this case, Model Optimizer shows error message like this: '[ ERROR ] Shape [ 1 -1 -1 256] is not fully defined for output X of "node_name".' -* If the `--input_shape [1, H, W, 3]` command line parameter is specified, Model Optimizer scales the specified input image height `H` and width `W` to satisfy the `min_dimension` and `max_dimension` constraints defined in the `keep_aspect_ratio_resizer`. The following function calculates the input operation height and width: -```python -def calculate_shape_keeping_aspect_ratio(H: int, W: int, min_dimension: int, max_dimension: int): - ratio_min = min_dimension / min(H, W) - ratio_max = max_dimension / max(H, W) - ratio = min(ratio_min, ratio_max) - return int(round(H * ratio)), int(round(W * ratio)) -``` -The `--input_shape` command line parameter should be specified only if the "pad_to_max_dimension" does not exist of is set to "false" in the `keep_aspect_ratio_resizer`. +Keeping Aspect Ratio Resizer Replacement +++++++++++++++++++++++++++++++++++++++++ -Models with `keep_aspect_ratio_resizer` were trained to recognize object in real aspect ratio, in contrast with most of the classification topologies trained to recognize objects stretched vertically and horizontally as well. By default, Model Optimizer converts topologies with `keep_aspect_ratio_resizer` to consume a square input image. If the non-square image is provided as input, it is stretched without keeping aspect ratio that results to object detection quality decrease. +* If the ``--input_shape`` command line parameter is not specified, Model Optimizer generates an input operation with both height and width equal to the value of parameter ``min_dimension`` in the ``keep_aspect_ratio_resizer``. -> **NOTE**: It is highly recommended to specify the `--input_shape` command line parameter for the models with `keep_aspect_ratio_resizer`, if the input image dimensions are known in advance. +* If the ``--input_shape [1, H, W, 3]`` command line parameter is specified, Model Optimizer scales the specified input image height ``H`` and width ``W`` to satisfy the ``min_dimension`` and ``max_dimension`` constraints defined in the ``keep_aspect_ratio_resizer``. The following function calculates the input operation height and width: -## Model Conversion Process in Detail +.. code-block:: py -This section is intended for users who want to understand how Model Optimizer performs Object Detection API models conversion in details. The information in this section is also useful for users having complex models that are not converted with Model Optimizer out of the box. It is highly recommended to read the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](../../customize_model_optimizer/Customize_Model_Optimizer.md) documentation first to understand sub-graph replacement concepts which are used here. + def calculate_shape_keeping_aspect_ratio(H: int, W: int, min_dimension: int, max_dimension: int): + ratio_min = min_dimension / min(H, W) + ratio_max = max_dimension / max(H, W) + ratio = min(ratio_min, ratio_max) + return int(round(H * ratio)), int(round(W * ratio)) -It is also important to open the model in the [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) to see the topology structure. Model Optimizer can create an event file that can be then fed to the TensorBoard tool. Run Model Optimizer, providing two command line parameters: -* `--input_model ` --- Path to the frozen model. -* `--tensorboard_logdir` --- Path to the directory where TensorBoard looks for the event files. +The ``--input_shape`` command line parameter should be specified only if the "pad_to_max_dimension" does not exist of is set to "false" in the ``keep_aspect_ratio_resizer``. -Implementation of the transformations for Object Detection API models is located in the [file](https://github.com/openvinotoolkit/openvino/blob/releases/2022/1/tools/mo/openvino/tools/mo/front/tf/ObjectDetectionAPI.py). Refer to the code in this file to understand the details of the conversion process. +Models with ``keep_aspect_ratio_resizer`` were trained to recognize object in real aspect ratio, in contrast with most of the classification topologies trained to recognize objects stretched vertically and horizontally as well. By default, Model Optimizer converts topologies with ``keep_aspect_ratio_resizer`` to consume a square input image. If the non-square image is provided as input, it is stretched without keeping aspect ratio that results to object detection quality decrease. +.. note:: + + It is highly recommended to specify the ``--input_shape`` command line parameter for the models with ``keep_aspect_ratio_resizer``, if the input image dimensions are known in advance. + +Model Conversion Process in Detail +################################## + +This section is intended for users who want to understand how Model Optimizer performs Object Detection API models conversion in details. The information in this section is also useful for users having complex models that are not converted with Model Optimizer out of the box. It is highly recommended to read the **Graph Transformation Extensions** section in the :doc:`Model Optimizer Extensibility ` documentation first to understand sub-graph replacement concepts which are used here. + +It is also important to open the model in the `TensorBoard `__ to see the topology structure. Model Optimizer can create an event file that can be then fed to the TensorBoard tool. Run Model Optimizer, providing two command line parameters: + +* ``--input_model `` --- Path to the frozen model. +* ``--tensorboard_logdir`` --- Path to the directory where TensorBoard looks for the event files. + +Implementation of the transformations for Object Detection API models is located in the `file `__. Refer to the code in this file to understand the details of the conversion process. + + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_RetinaNet_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_RetinaNet_From_Tensorflow.md index 16a77341c5ff04..6278f614415156 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_RetinaNet_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_RetinaNet_From_Tensorflow.md @@ -1,15 +1,21 @@ # Converting a TensorFlow RetinaNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_RetinaNet_From_Tensorflow} +@sphinxdirective + This tutorial explains how to convert a RetinaNet model to the Intermediate Representation (IR). -[Public RetinaNet model](https://github.com/fizyr/keras-retinanet) does not contain pretrained TensorFlow weights. -To convert this model to the TensorFlow format, follow the [Reproduce Keras to TensorFlow Conversion tutorial](@ref omz_models_model_retinanet_tf). +`Public RetinaNet model `__ does not contain pretrained TensorFlow weights. +To convert this model to the TensorFlow format, follow the `Reproduce Keras to TensorFlow Conversion tutorial `__. After converting the model to TensorFlow format, run the Model Optimizer command below: -```sh -mo --input "input_1[1,1333,1333,3]" --input_model retinanet_resnet50_coco_best_v2.1.0.pb --transformations_config front/tf/retinanet.json -``` -Where `transformations_config` command-line parameter specifies the configuration json file containing model conversion hints for the Model Optimizer. +.. code-block:: sh + + mo --input "input_1[1,1333,1333,3]" --input_model retinanet_resnet50_coco_best_v2.1.0.pb --transformations_config front/tf/retinanet.json + + +Where ``transformations_config`` command-line parameter specifies the configuration json file containing model conversion hints for the Model Optimizer. The json file contains some parameters that need to be changed if you train the model yourself. It also contains information on how to match endpoints to replace the subgraph nodes. After the model is converted to the OpenVINO IR format, the output nodes will be replaced with DetectionOutput layer. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md index fb8bd420b44e3f..ffecf5a24d30f7 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md @@ -1,76 +1,90 @@ # Converting TensorFlow Slim Image Classification Model Library Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Slim_Library_Models} -TensorFlow-Slim Image Classification Model Library is a library to define, train and evaluate classification models in TensorFlow. The library contains Python scripts defining the classification topologies together with checkpoint files for several pre-trained classification topologies. To convert a TensorFlow-Slim library model, complete the following steps: +@sphinxdirective -1. Download the TensorFlow-Slim models [git repository](https://github.com/tensorflow/models). -2. Download the pre-trained model [checkpoint](https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models). +`TensorFlow-Slim Image Classification Model Library `__ is a library to define, train and evaluate classification models in TensorFlow. The library contains Python scripts defining the classification topologies together with checkpoint files for several pre-trained classification topologies. To convert a TensorFlow-Slim library model, complete the following steps: + +1. Download the TensorFlow-Slim models `git repository `__. +2. Download the pre-trained model `checkpoint `__. 3. Export the inference graph. 4. Convert the model using the Model Optimizer. -The [Example of an Inception V1 Model Conversion](#example_of_an_inception_v1_model_conversion) below illustrates the process of converting an Inception V1 Model. +The `Example of an Inception V1 Model Conversion <#example_of_an_inception_v1_model_conversion>`__ below illustrates the process of converting an Inception V1 Model. + +Example of an Inception V1 Model Conversion +########################################### -## Example of an Inception V1 Model Conversion This example demonstrates how to convert the model on Linux OSes, but it could be easily adopted for the Windows OSes. **Step 1**. Create a new directory to clone the TensorFlow-Slim git repository to: -```sh -mkdir tf_models -``` -```sh -git clone https://github.com/tensorflow/models.git tf_models -``` +.. code-block:: sh + + mkdir tf_models + +.. code-block:: sh + + git clone https://github.com/tensorflow/models.git tf_models + + +**Step 2**. Download and unpack the `Inception V1 model checkpoint file `__: + +.. code-block:: sh + + wget http://download.tensorflow.org/models/inception_v1_2016_08_28.tar.gz + +.. code-block:: sh + + tar xzvf inception_v1_2016_08_28.tar.gz -**Step 2**. Download and unpack the [Inception V1 model checkpoint file](http://download.tensorflow.org/models/inception_v1_2016_08_28.tar.gz): +**Step 3**. Export the inference graph --- the protobuf file (``.pb``) containing the architecture of the topology. This file *does not* contain the neural network weights and cannot be used for inference. -```sh -wget http://download.tensorflow.org/models/inception_v1_2016_08_28.tar.gz -``` -```sh -tar xzvf inception_v1_2016_08_28.tar.gz -``` +.. code-block:: sh -**Step 3**. Export the inference graph --- the protobuf file (`.pb`) containing the architecture of the topology. This file *does not* contain the neural network weights and cannot be used for inference. + python3 tf_models/research/slim/export_inference_graph.py \ + --model_name inception_v1 \ + --output_file inception_v1_inference_graph.pb -```sh -python3 tf_models/research/slim/export_inference_graph.py \ - --model_name inception_v1 \ - --output_file inception_v1_inference_graph.pb -``` Model Optimizer comes with the summarize graph utility, which identifies graph input and output nodes. Run the utility to determine input/output nodes of the Inception V1 model: -```sh -python3 /openvino/tools/mo/utils/summarize_graph.py --input_model ./inception_v1_inference_graph.pb -``` +.. code-block:: sh -The output looks as follows:
-```sh -1 input(s) detected: -Name: input, type: float32, shape: (-1,224,224,3) -1 output(s) detected: -InceptionV1/Logits/Predictions/Reshape_1 -``` -The tool finds one input node with name `input`, type `float32`, fixed image size `(224,224,3)` and undefined batch size `-1`. The output node name is `InceptionV1/Logits/Predictions/Reshape_1`.
+ python3 /openvino/tools/mo/utils/summarize_graph.py --input_model ./inception_v1_inference_graph.pb + +The output looks as follows: + +.. code-block:: sh + + 1 input(s) detected: + Name: input, type: float32, shape: (-1,224,224,3) + 1 output(s) detected: + InceptionV1/Logits/Predictions/Reshape_1 + +The tool finds one input node with name ``input``, type ``float32``, fixed image size ``(224,224,3)`` and undefined batch size ``-1``. The output node name is ``InceptionV1/Logits/Predictions/Reshape_1``. **Step 4**. Convert the model with the Model Optimizer: -```sh -mo --input_model ./inception_v1_inference_graph.pb --input_checkpoint ./inception_v1.ckpt -b 1 --mean_value [127.5,127.5,127.5] --scale 127.5 -``` +.. code-block:: sh + + mo --input_model ./inception_v1_inference_graph.pb --input_checkpoint ./inception_v1.ckpt -b 1 --mean_value [127.5,127.5,127.5] --scale 127.5 -The `-b` command line parameter is required because the Model Optimizer cannot convert a model with undefined input size. -For the information on why `--mean_values` and `--scale` command-line parameters are used, refer to the [Mean and Scale Values for TensorFlow-Slim Models](#tf_slim_mean_scale_values). +The ``-b`` command line parameter is required because the Model Optimizer cannot convert a model with undefined input size. + +For the information on why ``--mean_values`` and ``--scale`` command-line parameters are used, refer to the `Mean and Scale Values for TensorFlow-Slim Models <#Mean-and-Scale-Values-for-TensorFlow-Slim-Models>`__. + +Mean and Scale Values for TensorFlow-Slim Models +################################################# -## Mean and Scale Values for TensorFlow-Slim Models The TensorFlow-Slim Models were trained with normalized input data. There are several different normalization algorithms used in the Slim library. OpenVINO classification sample does not perform image pre-processing except resizing to the input layer size. It is necessary to pass mean and scale values to the Model Optimizer so they are embedded into the generated IR in order to get correct classification results. -The file [preprocessing_factory.py](https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/preprocessing_factory.py) contains a dictionary variable `preprocessing_fn_map` defining mapping between the model type and pre-processing function to be used. The function code should be analyzed to figure out the mean/scale values. +The file `preprocessing_factory.py `__ contains a dictionary variable ``preprocessing_fn_map`` defining mapping between the model type and pre-processing function to be used. The function code should be analyzed to figure out the mean/scale values. + +The `inception_preprocessing.py `__ file defines the pre-processing function for the Inception models. The ``preprocess_for_eval`` function contains the following code: -The [inception_preprocessing.py](https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/inception_preprocessing.py) file defines the pre-processing function for the Inception models. The `preprocess_for_eval` function contains the following code: +.. code-block:: python -```python3 ... import tensorflow as tf if image.dtype != tf.float32: @@ -79,12 +93,14 @@ The [inception_preprocessing.py](https://github.com/tensorflow/models/blob/maste image = tf.subtract(image, 0.5) image = tf.multiply(image, 2.0) return image -``` -Firstly, the `image` is converted to data type `tf.float32` and the values in the tensor are scaled to the `[0, 1]` range using the [tf.image.convert_image_dtype](https://www.tensorflow.org/api_docs/python/tf/image/convert_image_dtype) function. Then the `0.5` is subtracted from the image values and values multiplied by `2.0`. The final image range of values is `[-1, 1]`. -OpenVINO classification sample reads an input image as a three-dimensional array of integer values from the range `[0, 255]`. In order to scale them to `[-1, 1]` range, the mean value `127.5` for each image channel should be specified as well as a scale factor `127.5`. +Firstly, the ``image`` is converted to data type `tf.float32` and the values in the tensor are scaled to the ``[0, 1]`` range using the `tf.image.convert_image_dtype `__ function. Then the ``0.5`` is subtracted from the image values and values multiplied by ``2.0``. The final image range of values is ``[-1, 1]``. + +OpenVINO classification sample reads an input image as a three-dimensional array of integer values from the range ``[0, 255]``. In order to scale them to ``[-1, 1]`` range, the mean value ``127.5`` for each image channel should be specified as well as a scale factor ``127.5``. Similarly, the mean/scale values can be determined for other Slim models. -The exact mean/scale values are defined in the table with list of supported TensorFlow-Slim models at the [Converting a TensorFlow Model](../Convert_Model_From_TensorFlow.md) guide. +The exact mean/scale values are defined in the table with list of supported TensorFlow-Slim models at the :doc:`Converting a TensorFlow Model ` guide. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_WideAndDeep_Family_Models.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_WideAndDeep_Family_Models.md index b3fd681df24649..c3583380d2e309 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_WideAndDeep_Family_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_WideAndDeep_Family_Models.md @@ -1,23 +1,40 @@ # Converting TensorFlow Wide and Deep Family Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_WideAndDeep_Family_Models} +@sphinxdirective + The Wide and Deep models is a combination of wide and deep parts for memorization and generalization of object features respectively. These models can contain different types of object features such as numerical, categorical, sparse and sequential features. These feature types are specified through Tensorflow tf.feature_column API. Table below presents what feature types are supported by the OpenVINO toolkit. -| numeric | (weighted) categorical | categorical with hash | bucketized | sequential | crossed | -|:-------:|:----------------------:|:---------------------:|:----------:|:----------:|:-------:| -| yes | yes | no | yes | yes | no | +.. list-table:: + :header-rows: 1 + + * - numeric + - (weighted) categorical + - categorical with hash + - bucketized + - sequential + - crossed + * - yes + - yes + - no + - yes + - yes + - no -> **NOTE**: The categorical with hash and crossed features are currently unsupported since OpenVINO does not cover tensors of the `string` type and operations with them. -## Preparing an Example of Wide and Deep Model +.. note:: The categorical with hash and crossed features are currently unsupported since OpenVINO does not cover tensors of the `string` type and operations with them. + +Preparing an Example of Wide and Deep Model +########################################### **Step 1**. Clone the GitHub repository with TensorFlow models and move to the directory with an example of Wide and Deep model: -```sh -git clone https://github.com/tensorflow/models.git --branch r2.2.0; -cd official/r1/wide_deep -``` +.. code-block:: sh + + git clone https://github.com/tensorflow/models.git --branch r2.2.0; + cd official/r1/wide_deep + The Wide and Deep model is no longer in the master branch of the repository but is still available in the r2.2.0 branch. @@ -25,109 +42,114 @@ The Wide and Deep model is no longer in the master branch of the repository but **Step 2**. Train the model As the OpenVINO™ toolkit does not support the categorical with hash and crossed features, such feature types must be switched off in the model -by changing the `build_model_columns()` function in `census_dataset.py` as follows: - -```python -def build_model_columns(): - """Builds a set of wide and deep feature columns.""" - # Continuous variable columns - age = tf.feature_column.numeric_column('age') - education_num = tf.feature_column.numeric_column('education_num') - capital_gain = tf.feature_column.numeric_column('capital_gain') - capital_loss = tf.feature_column.numeric_column('capital_loss') - hours_per_week = tf.feature_column.numeric_column('hours_per_week') - education = tf.feature_column.categorical_column_with_vocabulary_list( - 'education', [ - 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', - 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', - '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) - marital_status = tf.feature_column.categorical_column_with_vocabulary_list( - 'marital_status', [ - 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', - 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) - relationship = tf.feature_column.categorical_column_with_vocabulary_list( - 'relationship', [ - 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', - 'Other-relative']) - workclass = tf.feature_column.categorical_column_with_vocabulary_list( - 'workclass', [ - 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', - 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) - # To show an example of hashing: - #occupation = tf.feature_column.categorical_column_with_hash_bucket( - # 'occupation', hash_bucket_size=_HASH_BUCKET_SIZE) - # Transformations. - age_buckets = tf.feature_column.bucketized_column( - age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) - # Wide columns and deep columns. - base_columns = [ - education, marital_status, relationship, workclass, - age_buckets, - ] - crossed_columns = [] - wide_columns = base_columns + crossed_columns - deep_columns = [ - age, - education_num, - capital_gain, - capital_loss, - hours_per_week, - tf.feature_column.indicator_column(workclass), - tf.feature_column.indicator_column(education), - tf.feature_column.indicator_column(marital_status), - tf.feature_column.indicator_column(relationship), - # To show an example of embedding - ] - return wide_columns, deep_columns -``` +by changing the ``build_model_columns()`` function in `census_dataset.py` as follows: + +.. code-block:: python + + def build_model_columns(): + """Builds a set of wide and deep feature columns.""" + # Continuous variable columns + age = tf.feature_column.numeric_column('age') + education_num = tf.feature_column.numeric_column('education_num') + capital_gain = tf.feature_column.numeric_column('capital_gain') + capital_loss = tf.feature_column.numeric_column('capital_loss') + hours_per_week = tf.feature_column.numeric_column('hours_per_week') + education = tf.feature_column.categorical_column_with_vocabulary_list( + 'education', [ + 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', + 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', + '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) + marital_status = tf.feature_column.categorical_column_with_vocabulary_list( + 'marital_status', [ + 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', + 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) + relationship = tf.feature_column.categorical_column_with_vocabulary_list( + 'relationship', [ + 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', + 'Other-relative']) + workclass = tf.feature_column.categorical_column_with_vocabulary_list( + 'workclass', [ + 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', + 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) + # To show an example of hashing: + #occupation = tf.feature_column.categorical_column_with_hash_bucket( + # 'occupation', hash_bucket_size=_HASH_BUCKET_SIZE) + # Transformations. + age_buckets = tf.feature_column.bucketized_column( + age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) + # Wide columns and deep columns. + base_columns = [ + education, marital_status, relationship, workclass, + age_buckets, + ] + crossed_columns = [] + wide_columns = base_columns + crossed_columns + deep_columns = [ + age, + education_num, + capital_gain, + capital_loss, + hours_per_week, + tf.feature_column.indicator_column(workclass), + tf.feature_column.indicator_column(education), + tf.feature_column.indicator_column(marital_status), + tf.feature_column.indicator_column(relationship), + # To show an example of embedding + ] + return wide_columns, deep_columns After that, start training with the following command: -```sh -python census_main.py -``` +.. code-block:: sh + + python census_main.py -## Converting the Wide and Deep Model to IR + +Converting the Wide and Deep Model to IR +######################################## Use the following command line to convert the saved model file with the checkpoint: -```sh - mo ---input_checkpoint checkpoint --input_meta_graph model.ckpt.meta ---input "IteratorGetNext:0[2], - IteratorGetNext:1[2], - IteratorGetNext:2[2], - IteratorGetNext:4[2], - IteratorGetNext:7[2], - linear/linear_model/linear_model/linear_model/education/to_sparse_input/indices:0[10,2]{i64}, - linear/linear_model/linear_model/linear_model/education/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - linear/linear_model/linear_model/linear_model/education/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - linear/linear_model/linear_model/linear_model/marital_status/to_sparse_input/indices:0[10,2]{i64}, - linear/linear_model/linear_model/linear_model/marital_status/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - linear/linear_model/linear_model/linear_model/marital_status/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - linear/linear_model/linear_model/linear_model/relationship/to_sparse_input/indices:0[10,2]{i64}, - linear/linear_model/linear_model/linear_model/relationship/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - linear/linear_model/linear_model/linear_model/relationship/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - linear/linear_model/linear_model/linear_model/workclass/to_sparse_input/indices:0[10,2]{i64}, - linear/linear_model/linear_model/linear_model/workclass/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - linear/linear_model/linear_model/linear_model/workclass/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - dnn/input_from_feature_columns/input_layer/education_indicator/to_sparse_input/indices:0[10,2]{i64}, - dnn/input_from_feature_columns/input_layer/education_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - dnn/input_from_feature_columns/input_layer/education_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - dnn/input_from_feature_columns/input_layer/marital_status_indicator/to_sparse_input/indices:0[10,2]{i64}, - dnn/input_from_feature_columns/input_layer/marital_status_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - dnn/input_from_feature_columns/input_layer/marital_status_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - dnn/input_from_feature_columns/input_layer/relationship_indicator/to_sparse_input/indices:0[10,2]{i64}, - dnn/input_from_feature_columns/input_layer/relationship_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - dnn/input_from_feature_columns/input_layer/relationship_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50], - dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/indices:0[10,2]{i64}, - dnn/input_from_feature_columns/input_layer/workclass_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, - dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50]" ---output head/predictions/probabilities -``` - -The model contains operations unsupported by the OpenVINO™ toolkit such as `IteratorGetNext` and `LookupTableFindV2`, so the Model Optimizer must prune these nodes. -The pruning is specified through `--input` option. The prunings for `IteratorGetNext:*` nodes correspond to numeric features. -The pruning for each categorical feature consists of three prunings for the following nodes: `*/to_sparse_input/indices:0`, `*/hash_table_Lookup/LookupTableFindV2:0`, and `*/to_sparse_input/dense_shape:0`. +.. code-block:: sh + + mo + --input_checkpoint checkpoint --input_meta_graph model.ckpt.meta + --input "IteratorGetNext:0[2], + IteratorGetNext:1[2], + IteratorGetNext:2[2], + IteratorGetNext:4[2], + IteratorGetNext:7[2], + linear/linear_model/linear_model/linear_model/education/to_sparse_input/indices:0[10,2]{i64}, + linear/linear_model/linear_model/linear_model/education/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + linear/linear_model/linear_model/linear_model/education/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + linear/linear_model/linear_model/linear_model/marital_status/to_sparse_input/indices:0[10,2]{i64}, + linear/linear_model/linear_model/linear_model/marital_status/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + linear/linear_model/linear_model/linear_model/marital_status/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + linear/linear_model/linear_model/linear_model/relationship/to_sparse_input/indices:0[10,2]{i64}, + linear/linear_model/linear_model/linear_model/relationship/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + linear/linear_model/linear_model/linear_model/relationship/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + linear/linear_model/linear_model/linear_model/workclass/to_sparse_input/indices:0[10,2]{i64}, + linear/linear_model/linear_model/linear_model/workclass/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + linear/linear_model/linear_model/linear_model/workclass/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + dnn/input_from_feature_columns/input_layer/education_indicator/to_sparse_input/indices:0[10,2]{i64}, + dnn/input_from_feature_columns/input_layer/education_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + dnn/input_from_feature_columns/input_layer/education_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + dnn/input_from_feature_columns/input_layer/marital_status_indicator/to_sparse_input/indices:0[10,2]{i64}, + dnn/input_from_feature_columns/input_layer/marital_status_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + dnn/input_from_feature_columns/input_layer/marital_status_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + dnn/input_from_feature_columns/input_layer/relationship_indicator/to_sparse_input/indices:0[10,2]{i64}, + dnn/input_from_feature_columns/input_layer/relationship_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + dnn/input_from_feature_columns/input_layer/relationship_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50], + dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/indices:0[10,2]{i64}, + dnn/input_from_feature_columns/input_layer/workclass_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64}, + dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2,50]" + --output head/predictions/probabilities + + +The model contains operations unsupported by the OpenVINO™ toolkit such as ``IteratorGetNext`` and ``LookupTableFindV2``, so the Model Optimizer must prune these nodes. +The pruning is specified through `--input` option. The prunings for ``IteratorGetNext:*`` nodes correspond to numeric features. +The pruning for each categorical feature consists of three prunings for the following nodes: ``*/to_sparse_input/indices:0``, ``*/hash_table_Lookup/LookupTableFindV2:0``, and ``*/to_sparse_input/dense_shape:0``. The above command line generates an OpenVINO model for a batch of two objects, with the total number of actual categorical feature values equal to 10 and maximum size of a sparse categorical feature for one object equal to 50. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_XLNet_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_XLNet_From_Tensorflow.md index 1f0cf82bb83ea5..e85d952770bb9f 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_XLNet_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_XLNet_From_Tensorflow.md @@ -1,29 +1,35 @@ # Converting a TensorFlow XLNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_XLNet_From_Tensorflow} +@sphinxdirective + Pretrained models for XLNet (Bidirectional Encoder Representations from Transformers) are -[publicly available](https://github.com/zihangdai/xlnet). +`publicly available `__. + +Supported Models +################ -## Supported Models +The following models from the pretrained `XLNet model list `__ are currently supported: -The following models from the pretrained [XLNet model list](https://github.com/zihangdai/xlnet#pre-trained-models) are currently supported: +* `XLNet-Large, Cased `__ +* `XLNet-Base, Cased `__ -* [XLNet-Large, Cased](https://storage.googleapis.com/xlnet/released_models/cased_L-24_H-1024_A-16.zip) -* [XLNet-Base, Cased](https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip) +Downloading the Pretrained Base XLNet Model +########################################### -## Downloading the Pretrained Base XLNet Model +Download and unzip an archive with the `XLNet-Base, Cased `__. -Download and unzip an archive with the [XLNet-Base, Cased](https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip). +After the archive is unzipped, the directory ``cased_L-12_H-768_A-12`` is created and contains the following files: -After the archive is unzipped, the directory `cased_L-12_H-768_A-12` is created and contains the following files: -* TensorFlow checkpoint (`xlnet_model.ckpt`), containing the pretrained weights (which is actually 3 files) -* sentence piece model (`spiece.model`) used for (de)tokenization -* config file (`xlnet_config.json`), which specifies the hyperparameters of the model +* TensorFlow checkpoint (``xlnet_model.ckpt``), containing the pretrained weights (which is actually 3 files) +* sentence piece model (``spiece.model``) used for (de)tokenization +* config file (``xlnet_config.json``), which specifies the hyperparameters of the model To get pb-file from the archive contents, you need to do the following. 1. Run commands - ```sh + .. code-block:: sh + cd ~ mkdir XLNet-Base cd XLNet-Base @@ -31,84 +37,84 @@ To get pb-file from the archive contents, you need to do the following. wget https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip unzip cased_L-12_H-768_A-12.zip mkdir try_save - ``` - - - -2. Save and run the following Python script in `~/XLNet-Base/xlnet`: - - > **NOTE**: The original model repository has been tested with TensorFlow 1.13.1 under Python2. - - ```python - from collections import namedtuple - - import tensorflow as tf - from tensorflow.python.framework import graph_io - - import model_utils - import xlnet - - LENGTHS = 50 - BATCH = 1 - OUTPUT_DIR = '~/XLNet-Base/try_save/' - INIT_CKPT_PATH = '~/XLNet-Base/xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt' - XLNET_CONFIG_PATH = '~/XLNet-Base/xlnet_cased_L-12_H-768_A-12/xlnet_config.json' - FLags = namedtuple('FLags', 'use_tpu init_checkpoint') - FLAGS = FLags(use_tpu=False, init_checkpoint=INIT_CKPT_PATH) - - xlnet_config = xlnet.XLNetConfig(json_path=XLNET_CONFIG_PATH) - run_config = xlnet.RunConfig(is_training=False, use_tpu=False, use_bfloat16=False, dropout=0.1, dropatt=0.1,) - - - sentence_features_input_idx = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='input_ids') - sentence_features_segment_ids = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='seg_ids') - sentence_features_input_mask = tf.compat.v1.placeholder(tf.float32, shape=[LENGTHS, BATCH], name='input_mask') - - with tf.compat.v1.Session() as sess: - xlnet_model = xlnet.XLNetModel(xlnet_config=xlnet_config, run_config=run_config, - input_ids=sentence_features_input_idx, - seg_ids=sentence_features_segment_ids, - input_mask=sentence_features_input_mask) - - sess.run(tf.compat.v1.global_variables_initializer()) - model_utils.init_from_checkpoint(FLAGS, True) - - # Save the variables to disk. - saver = tf.compat.v1.train.Saver() - - # Saving checkpoint - save_path = saver.save(sess, OUTPUT_DIR + "model.ckpt") - - # Freezing model - outputs = ['model/transformer/dropout_2/Identity'] - graph_def_freezed = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph.as_graph_def(), outputs) - - # Saving non-frozen and frozen model to pb - graph_io.write_graph(sess.graph.as_graph_def(), OUTPUT_DIR, 'model.pb', as_text=False) - graph_io.write_graph(graph_def_freezed,OUTPUT_DIR, 'model_frozen.pb', - as_text=False) - - # Write to tensorboard - with tf.compat.v1.summary.FileWriter(logdir=OUTPUT_DIR, graph_def=graph_def_freezed) as writer: - writer.flush() - ``` - -## Downloading the Pretrained Large XLNet Model -Download and unzip an archive with the [XLNet-Large, Cased](https://storage.googleapis.com/xlnet/released_models/cased_L-24_H-1024_A-16.zip). - -After unzipping the archive, the directory `cased_L-12_H-1024_A-16` is created and contains the following files: - -* TensorFlow checkpoint (`xlnet_model.ckpt`) containing the pretrained weights (which is actually 3 files) -* sentence piece model (`spiece.model`) used for (de)tokenization -* config file (`xlnet_config.json`) which specifies the hyperparameters of the model +2. Save and run the following Python script in `~/XLNet-Base/xlnet`: -To get `pb-file` from the archive contents, follow the instructions below: + .. note:: The original model repository has been tested with TensorFlow 1.13.1 under Python2. + + .. code-block:: python + + from collections import namedtuple + + import tensorflow as tf + from tensorflow.python.framework import graph_io + + import model_utils + import xlnet + + LENGTHS = 50 + BATCH = 1 + OUTPUT_DIR = '~/XLNet-Base/try_save/' + INIT_CKPT_PATH = '~/XLNet-Base/xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt' + XLNET_CONFIG_PATH = '~/XLNet-Base/xlnet_cased_L-12_H-768_A-12/xlnet_config.json' + + FLags = namedtuple('FLags', 'use_tpu init_checkpoint') + FLAGS = FLags(use_tpu=False, init_checkpoint=INIT_CKPT_PATH) + + xlnet_config = xlnet.XLNetConfig(json_path=XLNET_CONFIG_PATH) + run_config = xlnet.RunConfig(is_training=False, use_tpu=False, use_bfloat16=False, dropout=0.1, dropatt=0.1,) + + + sentence_features_input_idx = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='input_ids') + sentence_features_segment_ids = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='seg_ids') + sentence_features_input_mask = tf.compat.v1.placeholder(tf.float32, shape=[LENGTHS, BATCH], name='input_mask') + + with tf.compat.v1.Session() as sess: + xlnet_model = xlnet.XLNetModel(xlnet_config=xlnet_config, run_config=run_config, + input_ids=sentence_features_input_idx, + seg_ids=sentence_features_segment_ids, + input_mask=sentence_features_input_mask) + + sess.run(tf.compat.v1.global_variables_initializer()) + model_utils.init_from_checkpoint(FLAGS, True) + + # Save the variables to disk. + saver = tf.compat.v1.train.Saver() + + # Saving checkpoint + save_path = saver.save(sess, OUTPUT_DIR + "model.ckpt") + + # Freezing model + outputs = ['model/transformer/dropout_2/Identity'] + graph_def_freezed = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph.as_graph_def(), outputs) + + # Saving non-frozen and frozen model to pb + graph_io.write_graph(sess.graph.as_graph_def(), OUTPUT_DIR, 'model.pb', as_text=False) + graph_io.write_graph(graph_def_freezed,OUTPUT_DIR, 'model_frozen.pb', + as_text=False) + + # Write to tensorboard + with tf.compat.v1.summary.FileWriter(logdir=OUTPUT_DIR, graph_def=graph_def_freezed) as writer: + writer.flush() + +Downloading the Pretrained Large XLNet Model +############################################ + +Download and unzip an archive with the `XLNet-Base, Cased `__. + +After unzipping the archive, the directory ``cased_L-12_H-1024_A-16`` is created and contains the following files: + +* TensorFlow checkpoint (``xlnet_model.ckpt``) containing the pretrained weights (which is actually 3 files) +* sentence piece model (``spiece.model``) used for (de)tokenization +* config file (``xlnet_config.json``) which specifies the hyperparameters of the model + +To get ``pb-file`` from the archive contents, follow the instructions below: 1. Run commands - ```sh + .. code-block:: sh + cd ~ mkdir XLNet-Large cd XLNet-Large @@ -116,74 +122,76 @@ To get `pb-file` from the archive contents, follow the instructions below: wget https://storage.googleapis.com/xlnet/released_models/cased_L-24_H-1024_A-16.zip unzip cased_L-24_H-1024_A-16.zip mkdir try_save - ``` - -2. Save and run the following Python script in `~/XLNet-Large/xlnet`: - - ```python - from collections import namedtuple - - import tensorflow as tf - from tensorflow.python.framework import graph_io - - import model_utils - import xlnet - - LENGTHS = 50 - BATCH = 1 - OUTPUT_DIR = '~/XLNet-Large/try_save' - INIT_CKPT_PATH = '~/XLNet-Large/cased_L-24_H-1024_A-16/xlnet_model.ckpt' - XLNET_CONFIG_PATH = '~/XLNet-Large/cased_L-24_H-1024_A-16/xlnet_config.json' - - FLags = namedtuple('FLags', 'use_tpu init_checkpoint') - FLAGS = FLags(use_tpu=False, init_checkpoint=INIT_CKPT_PATH) - - xlnet_config = xlnet.XLNetConfig(json_path=XLNET_CONFIG_PATH) - run_config = xlnet.RunConfig(is_training=False, use_tpu=False, use_bfloat16=False, dropout=0.1, dropatt=0.1,) - - sentence_features_input_idx = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='input_ids') - sentence_features_segment_ids = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='seg_ids') - sentence_features_input_mask = tf.compat.v1.placeholder(tf.float32, shape=[LENGTHS, BATCH], name='input_mask') - - with tf.compat.v1.Session() as sess: - xlnet_model = xlnet.XLNetModel(xlnet_config=xlnet_config, run_config=run_config, - input_ids=sentence_features_input_idx, - seg_ids=sentence_features_segment_ids, - input_mask=sentence_features_input_mask) - - sess.run(tf.compat.v1.global_variables_initializer()) - model_utils.init_from_checkpoint(FLAGS, True) - - # Save the variables to disk. - saver = tf.compat.v1.train.Saver() - - # Saving checkpoint - save_path = saver.save(sess, OUTPUT_DIR + "model.ckpt") - - # Freezing model - outputs = ['model/transformer/dropout_2/Identity'] - graph_def_freezed = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph.as_graph_def(), outputs) - - # Saving non-frozen and frozen model to pb - graph_io.write_graph(sess.graph.as_graph_def(), OUTPUT_DIR, 'model.pb', as_text=False) - graph_io.write_graph(graph_def_freezed,OUTPUT_DIR, 'model_frozen.pb', - as_text=False) - - # Write to tensorboard - with tf.compat.v1.summary.FileWriter(logdir=OUTPUT_DIR, graph_def=graph_def_freezed) as writer: - writer.flush() - ``` - -The script should save into `~/XLNet-Large/xlnet`. - -## Converting a frozen TensorFlow XLNet Model to IR +2. Save and run the following Python script in ``~/XLNet-Large/xlnet``: + + .. code-block:: python + + from collections import namedtuple + + import tensorflow as tf + from tensorflow.python.framework import graph_io + + import model_utils + import xlnet + + LENGTHS = 50 + BATCH = 1 + OUTPUT_DIR = '~/XLNet-Large/try_save' + INIT_CKPT_PATH = '~/XLNet-Large/cased_L-24_H-1024_A-16/xlnet_model.ckpt' + XLNET_CONFIG_PATH = '~/XLNet-Large/cased_L-24_H-1024_A-16/xlnet_config.json' + + FLags = namedtuple('FLags', 'use_tpu init_checkpoint') + FLAGS = FLags(use_tpu=False, init_checkpoint=INIT_CKPT_PATH) + + xlnet_config = xlnet.XLNetConfig(json_path=XLNET_CONFIG_PATH) + run_config = xlnet.RunConfig(is_training=False, use_tpu=False, use_bfloat16=False, dropout=0.1, dropatt=0.1,) + + + sentence_features_input_idx = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='input_ids') + sentence_features_segment_ids = tf.compat.v1.placeholder(tf.int32, shape=[LENGTHS, BATCH], name='seg_ids') + sentence_features_input_mask = tf.compat.v1.placeholder(tf.float32, shape=[LENGTHS, BATCH], name='input_mask') + + with tf.compat.v1.Session() as sess: + xlnet_model = xlnet.XLNetModel(xlnet_config=xlnet_config, run_config=run_config, + input_ids=sentence_features_input_idx, + seg_ids=sentence_features_segment_ids, + input_mask=sentence_features_input_mask) + + sess.run(tf.compat.v1.global_variables_initializer()) + model_utils.init_from_checkpoint(FLAGS, True) + + # Save the variables to disk. + saver = tf.compat.v1.train.Saver() + + # Saving checkpoint + save_path = saver.save(sess, OUTPUT_DIR + "model.ckpt") + + # Freezing model + outputs = ['model/transformer/dropout_2/Identity'] + graph_def_freezed = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph.as_graph_def(), outputs) + + # Saving non-frozen and frozen model to pb + graph_io.write_graph(sess.graph.as_graph_def(), OUTPUT_DIR, 'model.pb', as_text=False) + graph_io.write_graph(graph_def_freezed,OUTPUT_DIR, 'model_frozen.pb', + as_text=False) + + # Write to tensorboard + with tf.compat.v1.summary.FileWriter(logdir=OUTPUT_DIR, graph_def=graph_def_freezed) as writer: + writer.flush() + + +The script should save into ``~/XLNet-Large/xlnet``. + +Converting a frozen TensorFlow XLNet Model to IR +################################################# To generate the XLNet Intermediate Representation (IR) of the model, run Model Optimizer with the following parameters: -```sh -mo --input_model path-to-model/model_frozen.pb \ - --input "input_mask[50,1],input_ids[50,1],seg_ids[50,1]" -``` +.. code-block:: sh + + mo --input_model path-to-model/model_frozen.pb \ + --input "input_mask[50,1],input_ids[50,1],seg_ids[50,1]" +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_YOLO_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_YOLO_From_Tensorflow.md index 648c15df7fa104..8e2c661950dc88 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_YOLO_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_YOLO_From_Tensorflow.md @@ -1,8 +1,11 @@ # Converting TensorFlow YOLO Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow} +@sphinxdirective + This document explains how to convert real-time object detection YOLOv1, YOLOv2, YOLOv3 and YOLOv4 public models to the Intermediate Representation (IR). All YOLO models are originally implemented in the DarkNet framework and consist of two files: -* The `.cfg` file with model configurations -* The `.weights` file with model weights + +* The ``.cfg`` file with model configurations +* The ``.weights`` file with model weights Depending on a YOLO model version, the Model Optimizer converts it differently: @@ -10,232 +13,294 @@ Depending on a YOLO model version, the Model Optimizer converts it differently: - YOLOv3 has several implementations. This tutorial uses a TensorFlow implementation of YOLOv3 model, which can be directly converted to an IR. - YOLOv1 and YOLOv2 models must be first converted to TensorFlow using DarkFlow. -## Converting a YOLOv4 Model to IR +Converting a YOLOv4 Model to IR +############################### -This section explains how to convert the YOLOv4 Keras model from the [repository](https://github.com/david8862/keras-YOLOv3-model-set) to an IR. To convert the YOLOv4 model, follow the instructions below: +This section explains how to convert the YOLOv4 Keras model from the `repository `__ to an IR. To convert the YOLOv4 model, follow the instructions below: 1. Download YOLOv4 weights and associated with it cfg file: - - for YOLOv4 ([weights](https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights)/[config file](https://github.com/david8862/keras-YOLOv3-model-set/raw/6c9aff7bb0c1660704ad07c85739e95885676e5b/cfg/yolov4.cfg)) - - for YOLOv4-tiny ([weights](https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v4_pre/yolov4-tiny.weights)/[config file](https://raw.githubusercontent.com/david8862/keras-YOLOv3-model-set/6b4a0ee63771262363e8224b0ee915cad6c5e93e/cfg/yolov4-tiny.cfg)) + - for YOLOv4 ( `weights `__ / `config file `__ ) + - for YOLOv4-tiny ( `weights `__ / `config file `__ ) 2. Clone the repository with the YOLOv4 model: - ```sh - git clone https://github.com/david8862/keras-YOLOv3-model-set - ``` + .. code-block:: sh + + git clone https://github.com/david8862/keras-YOLOv3-model-set + 3. Convert the model to the TensorFlow 2 format: - - for YOLOv4: + - for YOLOv4: + + .. code-block:: sh + + python keras-YOLOv3-model-set/tools/model_converter/convert.py /yolov4.cfg /yolov4.weights + - ```sh - python keras-YOLOv3-model-set/tools/model_converter/convert.py /yolov4.cfg /yolov4.weights - ``` + - for YOLOv4-tiny: - - for YOLOv4-tiny: + .. code-block:: sh + + python keras-YOLOv3-model-set/tools/model_converter/convert.py /yolov4-tiny.cfg /yolov4-tiny.weights - ```sh - python keras-YOLOv3-model-set/tools/model_converter/convert.py /yolov4-tiny.cfg /yolov4-tiny.weights - ``` 4. Run Model Optimizer to converter the model from the TensorFlow 2 format to an IR: -> **NOTE**: Before you run the conversion, make sure you have installed all the Model Optimizer dependencies for TensorFlow 2. -> ```sh -> mo --saved_model_dir yolov4 --output_dir models/IRs --input_shape [1,608,608,3] --model_name yolov4 -> ``` + .. note:: + + Before you run the conversion, make sure you have installed all the Model Optimizer dependencies for TensorFlow 2. + + .. code-block:: sh + + mo --saved_model_dir yolov4 --output_dir models/IRs --input_shape [1,608,608,3] --model_name yolov4 -## Converting YOLOv3 Model to the OpenVINO format + +Converting YOLOv3 Model to the OpenVINO format +############################################## There are several public versions of TensorFlow YOLOv3 model implementation available on GitHub. This section explains how to convert YOLOv3 model from -the [repository](https://github.com/mystic123/tensorflow-yolo-v3) (commit ed60b90) to an IR , but the process is similar for other versions of TensorFlow YOLOv3 model. - -### Overview of YOLOv3 Model Architecture -Originally, YOLOv3 model includes feature extractor called `Darknet-53` with three branches at the end that make detections at three different scales. These branches must end with the YOLO `Region` layer. - -`Region` layer was first introduced in the DarkNet framework. Other frameworks, including TensorFlow, do not have the -`Region` implemented as a single layer, so every author of public YOLOv3 model creates it using -simple layers. This badly affects performance. For this reason, the main idea of YOLOv3 model conversion to IR is to cut off these -custom `Region`-like parts of the model and complete the model with the `Region` layers where required. - -### Dumping a YOLOv3 TensorFlow Model -To dump TensorFlow model out of [GitHub repository](https://github.com/mystic123/tensorflow-yolo-v3) (commit ed60b90), follow the instructions below: - -1. Clone the repository:
-```sh -git clone https://github.com/mystic123/tensorflow-yolo-v3.git -cd tensorflow-yolo-v3 -``` -2. (Optional) Checkout to the commit that the conversion was tested on:
-```sh -git checkout ed60b90 -``` -3. Download [coco.names](https://github.com/AlexeyAB/darknet/blob/master/data/coco.names) file from the DarkNet website **OR** use labels that fit your task. -4. Download the [yolov3.weights](https://pjreddie.com/media/files/yolov3.weights) (for the YOLOv3 model) or [yolov3-tiny.weights](https://pjreddie.com/media/files/yolov3-tiny.weights) (for the YOLOv3-tiny model) file **OR** use your pre-trained weights with the same structure. +the `repository `__ (commit ed60b90) to an IR , but the process is similar for other versions of TensorFlow YOLOv3 model. + +Overview of YOLOv3 Model Architecture ++++++++++++++++++++++++++++++++++++++ + +Originally, YOLOv3 model includes feature extractor called ``Darknet-53`` with three branches at the end that make detections at three different scales. These branches must end with the YOLO ``Region`` layer. + +``Region`` layer was first introduced in the DarkNet framework. Other frameworks, including TensorFlow, do not have the ``Region`` implemented as a single layer, so every author of public YOLOv3 model creates it using simple layers. This badly affects performance. For this reason, the main idea of YOLOv3 model conversion to IR is to cut off these custom ``Region`` -like parts of the model and complete the model with the ``Region`` layers where required. + +Dumping a YOLOv3 TensorFlow Model ++++++++++++++++++++++++++++++++++ + +To dump TensorFlow model out of `GitHub repository `__ (commit ed60b90), follow the instructions below: + +1. Clone the repository: + + .. code-block:: sh + + git clone https://github.com/mystic123/tensorflow-yolo-v3.git + cd tensorflow-yolo-v3 + + +2. (Optional) Checkout to the commit that the conversion was tested on: + + .. code-block:: sh + + git checkout ed60b90 + + +3. Download `coco.names `__ file from the DarkNet website **OR** use labels that fit your task. +4. Download the `yolov3.weights `__ (for the YOLOv3 model) or `yolov3-tiny.weights `__ (for the YOLOv3-tiny model) file **OR** use your pre-trained weights with the same structure. 5. Install PIL, which is used by the conversion script in the repo: -```sh -pip install pillow -``` + + .. code-block:: sh + + pip install pillow + + 6. Run a converter: -> **NOTE**: This converter works with TensorFlow 1.x and numpy 1.19 or lower. - -- For YOLO-v3: -```sh -python3 convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3.weights -``` -- For YOLOv3-tiny: -```sh -python3 convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3-tiny.weights --tiny -``` -At this step, you may receive a warning like `WARNING:tensorflow:Entity <...> could not be transformed and will be executed as-is.`. To work around this issue, switch to gast 0.2.2 with the following command: -```sh -pip3 install --user gast==0.2.2 -``` - -If you have YOLOv3 weights trained for an input image with the size different from 416 (320, 608 or your own), provide the `--size` key with the size of your image specified while running the converter. For example, run the following command for an image with size 608: -```sh -python3 convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3_608.weights --size 608 -``` - -### Converting a YOLOv3 TensorFlow Model to the OpenVINO format - -To solve the problems explained in the YOLOv3 architecture overview section, use the `yolo_v3.json` or `yolo_v3_tiny.json` (depending on a model) configuration file with custom operations located in the `/tools/model_optimizer/extensions/front/tf` repository. - -It consists of several attributes:
-```sh -[ - { - "id": "TFYOLOV3", - "match_kind": "general", - "custom_attributes": { - "classes": 80, - "anchors": [10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326], - "coords": 4, - "num": 9, - "masks":[[6, 7, 8], [3, 4, 5], [0, 1, 2]], - "entry_points": ["detector/yolo-v3/Reshape", "detector/yolo-v3/Reshape_4", "detector/yolo-v3/Reshape_8"] - } - } -] -``` + + .. note:: This converter works with TensorFlow 1.x and numpy 1.19 or lower. + + + - For YOLO-v3: + + .. code-block:: sh + + python3 convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3.weights + + + - For YOLOv3-tiny: + + .. code-block:: sh + + python3 convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3-tiny.weights --tiny + + + At this step, you may receive a warning like ``WARNING:tensorflow:Entity <...> could not be transformed and will be executed as-is.``. To work around this issue, switch to gast 0.2.2 with the following command: + + .. code-block:: sh + + pip3 install --user gast==0.2.2 + + +If you have YOLOv3 weights trained for an input image with the size different from 416 (320, 608 or your own), provide the ``--size`` key with the size of your image specified while running the converter. For example, run the following command for an image with size 608: + +.. code-block:: sh + + python3 convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3_608.weights --size 608 + + +Converting a YOLOv3 TensorFlow Model to the OpenVINO format ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +To solve the problems explained in the `YOLOv3 architecture overview <#overview-of-yolov3-model-architecture>`__ section, use the ``yolo_v3.json`` or ``yolo_v3_tiny.json`` (depending on a model) configuration file with custom operations located in the ``/tools/model_optimizer/extensions/front/tf`` repository. + +It consists of several attributes: + +.. code-block:: sh + + [ + { + "id": "TFYOLOV3", + "match_kind": "general", + "custom_attributes": { + "classes": 80, + "anchors": [10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326], + "coords": 4, + "num": 9, + "masks":[[6, 7, 8], [3, 4, 5], [0, 1, 2]], + "entry_points": ["detector/yolo-v3/Reshape", "detector/yolo-v3/Reshape_4", "detector/yolo-v3/Reshape_8"] + } + } + ] + + where: -- `id` and `match_kind` are parameters that you cannot change. -- `custom_attributes` is a parameter that stores all the YOLOv3 specific attributes: - - `classes`, `coords`, `num`, and `masks` are attributes that you should copy from the configuration - file that was used for model training. If you used DarkNet officially shared weights, - you can use `yolov3.cfg` or `yolov3-tiny.cfg` configuration file from [GitHub repository](https://github.com/david8862/keras-YOLOv3-model-set/tree/master/cfg). Replace the default values in `custom_attributes` with the parameters that - follow the `[yolo]` titles in the configuration file. - - `anchors` is an optional parameter that is not used while inference of the model, but it used in a demo to parse `Region` layer output - - `entry_points` is a node name list to cut off the model and append the `Region` layer with custom attributes specified above. - - -To generate an IR of the YOLOv3 TensorFlow model, run:
-```sh - mo \ ---input_model /path/to/yolo_v3.pb \ ---transformations_config front/tf/yolo_v3.json \ ---batch 1 \ ---output_dir -``` - -To generate an IR of the YOLOv3-tiny TensorFlow model, run:
-```sh - mo \ ---input_model /path/to/yolo_v3_tiny.pb \ ---transformations_config front/tf/yolo_v3_tiny.json \ ---batch 1 \ ---output_dir -``` + +- ``id`` and ``match_kind`` are parameters that you cannot change. +- ``custom_attributes`` is a parameter that stores all the YOLOv3 specific attributes: + + - ``classes``, ``coords``, ``num``, and ``masks`` are attributes that you should copy from the configuration file that was used for model training. If you used DarkNet officially shared weights, you can use ``yolov3.cfg`` or ``yolov3-tiny.cfg`` configuration file from `GitHub repository `__. eplace the default values in ``custom_attributes`` with the parameters that follow the ``[yolo]`` titles in the configuration file. + - ``anchors`` is an optional parameter that is not used while inference of the model, but it used in a demo to parse ``Region`` layer output + - ``entry_points`` is a node name list to cut off the model and append the ``Region`` layer with custom attributes specified above. + + +To generate an IR of the YOLOv3 TensorFlow model, run: + +.. code-block:: sh + + mo \ + --input_model /path/to/yolo_v3.pb \ + --transformations_config front/tf/yolo_v3.json \ + --batch 1 \ + --output_dir + + +To generate an IR of the YOLOv3-tiny TensorFlow model, run: + +.. code-block:: sh + + mo \ + --input_model /path/to/yolo_v3_tiny.pb \ + --transformations_config front/tf/yolo_v3_tiny.json \ + --batch 1 \ + --output_dir + where: -* `--batch` defines shape of model input. In the example, `--batch` is equal to 1, but you can also specify other integers larger than 1. -* `--transformations_config` adds missing `Region` layers to the model. In the IR, the `Region` layer has name `RegionYolo`. +* ``--batch`` defines shape of model input. In the example, ``--batch`` is equal to 1, but you can also specify other integers larger than 1. +* ``--transformations_config`` adds missing ``Region`` layers to the model. In the IR, the ``Region`` layer has name ``RegionYolo``. + +.. note:: + + The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the ``RGB<->BGR`` conversion specifying the command-line parameter: ``--reverse_input_channels``. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the :doc:`Converting a Model to Intermediate Representation (IR) ` guide. -> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the [Converting a Model to Intermediate Representation (IR)](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. -OpenVINO toolkit provides a demo that uses YOLOv3 model. Refer to the [Object Detection C++ Demo](@ref omz_demos_object_detection_demo_cpp) for more information. +OpenVINO toolkit provides a demo that uses YOLOv3 model. Refer to the :doc:`Object Detection C++ Demo ` for more information. -## Converting YOLOv1 and YOLOv2 Models to the IR +Converting YOLOv1 and YOLOv2 Models to the IR +############################################# Before converting, choose a YOLOv1 or YOLOv2 model version that best suits your task. Download model configuration file and corresponding weight file: -* From [DarkFlow repository](https://github.com/thtrieu/darkflow): configuration files are stored in the `cfg` directory, links to weight files are given in the `README.md` file. The files from this repository are adapted for conversion to TensorFlow using DarkFlow. -* From DarkNet website and repository: configuration files are stored in the `cfg` directory of the [repository](https://github.com/pjreddie/darknet), links to weight files are given on the [YOLOv1](https://pjreddie.com/darknet/yolov1/) and [YOLOv2](https://pjreddie.com/darknet/yolov2/) websites. + +* From `DarkFlow repository `__ : configuration files are stored in the ``cfg`` directory, links to weight files are given in the ``README.md`` file. The files from this repository are adapted for conversion to TensorFlow using DarkFlow. +* From DarkNet website and repository: configuration files are stored in the ``cfg`` directory of the `repository `__, links to weight files are given on the `YOLOv1 `__ and `YOLOv2 `__ websites. To convert DarkNet YOLOv1 and YOLOv2 models to the OpenVINO format, follow these steps: -1. [Install DarkFlow](@ref install_darkflow) -2. [Convert DarkNet YOLOv1 or YOLOv2 model to TensorFlow](@ref yolov1_v2_to_tf) using DarkFlow -3. [Convert TensorFlow YOLOv1 or YOLOv2 model to IR](@ref yolov1_v2_to_ir) +1. `Install DarkFlow <#installing-darkflow>`__ +2. `Convert DarkNet YOLOv1 or YOLOv2 model to TensorFlow <#converting-a-darknet-yolov1-or-yolov2-model-to-tensorflow>`__ using DarkFlow +3. `Convert TensorFlow YOLOv1 or YOLOv2 model to IR <#converting-a-tensorflow-yolov1-or-yolov2-model-to-the-ir>`__ + -@anchor install_darkflow -#### Installing DarkFlow +Installing DarkFlow +-------------------------------------------------------------- You need DarkFlow to convert YOLOv1 and YOLOv2 models to TensorFlow. To install DarkFlow: -1. Install DarkFlow [required dependencies](https://github.com/thtrieu/darkflow#dependencies). -2. Clone DarkFlow git repository:
-```sh -git clone https://github.com/thtrieu/darkflow.git -``` -3. Go to the root directory of the cloned repository:
-```sh -cd darkflow -``` -4. Install DarkFlow, using the instructions from the `README.md` file in the [DarkFlow repository](https://github.com/thtrieu/darkflow/blob/master/README.md#getting-started). - -@anchor yolov1_v2_to_tf -#### Converting a DarkNet YOLOv1 or YOLOv2 Model to TensorFlow - -To convert YOLOv1 or YOLOv2 model to TensorFlow, go to the root directory of the cloned DarkFlow repository, place the previously downloaded \*.cfg and \*.weights files in the current directory and run the following command:
+ +1. Install DarkFlow `required dependencies `__. +2. Clone DarkFlow git repository: + + .. code-block:: sh + + git clone https://github.com/thtrieu/darkflow.git + + +3. Go to the root directory of the cloned repository: + + .. code-block:: sh + + cd darkflow + + +4. Install DarkFlow, using the instructions from the ``README.md`` file in the `DarkFlow repository `__. + + +Converting a DarkNet YOLOv1 or YOLOv2 Model to TensorFlow +-------------------------------------------------------------- + +To convert YOLOv1 or YOLOv2 model to TensorFlow, go to the root directory of the cloned DarkFlow repository, place the previously downloaded \*.cfg and \*.weights files in the current directory and run the following command: + - For YOLOv1: -```sh -python3 flow --model yolov1.cfg --load yolov1.weights --savepb -``` - -- For YOLOv2 with VOC dataset `--labels` argument should be specified and additional changes in the original exporting script are required. -In the [file](https://github.com/thtrieu/darkflow/blob/b187c65630f9aa1bb8b809c33ec67c8cc5d60124/darkflow/utils/loader.py#L121) -change line 121 from `self.offset = 16` to `self.offset = 20`. Then run: -```sh -python3 flow --model yolov2-voc.cfg --load yolov2-voc.weights --labels voc-labels.txt --savepb -``` -VOC labels can be found on the following [link](https://raw.githubusercontent.com/szaza/android-yolo-v2/master/assets/tiny-yolo-voc-labels.txt) + + .. code-block:: sh + + python3 flow --model yolov1.cfg --load yolov1.weights --savepb + + +- For YOLOv2 with VOC dataset ``--labels`` argument should be specified and additional changes in the original exporting script are required. In the `file `__ change line 121 from ``self.offset = 16`` to ``self.offset = 20``. Then run: + + .. code-block:: sh + + python3 flow --model yolov2-voc.cfg --load yolov2-voc.weights --labels voc-labels.txt --savepb + + +VOC labels can be found on the following `link `__ General conversion command is: -```sh -python3 flow --model /.cfg --load /.weights --labels --savepb -``` -For YOLOv1, the `--labels` argument can be skipped. If the model was successfully converted, you can find the `.meta` and `.pb` files. -in `built_graph` subdirectory of the cloned DarkFlow repository. - -File `.pb` is a TensorFlow representation of the YOLO model. - -@anchor yolov1_v2_to_ir -#### Converting a TensorFlow YOLOv1 or YOLOv2 Model to the IR - -Converted TensorFlow YOLO model is missing `Region` layer and its parameters. Original YOLO `Region` layer parameters are stored in the configuration `/.cfg` -file under the `[region]` title. - -To recreate the original model structure, use the corresponding yolo `.json` configuration file with custom operations and `Region` layer -parameters when converting the model to the IR. This file is located in the `/tools/model_optimizer/extensions/front/tf` directory. - -If chosen model has specific values of these parameters, -create another configuration file with custom operations and use it for conversion. - -To generate the IR of the YOLOv1 model, provide TensorFlow YOLOv1 or YOLOv2 model to Model Optimizer with the following parameters:
-```sh - mo ---input_model /.pb \ ---batch 1 \ ---scale 255 \ ---transformations_config front/tf/.json -``` + +.. code-block:: sh + + python3 flow --model /.cfg --load /.weights --labels --savepb + + +For YOLOv1, the ``--labels`` argument can be skipped. If the model was successfully converted, you can find the ``.meta`` and ``.pb`` files. +in ``built_graph`` subdirectory of the cloned DarkFlow repository. + +File ``.pb`` is a TensorFlow representation of the YOLO model. + +Converting a TensorFlow YOLOv1 or YOLOv2 Model to the IR +--------------------------------------------------------- + +Converted TensorFlow YOLO model is missing ``Region`` layer and its parameters. Original YOLO ``Region`` layer parameters are stored in the configuration ``/.cfg`` file under the ``[region]`` title. + +To recreate the original model structure, use the corresponding yolo ``.json`` configuration file with custom operations and ``Region`` layer parameters when converting the model to the IR. This file is located in the ``/tools/model_optimizer/extensions/front/tf`` directory. + +If chosen model has specific values of these parameters, create another configuration file with custom operations and use it for conversion. + +To generate the IR of the YOLOv1 model, provide TensorFlow YOLOv1 or YOLOv2 model to Model Optimizer with the following parameters: + +.. code-block:: sh + + mo + --input_model /.pb \ + --batch 1 \ + --scale 255 \ + --transformations_config front/tf/.json + + where: -* `--batch` defines shape of model input. In the example, `--batch` is equal to 1, but you can also specify other integers larger than 1. -* `--scale` specifies scale factor that input values will be divided by. -The model was trained with input values in the range `[0,1]`. OpenVINO toolkit samples read input images as values in `[0,255]` range, so the scale 255 must be applied. -* `--transformations_config` adds missing `Region` layers to the model. In the IR, the `Region` layer has name `RegionYolo`. -For other applicable parameters, refer to the [Convert Model from TensorFlow](@ref openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow) guide. +* ``--batch`` defines shape of model input. In the example, ``--batch`` is equal to 1, but you can also specify other integers larger than 1. +* ``--scale`` specifies scale factor that input values will be divided by. The model was trained with input values in the range ``[0,1]``. OpenVINO toolkit samples read input images as values in ``[0,255]`` range, so the scale 255 must be applied. +* ``--transformations_config`` adds missing ``Region`` layers to the model. In the IR, the ``Region`` layer has name ``RegionYolo``. For other applicable parameters, refer to the :doc:`Convert Model from TensorFlow ` guide. + +.. note:: + + The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the ``RGB<->BGR`` conversion specifying the command-line parameter: ``--reverse_input_channels``. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the :doc:`Converting a Model to Intermediate Representation (IR) ` guide. + -> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the [Converting a Model to Intermediate Representation (IR)](@ref openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model) guide. +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_lm_1b_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_lm_1b_From_Tensorflow.md index 5c441717b02f8b..7cd7ccf23c4194 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_lm_1b_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_lm_1b_From_Tensorflow.md @@ -1,100 +1,121 @@ # Converting a TensorFlow Language Model on One Billion Word Benchmark {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_lm_1b_From_Tensorflow} -## Downloading a Pre-trained Language Model on One Billion Word Benchmark +@sphinxdirective -TensorFlow provides a pretrained [Language Model on One Billion Word Benchmark](https://github.com/tensorflow/models/tree/r2.3.0/research/lm_1b). +Downloading a Pre-trained Language Model on One Billion Word Benchmark +###################################################################### + +TensorFlow provides a pretrained `Language Model on One Billion Word Benchmark `__. To download the model for IR conversion, follow the instructions: + 1. Create new directory to store the model: -```shell -mkdir lm_1b -``` -2. Go to the `lm_1b` directory: -```shell -cd lm_1b -``` + +.. code-block:: shell + + mkdir lm_1b + +2. Go to the ``lm_1b`` directory: + +.. code-block:: shell + + cd lm_1b + 3. Download the model GraphDef file: -``` -wget http://download.tensorflow.org/models/LM_LSTM_CNN/graph-2016-09-10.pbtxt -``` + +.. code-block:: shell + + wget http://download.tensorflow.org/models/LM_LSTM_CNN/graph-2016-09-10.pbtxt + 4. Create new directory to store 12 checkpoint shared files: -```shell -mkdir ckpt -``` -5. Go to the `ckpt` directory: -```shell -cd ckpt -``` + +.. code-block:: shell + + mkdir ckpt + +5. Go to the ``ckpt`` directory: + +.. code-block:: shell + + cd ckpt + 6. Download 12 checkpoint shared files: -``` -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-base -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-char-embedding -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-lstm -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax0 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax1 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax2 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax3 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax4 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax5 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax6 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax7 -wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax8 -``` - -Once you have downloaded the pretrained model files, you will have the `lm_1b` directory with the following hierarchy: - -``` -lm_1b/ - graph-2016-09-10.pbtxt - ckpt/ - ckpt-base - ckpt-char-embedding - ckpt-lstm - ckpt-softmax0 - ckpt-softmax1 - ckpt-softmax2 - ckpt-softmax3 - ckpt-softmax4 - ckpt-softmax5 - ckpt-softmax6 - ckpt-softmax7 - ckpt-softmax8 -``` - - -![lm_1b model view](../../../img/lm_1b.svg) - -The frozen model still has two variables: `Variable` and `Variable_1`. + +.. code-block:: shell + + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-base + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-char-embedding + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-lstm + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax0 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax1 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax2 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax3 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax4 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax5 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax6 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax7 + wget http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax8 + + +Once you have downloaded the pretrained model files, you will have the ``lm_1b`` directory with the following hierarchy: + +.. code-block:: shell + + lm_1b/ + graph-2016-09-10.pbtxt + ckpt/ + ckpt-base + ckpt-char-embedding + ckpt-lstm + ckpt-softmax0 + ckpt-softmax1 + ckpt-softmax2 + ckpt-softmax3 + ckpt-softmax4 + ckpt-softmax5 + ckpt-softmax6 + ckpt-softmax7 + ckpt-softmax8 + + + +.. image:: ./_static/images/lm_1b.svg + +The frozen model still has two variables: ``Variable`` and ``Variable_1``. It means that the model keeps training those variables at each inference. At the first inference of this graph, the variables are initialized by initial values. -After executing the `lstm` nodes, results of execution are assigned to these two variables. +After executing the ``lstm`` nodes, results of execution are assigned to these two variables. -With each inference of the `lm_1b` graph, `lstm` initial states data is taken from previous inference -from variables, and states of current inference of `lstm` is reassigned to the same variables. +With each inference of the ``lm_1b`` graph, ``lstm`` initial states data is taken from previous inference +from variables, and states of current inference of ``lstm`` is reassigned to the same variables. It helps the model to remember the context of the words that it takes as input. -## Converting a TensorFlow Language Model on One Billion Word Benchmark to IR +Converting a TensorFlow Language Model on One Billion Word Benchmark to IR +########################################################################## Model Optimizer assumes that output model is for inference only. Therefore, you should cut those variables off and resolve keeping cell and hidden states on application level. There is a certain limitation for the model conversion: the original model cannot be reshaped, so you should keep original shapes. -To generate the `lm_1b` Intermediate Representation (IR), provide TensorFlow `lm_1b` model to the +To generate the ``lm_1b`` Intermediate Representation (IR), provide TensorFlow ``lm_1b`` model to the Model Optimizer with parameters: -```sh - mo ---input_model lm_1b/graph-2016-09-10.pbtxt \ ---input_checkpoint lm_1b/ckpt \ ---input_model_is_text \ ---input_shape [50],[50],[1,9216],[1,9216] \ ---output softmax_out,lstm/lstm_0/concat_2,lstm/lstm_1/concat_2 \ ---input char_embedding/EmbeddingLookupUnique/Unique:0,char_embedding/EmbeddingLookupUnique/Unique:1,Variable/read,Variable_1/read -``` + +.. code-block:: shell + + mo + --input_model lm_1b/graph-2016-09-10.pbtxt \ + --input_checkpoint lm_1b/ckpt \ + --input_model_is_text \ + --input_shape [50],[50],[1,9216],[1,9216] \ + --output softmax_out,lstm/lstm_0/concat_2,lstm/lstm_1/concat_2 \ + --input char_embedding/EmbeddingLookupUnique/Unique:0,char_embedding/EmbeddingLookupUnique/Unique:1,Variable/read,Variable_1/read Where: -* `--input char_embedding/EmbeddingLookupUnique/Unique:0,char_embedding/EmbeddingLookupUnique/Unique:1,Variable/read,Variable_1/read` - and `--input_shape [50],[50],[1,9216],[1,9216]` replace the variables with a placeholder. -* `--output softmax_out,lstm/lstm_0/concat_2,lstm/lstm_1/concat_2` specifies output node name and names of LSTM cell states. + +* ``--input char_embedding/EmbeddingLookupUnique/Unique:0,char_embedding/EmbeddingLookupUnique/Unique:1,Variable/read,Variable_1/read`` and ``--input_shape [50],[50],[1,9216],[1,9216]`` replace the variables with a placeholder. +* ``--output softmax_out,lstm/lstm_0/concat_2,lstm/lstm_1/concat_2`` specifies output node name and names of LSTM cell states. + +@endsphinxdirective diff --git a/docs/MO_DG/prepare_model/model_inputs_outputs.md b/docs/MO_DG/prepare_model/model_inputs_outputs.md index f64f6aa6719029..abd8e36da5096f 100644 --- a/docs/MO_DG/prepare_model/model_inputs_outputs.md +++ b/docs/MO_DG/prepare_model/model_inputs_outputs.md @@ -1,13 +1,17 @@ # Model Inputs and Outputs, Shapes and Layouts {#openvino_docs_model_inputs_outputs} -Users interact with a model by passing data to its _inputs_ before the inference and retrieving data from its _outputs_ after the inference. A model may have one or multiple inputs and outputs. Normally, in OpenVINO™ toolkit, all inputs and outputs in the converted model are identified in the same way as in the original framework model. +@sphinxdirective -OpenVINO uses the _names of tensors_ for identification. Depending on the framework, the names of tensors are formed differently. +Users interact with a model by passing data to its *inputs* before the inference and retrieving data from its *outputs* after the inference. A model may have one or multiple inputs and outputs. Normally, in OpenVINO™ toolkit, all inputs and outputs in the converted model are identified in the same way as in the original framework model. -A model accepts inputs and produces outputs of some _shape_. Shape defines the number of dimensions in a tensor and their order. For example, an image classification model can accept tensor of shape [1, 3, 240, 240] and produces tensor of shape [1, 1000]. +OpenVINO uses the *names of tensors* for identification. Depending on the framework, the names of tensors are formed differently. -The meaning of each dimension in the shape is specified by its _layout_. Layout is an interpretation of shape dimensions. OpenVINO toolkit conversion tools and APIs keep all dimensions and their order unchanged and aligned with the original framework model. Usually, original models do not contain layout information explicitly, but in various pre-processing and post-processing scenarios in the OpenVINO Runtime API, sometimes it is required to have the layout specified explicitly. We recommend specifying layouts for inputs/outputs during the model conversion. +A model accepts inputs and produces outputs of some *shape*. Shape defines the number of dimensions in a tensor and their order. For example, an image classification model can accept tensor of shape [1, 3, 240, 240] and produces tensor of shape [1, 1000]. -OpenVINO also supports _partially defined shapes_, where part of the dimensions is undefined. Undefined dimensions are also kept intact in the final IR file and you can define them later, during runtime. Undefined dimensions can be used as [dynamic dimensions](../../OV_Runtime_UG/ov_dynamic_shapes.md) for certain hardware and models, which enables you to change shapes of input data dynamically in each infer request. For example, the sequence length dimension in the BERT model can be left undefined and variously sized data along this dimension can be fed on the CPU. +The meaning of each dimension in the shape is specified by its *layout*. Layout is an interpretation of shape dimensions. OpenVINO toolkit conversion tools and APIs keep all dimensions and their order unchanged and aligned with the original framework model. Usually, original models do not contain layout information explicitly, but in various pre-processing and post-processing scenarios in the OpenVINO Runtime API, sometimes it is required to have the layout specified explicitly. We recommend specifying layouts for inputs/outputs during the model conversion. -To learn about how the model is represented in OpenVINO™ Runtime, see the [Model Representation in OpenVINO™ Runtime](../../OV_Runtime_UG/model_representation.md). \ No newline at end of file +OpenVINO also supports *partially defined shapes*, where part of the dimensions is undefined. Undefined dimensions are also kept intact in the final IR file and you can define them later, during runtime. Undefined dimensions can be used as :doc:`dynamic dimensions ` for certain hardware and models, which enables you to change shapes of input data dynamically in each infer request. For example, the sequence length dimension in the BERT model can be left undefined and variously sized data along this dimension can be fed on the CPU. + +To learn about how the model is represented in OpenVINO™ Runtime, see the :doc:`Model Representation in OpenVINO™ Runtime `. + +@endsphinxdirective \ No newline at end of file diff --git a/docs/OV_Runtime_UG/Samples_Overview.md b/docs/OV_Runtime_UG/Samples_Overview.md index 65e81ea9b4c8e9..6643ad288104bf 100644 --- a/docs/OV_Runtime_UG/Samples_Overview.md +++ b/docs/OV_Runtime_UG/Samples_Overview.md @@ -32,69 +32,91 @@ openvino_inference_engine_samples_benchmark_app_README openvino_inference_engine_tools_benchmark_tool_README -@endsphinxdirective The OpenVINO™ samples are simple console applications that show how to utilize specific OpenVINO API capabilities within an application. They can assist you in executing specific tasks such as loading a model, running inference, querying specific device capabilities, etc. If you installed OpenVINO Runtime via archive files, sample applications for С, and C++, and Python are created in the following directories: -* `/samples/c` -* `/samples/cpp` -* `/samples/python` -If you installed OpenVINO via PyPI, download [the OpenVINO repository](https://github.com/openvinotoolkit/openvino/) and use samples from `samples/python`. +* ``/samples/c`` +* ``/samples/cpp`` +* ``/samples/python`` + +If you installed OpenVINO via PyPI, download `the OpenVINO repository `__ and use samples from ``samples/python``. The applications include: - **Speech Sample** - Acoustic model inference based on Kaldi neural networks and speech feature vectors. - - [Automatic Speech Recognition C++ Sample](../../samples/cpp/speech_sample/README.md) - - [Automatic Speech Recognition Python Sample](../../samples/python/speech_sample/README.md) + + - :doc:`Automatic Speech Recognition C++ Sample ` + - :doc:`Automatic Speech Recognition Python Sample ` + - **Hello Classification Sample** – Inference of image classification networks like AlexNet and GoogLeNet using Synchronous Inference Request API. Input of any size and layout can be set to an infer request which will be pre-processed automatically during inference (the sample supports only images as inputs and supports Unicode paths). - - [Hello Classification C++ Sample](../../samples/cpp/hello_classification/README.md) - - [Hello Classification C Sample](../../samples/c/hello_classification/README.md) - - [Hello Classification Python Sample](../../samples/python/hello_classification/README.md) + + - :doc:`Hello Classification C++ Sample ` + - :doc:`Hello Classification C Sample ` + - :doc:`Hello Classification Python Sample ` + - **Hello NV12 Input Classification Sample** – Input of any size and layout can be provided to an infer request. The sample transforms the input to the NV12 color format and pre-process it automatically during inference. The sample supports only images as inputs. - - [Hello NV12 Input Classification C++ Sample](../../samples/cpp/hello_nv12_input_classification/README.md) - - [Hello NV12 Input Classification C Sample](../../samples/c/hello_nv12_input_classification/README.md) + + - :doc:`Hello NV12 Input Classification C++ Sample ` + - :doc:`Hello NV12 Input Classification C Sample ` + - **Hello Query Device Sample** – Query of available OpenVINO devices and their metrics, configuration values. - - [Hello Query Device C++ Sample](../../samples/cpp/hello_query_device/README.md) - - [Hello Query Device Python* Sample](../../samples/python/hello_query_device/README.md) + + - :doc:`Hello Query Device C++ Sample ` + - :doc:`Hello Query Device Python* Sample ` + - **Hello Reshape SSD Sample** – Inference of SSD networks resized by ShapeInfer API according to an input size. - - [Hello Reshape SSD C++ Sample**](../../samples/cpp/hello_reshape_ssd/README.md) - - [Hello Reshape SSD Python Sample**](../../samples/python/hello_reshape_ssd/README.md) + + - :doc:`Hello Reshape SSD C++ Sample** ` + - :doc:`Hello Reshape SSD Python Sample** ` + - **Image Classification Sample Async** – Inference of image classification networks like AlexNet and GoogLeNet using Asynchronous Inference Request API (the sample supports only images as inputs). - - [Image Classification Async C++ Sample](../../samples/cpp/classification_sample_async/README.md) - - [Image Classification Async Python* Sample](../../samples/python/classification_sample_async/README.md) + + - :doc:`Image Classification Async C++ Sample ` + - :doc:`Image Classification Async Python* Sample ` + - **OpenVINO Model Creation Sample** – Construction of the LeNet model using the OpenVINO model creation sample. - - [OpenVINO Model Creation C++ Sample](../../samples/cpp/model_creation_sample/README.md) - - [OpenVINO Model Creation Python Sample](../../samples/python/model_creation_sample/README.md) + + - :doc:`OpenVINO Model Creation C++ Sample ` + - :doc:`OpenVINO Model Creation Python Sample ` + - **Benchmark Samples** - Simple estimation of a model inference performance - - [Sync Benchmark C++ Sample](../../samples/cpp/benchmark/sync_benchmark/README.md) - - [Sync Benchmark Python* Sample](../../samples/python/benchmark/sync_benchmark/README.md) - - [Throughput Benchmark C++ Sample](../../samples/cpp/benchmark/throughput_benchmark/README.md) - - [Throughput Benchmark Python* Sample](../../samples/python/benchmark/throughput_benchmark/README.md) - - [Bert Benchmark Python* Sample](../../samples/python/benchmark/bert_benchmark/README.md) + - :doc:`Sync Benchmark C++ Sample ` + - :doc:`Sync Benchmark Python* Sample ` + - :doc:`Throughput Benchmark C++ Sample ` + - :doc:`Throughput Benchmark Python* Sample ` + - :doc:`Bert Benchmark Python* Sample ` - **Benchmark Application** – Estimates deep learning inference performance on supported devices for synchronous and asynchronous modes. - - [Benchmark C++ Tool](../../samples/cpp/benchmark_app/README.md) + + - :doc:`Benchmark C++ Tool ` + + + Note that the Python version of the benchmark tool is currently available only through the :doc:`OpenVINO Development Tools installation `. It is not created in the samples directory but can be launched with the following command: ``benchmark_app -m -i -d ``. For more information, check the :doc:`Benchmark Python Tool ` documentation. + +.. note:: - Note that the Python version of the benchmark tool is currently available only through the [OpenVINO Development Tools installation](../install_guides/installing-model-dev-tools.md). It is not created in the samples directory but can be launched with the following command: - `benchmark_app -m -i -d ` - For more information, check the [Benchmark Python Tool](../../tools/benchmark_tool/README.md) documentation. + All C++ samples support input paths containing only ASCII characters, except for the Hello Classification Sample, that supports Unicode. -> **NOTE**: All C++ samples support input paths containing only ASCII characters, except for the Hello Classification Sample, that supports Unicode. +Media Files Available for Samples +################################# -## Media Files Available for Samples +To run the sample applications, you can use images and videos from the media files collection available `here `__ . -To run the sample applications, you can use images and videos from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. +Samples that Support Pre-Trained Models +####################################### -## Samples that Support Pre-Trained Models +To run the sample, you can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. -To run the sample, you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). +Build the Sample Applications +############################# -## Build the Sample Applications +.. _build-samples-linux: -### Build the Sample Applications on Linux +Build the Sample Applications on Linux +++++++++++++++++++++++++++++++++++++++ The officially supported Linux build environment is the following: @@ -102,76 +124,100 @@ The officially supported Linux build environment is the following: * GCC 7.5.0 (for Ubuntu 18.04) or GCC 9.3.0 (for Ubuntu 20.04) * CMake version 3.10 or higher -> **NOTE**: For building samples from the open-source version of OpenVINO toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md). +.. note:: + + For building samples from the open-source version of OpenVINO toolkit, see the `build instructions on GitHub `__ . + +To build the C or C++ sample applications for Linux, go to the ``/samples/c`` or ``/samples/cpp`` directory, respectively, and run the ``build_samples.sh`` script: -To build the C or C++ sample applications for Linux, go to the `/samples/c` or `/samples/cpp` directory, respectively, and run the `build_samples.sh` script: -```sh -build_samples.sh -``` +.. code-block:: sh + + build_samples.sh Once the build is completed, you can find sample binaries in the following folders: -* C samples: `~/openvino_c_samples_build//Release` -* C++ samples: `~/openvino_cpp_samples_build//Release` -where the is the output of `uname -m`, for example, `intel64`, `armhf`, or `aarch64`. + +* C samples: ``~/openvino_c_samples_build//Release`` +* C++ samples: ``~/openvino_cpp_samples_build//Release`` where the is the output of ``uname -m``, for example, ``intel64``, ``armhf``, or ``aarch64``. You can also build the sample applications manually: -> **NOTE**: If you have installed the product as a root user, switch to root mode before you continue: `sudo -i` +.. note:: -1. Navigate to a directory that you have write access to and create a samples build directory. This example uses a directory named `build`: -```sh -mkdir build -``` -> **NOTE**: If you run the Image Classification verification script during the installation, the C++ samples build directory is created in your home directory: `~/openvino_cpp_samples_build/` + If you have installed the product as a root user, switch to root mode before you continue: ``sudo -i`` . -2. Go to the created directory: -```sh -cd build -``` +1. Navigate to a directory that you have write access to and create a samples build directory. This example uses a directory named ``build``: + .. code-block:: sh + + mkdir build + + .. note:: + + If you run the Image Classification verification script during the installation, the C++ samples build directory is created in your home directory: ``~/openvino_cpp_samples_build/`` + +2. Go to the created directory: + + .. code-block:: sh + + cd build + 3. Run CMake to generate the Make files for release or debug configuration. For example, for C++ samples: - - For release configuration: - ```sh - cmake -DCMAKE_BUILD_TYPE=Release /samples/cpp - ``` - - For debug configuration: - ```sh - cmake -DCMAKE_BUILD_TYPE=Debug /samples/cpp - ``` -4. Run `make` to build the samples: -```sh -make -``` - -For the release configuration, the sample application binaries are in `//Release/`; -for the debug configuration — in `//Debug/`. - -### Build the Sample Applications on Microsoft Windows + + - For release configuration: + + .. code-block:: sh + + cmake -DCMAKE_BUILD_TYPE=Release /samples/cpp + + - For debug configuration: + + .. code-block:: sh + + cmake -DCMAKE_BUILD_TYPE=Debug /samples/cpp + +4. Run ``make`` to build the samples: + + .. code-block:: sh + + make + +For the release configuration, the sample application binaries are in ``//Release/``; +for the debug configuration — in ``//Debug/``. + +.. _build-samples-windows: + +Build the Sample Applications on Microsoft Windows +++++++++++++++++++++++++++++++++++++++++++++++++++ The recommended Windows build environment is the following: + * Microsoft Windows 10 * Microsoft Visual Studio 2019 * CMake version 3.10 or higher -> **NOTE**: If you want to use Microsoft Visual Studio 2019, you are required to install CMake 3.14 or higher. +.. note:: + + If you want to use Microsoft Visual Studio 2019, you are required to install CMake 3.14 or higher. -To build the C or C++ sample applications on Windows, go to the `\samples\c` or `\samples\cpp` directory, respectively, and run the `build_samples_msvc.bat` batch file: -```sh -build_samples_msvc.bat -``` +To build the C or C++ sample applications on Windows, go to the ``\samples\c`` or ``\samples\cpp`` directory, respectively, and run the ``build_samples_msvc.bat`` batch file: + +.. code-block:: sh + + build_samples_msvc.bat By default, the script automatically detects the highest Microsoft Visual Studio version installed on the machine and uses it to create and build a solution for a sample code Once the build is completed, you can find sample binaries in the following folders: -* C samples: `C:\Users\\Documents\Intel\OpenVINO\openvino_c_samples_build\\Release` -* C++ samples: `C:\Users\\Documents\Intel\OpenVINO\openvino_cpp_samples_build\\Release` -where the is the output of `echo %PROCESSOR_ARCHITECTURE%`, for example, `intel64` (AMD64), or `arm64`. -You can also build a generated solution manually. For example, if you want to build C++ sample binaries in Debug configuration, run the appropriate version of the -Microsoft Visual Studio and open the generated solution file from the `C:\Users\\Documents\Intel\OpenVINO\openvino_cpp_samples_build\Samples.sln` -directory. +* C samples: ``C:\Users\\Documents\Intel\OpenVINO\openvino_c_samples_build\\Release`` +* C++ samples: ``C:\Users\\Documents\Intel\OpenVINO\openvino_cpp_samples_build\\Release`` where the is the output of ``echo PROCESSOR_ARCHITECTURE%``, for example, ``intel64`` (AMD64), or ``arm64``. + +You can also build a generated solution manually. For example, if you want to build C++ sample binaries in Debug configuration, run the appropriate version of the Microsoft Visual Studio and open the generated solution file from the ``C:\Users\\Documents\Intel\OpenVINO\openvino_cpp_samples_build\Samples.sln`` directory. + +.. _build-samples-macos: -### Build the Sample Applications on macOS +Build the Sample Applications on macOS +++++++++++++++++++++++++++++++++++++++ The officially supported macOS build environment is the following: @@ -179,112 +225,131 @@ The officially supported macOS build environment is the following: * Clang compiler from Xcode 10.1 or higher * CMake version 3.13 or higher -> **NOTE**: For building samples from the open-source version of OpenVINO toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md). +.. note:: + + For building samples from the open-source version of OpenVINO toolkit, see the `build instructions on GitHub `__ . + +To build the C or C++ sample applications for macOS, go to the ``/samples/c`` or ``/samples/cpp`` directory, respectively, and run the ``build_samples.sh`` script: -To build the C or C++ sample applications for macOS, go to the `/samples/c` or `/samples/cpp` directory, respectively, and run the `build_samples.sh` script: -```sh -build_samples.sh -``` +.. code-block:: sh + + build_samples.sh Once the build is completed, you can find sample binaries in the following folders: -* C samples: `~/openvino_c_samples_build//Release` -* C++ samples: `~/openvino_cpp_samples_build//Release` + +* C samples: ``~/openvino_c_samples_build//Release`` +* C++ samples: ``~/openvino_cpp_samples_build//Release`` You can also build the sample applications manually: -> **NOTE**: If you have installed the product as a root user, switch to root mode before you continue: `sudo -i` +.. note:: + + If you have installed the product as a root user, switch to root mode before you continue: ``sudo -i`` + +.. note:: + + Before proceeding, make sure you have OpenVINO™ environment set correctly. This can be done manually by + +.. code-block:: sh -> **NOTE**: Before proceeding, make sure you have OpenVINO™ environment set correctly. This can be done manually by -```sh -cd / -source setupvars.sh -``` + cd / + source setupvars.sh -1. Navigate to a directory that you have write access to and create a samples build directory. This example uses a directory named `build`: -```sh -mkdir build -``` -> **NOTE**: If you ran the Image Classification verification script during the installation, the C++ samples build directory was already created in your home directory: `~/openvino_cpp_samples_build/` +1. Navigate to a directory that you have write access to and create a samples build directory. This example uses a directory named ``build``: + .. code-block:: sh + + mkdir build + + .. note:: + + If you ran the Image Classification verification script during the installation, the C++ samples build directory was already created in your home directory: ``~/openvino_cpp_samples_build/`` + 2. Go to the created directory: -```sh -cd build -``` + + .. code-block:: sh + + cd build 3. Run CMake to generate the Make files for release or debug configuration. For example, for C++ samples: - - For release configuration: - ```sh - cmake -DCMAKE_BUILD_TYPE=Release /samples/cpp - ``` - - For debug configuration: - ```sh - cmake -DCMAKE_BUILD_TYPE=Debug /samples/cpp - ``` -4. Run `make` to build the samples: -```sh -make -``` - -For the release configuration, the sample application binaries are in `//Release/`; -for the debug configuration — in `//Debug/`. - -## Get Ready for Running the Sample Applications - -### Get Ready for Running the Sample Applications on Linux - -Before running compiled binary files, make sure your application can find the -OpenVINO Runtime libraries. -Run the `setupvars` script to set all necessary environment variables: -```sh -source /setupvars.sh -``` - -#### (Optional) Set Environment Variables Permanently + + - For release configuration: + + .. code-block:: sh + + cmake -DCMAKE_BUILD_TYPE=Release /samples/cpp + + - For debug configuration: + + .. code-block:: sh + + cmake -DCMAKE_BUILD_TYPE=Debug /samples/cpp + +4. Run ``make`` to build the samples: + + .. code-block:: sh + + make + +For the release configuration, the sample application binaries are in ``//Release/``; for the debug configuration — in ``//Debug/``. + +Get Ready for Running the Sample Applications +############################################# + +Get Ready for Running the Sample Applications on Linux +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Before running compiled binary files, make sure your application can find the OpenVINO Runtime libraries. Run the ``setupvars`` script to set all necessary environment variables: + +.. code-block:: sh + + source /setupvars.sh + +(Optional) Set Environment Variables Permanently +------------------------------------------------ The OpenVINO environment variables are removed when you close the shell. As an option, you can permanently set the environment variables as follows: -1. Open the `.bashrc` file in ``: -```sh -vi /.bashrc -``` +1. Open the ``.bashrc`` file in ````: + + .. code-block:: sh + + vi /.bashrc 2. Add this line to the end of the file: -```sh -source /opt/intel/openvino_2022/setupvars.sh -``` - -3. Save and close the file: press the **Esc** key, type `:wq` and press the **Enter** key. -4. To test your change, open a new terminal. You will see `[setupvars.sh] OpenVINO environment initialized`. - -You are ready to run sample applications. To learn about how to run a particular -sample, read the sample documentation by clicking the sample name in the samples -list above. - -### Get Ready for Running the Sample Applications on Windows - -Before running compiled binary files, make sure your application can find the -OpenVINO Runtime libraries. -Use the `setupvars` script, which sets all necessary environment variables: -```sh -\setupvars.bat -``` - -To debug or run the samples on Windows in Microsoft Visual Studio, make sure you -have properly configured **Debugging** environment settings for the **Debug** -and **Release** configurations. Set correct paths to the OpenCV libraries, and -debug and release versions of the OpenVINO Runtime libraries. -For example, for the **Debug** configuration, go to the project's -**Configuration Properties** to the **Debugging** category and set the `PATH` -variable in the **Environment** field to the following: - -```sh -PATH=\runtime\bin;%PATH% -``` -where `` is the directory in which the OpenVINO toolkit is installed. - -You are ready to run sample applications. To learn about how to run a particular -sample, read the sample documentation by clicking the sample name in the samples -list above. - -## See Also -* [OpenVINO Runtime User Guide](openvino_intro.md) + + .. code-block:: sh + + source /opt/intel/openvino_2022/setupvars.sh + +3. Save and close the file: press the **Esc** key, type ``:wq`` and press the **Enter** key. +4. To test your change, open a new terminal. You will see ``[setupvars.sh] OpenVINO environment initialized``. + +You are ready to run sample applications. To learn about how to run a particular sample, read the sample documentation by clicking the sample name in the samples list above. + +Get Ready for Running the Sample Applications on Windows +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Before running compiled binary files, make sure your application can find the OpenVINO Runtime libraries. Use the ``setupvars`` script, which sets all necessary environment variables: + +.. code-block:: sh + + \setupvars.bat + +To debug or run the samples on Windows in Microsoft Visual Studio, make sure you have properly configured **Debugging** environment settings for the **Debug** and **Release** configurations. Set correct paths to the OpenCV libraries, and debug and release versions of the OpenVINO Runtime libraries. For example, for the **Debug** configuration, go to the project's **Configuration Properties** to the **Debugging** category and set the ``PATH`` variable in the **Environment** field to the following: + +.. code-block:: sh + + PATH=\runtime\bin;%PATH% + +where ```` is the directory in which the OpenVINO toolkit is installed. + +You are ready to run sample applications. To learn about how to run a particular sample, read the sample documentation by clicking the sample name in the samples list above. + +See Also +######## + +* :doc:`OpenVINO Runtime User Guide ` + +@endsphinxdirective + diff --git a/docs/OV_Runtime_UG/auto_device_selection.md b/docs/OV_Runtime_UG/auto_device_selection.md index 4567100e9d50f2..d2089a078f131a 100644 --- a/docs/OV_Runtime_UG/auto_device_selection.md +++ b/docs/OV_Runtime_UG/auto_device_selection.md @@ -135,6 +135,15 @@ Following the OpenVINO™ naming convention, the Automatic Device Selection mode | | | | beginning. The default value is ``true``, indicating that CPU is | | | | | used as acceleration by default. | +-----------------------------------------------+----------------------------------------------------------------------+ +| | ``ov::intel_auto::enable_runtime_fallback`` | | **Values**: | +| | | | ``true`` | +| | | | ``false`` | +| | | | | +| | | | Enables/disables runtime fallback to other devices and performs | +| | | | the failed inference request again, if inference request fails on | +| | | | the currently selected device. | +| | | | The default value is ``true``. | ++-----------------------------------------------+----------------------------------------------------------------------+ Inference with AUTO is configured similarly to when device plugins are used: you compile the model on the plugin with configuration and execute inference. diff --git a/docs/OV_Runtime_UG/automatic_batching.md b/docs/OV_Runtime_UG/automatic_batching.md index 836c6daf231d37..618a7644eeb562 100644 --- a/docs/OV_Runtime_UG/automatic_batching.md +++ b/docs/OV_Runtime_UG/automatic_batching.md @@ -1,155 +1,238 @@ # Automatic Batching {#openvino_docs_OV_UG_Automatic_Batching} -The Automatic Batching Execution mode (or Auto-batching for short) performs automatic batching on-the-fly to improve device utilization by grouping inference requests together, with no programming effort from the user. -With Automatic Batching, gathering the input and scattering the output from the individual inference requests required for the batch happen transparently, without affecting the application code. +@sphinxdirective -This article provides a preview of the new Automatic Batching function, including how it works, its configurations, and testing performance. +The Automatic Batching Execution mode (or Auto-batching for short) performs automatic batching on-the-fly to improve device utilization by grouping inference requests together, without programming effort from the user. +With Automatic Batching, gathering the input and scattering the output from the individual inference requests required for the batch happen transparently, without affecting the application code. -## Enabling/Disabling Automatic Batching +Auto Batching can be used :ref:`directly as a virtual device ` or as an :ref:`option for inference on CPU/GPU/VPU ` (by means of configuration/hint). These 2 ways are provided for the user to enable the BATCH devices **explicitly** or **implicitly**, with the underlying logic remaining the same. An example of the difference is that the CPU device doesn’t support implicitly to enable BATCH device, commands such as ``./benchmark_app -m -d CPU -hint tput`` will not apply BATCH device **implicitly**, but ``./benchmark_app -m -d "BATCH:CPU(16)`` can **explicitly** load BATCH device. -Auto-batching primarily targets the existing code written for inferencing many requests, each instance with the batch size 1. To obtain corresponding performance improvements, the application **must be running many inference requests simultaneously**. +Auto-batching primarily targets the existing code written for inferencing many requests, each instance with the batch size 1. To get corresponding performance improvements, the application **must be running multiple inference requests simultaneously**. Auto-batching can also be used via a particular *virtual* device. -Batching is a straightforward way of leveraging the compute power of GPU and saving on communication overheads. Automatic Batching is "implicitly" triggered on the GPU when `ov::hint::PerformanceMode::THROUGHPUT` is specified for the `ov::hint::performance_mode` property for the `compile_model` or `set_property` calls. +This article provides a preview of the Automatic Batching function, including how it works, its configurations, and testing performance. + +How Automatic Batching Works +############################ + +.. tab-set:: + + .. tab-item:: Enabling Automatic Batching + :sync: enabling-automatic-batching + + Batching is a straightforward way of leveraging the compute power of GPU and saving on communication overheads. Automatic Batching is "implicitly" triggered on the GPU when ``ov::hint::PerformanceMode::THROUGHPUT`` is specified for the ``ov::hint::performance_mode`` property for the ``compile_model`` or ``set_property`` calls. + + .. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [compile_model] + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: Python + :fragment: [compile_model] + + To enable Auto-batching in the legacy apps not akin to the notion of performance hints, you need to use the **explicit** device notion, such as ``BATCH:GPU``. + + .. tab-item:: Disabling Automatic Batching + :sync: disabling-automatic-batching + + Auto-Batching can be disabled (for example, for the GPU device) to prevent being triggered by ``ov::hint::PerformanceMode::THROUGHPUT``. To do that, set ``ov::hint::allow_auto_batching`` to **false** in addition to the ``ov::hint::performance_mode``, as shown below: + + .. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [compile_model_no_auto_batching] + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: Python + :fragment: [compile_model_no_auto_batching] + + +Configuring Automatic Batching +++++++++++++++++++++++++++++++ -@sphinxtabset +Following the OpenVINO naming convention, the *batching* device is assigned the label of *BATCH*. The configuration options are as follows: -@sphinxtab{C++} ++----------------------------+------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Parameter name | Parameter description | Examples | ++============================+======================================================================================================+==================================================================================================================================================================================================================================================+ +| ``AUTO_BATCH_DEVICE`` | The name of the device to apply Automatic batching, with the optional batch size value in brackets. | ``BATCH:GPU`` triggers the automatic batch size selection. ``BATCH:GPU(4)`` directly specifies the batch size. | ++----------------------------+------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ``ov::auto_batch_timeout`` | The timeout value, in ms. (1000 by default) | You can reduce the timeout value to avoid performance penalty when the data arrives too unevenly. For example, set it to "100", or the contrary, i.e., make it large enough to accommodate input preparation (e.g. when it is a serial process). | ++----------------------------+------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -@snippet docs/snippets/ov_auto_batching.cpp compile_model +Automatic Batch Size Selection +++++++++++++++++++++++++++++++ -@endsphinxtab +In both the THROUGHPUT hint and the explicit BATCH device cases, the optimal batch size is selected automatically, as the implementation queries the ``ov::optimal_batch_size`` property from the device and passes the model graph as the parameter. The actual value depends on the model and device specifics, for example, the on-device memory for dGPUs. +The support for Auto-batching is not limited to GPU. However, if a device does not support ``ov::optimal_batch_size`` yet, to work with Auto-batching, an explicit batch size must be specified, e.g., ``BATCH:(16)``. -@sphinxtab{Python} +This "automatic batch size selection" works on the presumption that the application queries ``ov::optimal_number_of_infer_requests`` to create the requests of the returned number and run them simultaneously: -@snippet docs/snippets/ov_auto_batching.py compile_model +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [query_optimal_num_requests] -@endsphinxtab + .. tab-item:: Python + :sync: py -@endsphinxtabset + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: Python + :fragment: [query_optimal_num_requests] -To enable Auto-batching in the legacy apps not akin to the notion of performance hints, you need to use the **explicit** device notion, such as `BATCH:GPU`. +.. _limiting-batch-size: -### Disabling Automatic Batching +Optimizing Performance by Limiting Batch Size +--------------------------------------------- -Auto-Batching can be disabled (for example, for the GPU device) to prevent being triggered by `ov::hint::PerformanceMode::THROUGHPUT`. To do that, set `ov::hint::allow_auto_batching` to **false** in addition to the `ov::hint::performance_mode`, as shown below: +If not enough inputs were collected, the ``timeout`` value makes the transparent execution fall back to the execution of individual requests. This value can be configured via the ``AUTO_BATCH_TIMEOUT`` property. +The timeout, which adds itself to the execution time of the requests, heavily penalizes the performance. To avoid this, when your parallel slack is bounded, provide OpenVINO with an additional hint. -@sphinxtabset +For example, when the application processes only 4 video streams, there is no need to use a batch larger than 4. The most future-proof way to communicate the limitations on the parallelism is to equip the performance hint with the optional ``ov::hint::num_requests`` configuration key set to 4. This will limit the batch size for the GPU and the number of inference streams for the CPU, hence each device uses ``ov::hint::num_requests`` while converting the hint to the actual device configuration options: -@sphinxtab{C++} -@snippet docs/snippets/ov_auto_batching.cpp compile_model_no_auto_batching +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [hint_num_requests] -@endsphinxtab + .. tab-item:: Python + :sync: py -@sphinxtab{Python} + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: Python + :fragment: [hint_num_requests] -@snippet docs/snippets/ov_auto_batching.py compile_model_no_auto_batching -@endsphinxtab +For the *explicit* usage, you can limit the batch size by using ``BATCH:GPU(4)``, where 4 is the number of requests running in parallel. -@endsphinxtabset +.. _auto-batching-as-device: -## Configuring Automatic Batching -Following the OpenVINO naming convention, the *batching* device is assigned the label of *BATCH*. The configuration options are as follows: +Automatic Batching as an explicit device +++++++++++++++++++++++++++++++++++++++++ -| Parameter name | Parameter description | Examples | -| :--- | :--- |:-----------------------------------------------------------------------------| -| `AUTO_BATCH_DEVICE` | The name of the device to apply Automatic batching, with the optional batch size value in brackets. | `BATCH:GPU` triggers the automatic batch size selection. `BATCH:GPU(4)` directly specifies the batch size. | -| `ov::auto_batch_timeout` | The timeout value, in ms. (1000 by default) | You can reduce the timeout value to avoid performance penalty when the data arrives too unevenly. For example, set it to "100", or the contrary, i.e., make it large enough to accommodate input preparation (e.g. when it is a serial process). | +The below examples show how AUTO Batching can be used in the form of device that the user can apply to perform inference directly: -## Automatic Batch Size Selection +.. code-block:: sh -In both the THROUGHPUT hint and the explicit BATCH device cases, the optimal batch size is selected automatically, as the implementation queries the `ov::optimal_batch_size` property from the device and passes the model graph as the parameter. The actual value depends on the model and device specifics, for example, the on-device memory for dGPUs. -The support for Auto-batching is not limited to GPU. However, if a device does not support `ov::optimal_batch_size` yet, to work with Auto-batching, an explicit batch size must be specified, e.g., `BATCH:(16)`. + ./benchmark_app -m -d "BATCH:GPU" + ./benchmark_app -m -d "BATCH:GPU(16)" + ./benchmark_app -m -d "BATCH:CPU(16)" -This "automatic batch size selection" works on the presumption that the application queries `ov::optimal_number_of_infer_requests` to create the requests of the returned number and run them simultaneously: -@sphinxtabset +* ``BATCH`` -- load BATCH device explicitly, +* ``:GPU(16)`` -- BATCH devices configuration, which tell BATCH device to apply GPU device with batch size = 16. -@sphinxtab{C++} +.. _auto-batching-as-option: -@snippet docs/snippets/ov_auto_batching.cpp query_optimal_num_requests +Automatic Batching as underlying device configured to other devices ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -@endsphinxtab +In the following example, BATCH device will be configured to another device in case of ``tput/ctput mode``. -@sphinxtab{Python} +.. code-block:: sh -@snippet docs/snippets/ov_auto_batching.py query_optimal_num_requests + ./benchmark_app -m -d GPU -hint tput + ./benchmark_app -m -d AUTO -hint tput + ./benchmark_app -m -d AUTO -hint ctput + ./benchmark_app -m -d AUTO:GPU -hint ctput -@endsphinxtab +.. note:: + If you run ``./benchmark_app``, do not set ``batch_size`` by ``-b ``, otherwise AUTO mode will not be applied. -@endsphinxtabset +Other Performance Considerations +################################ +To achieve the best performance with Automatic Batching, the application should: -### Optimizing Performance by Limiting Batch Size +- Operate inference requests of the number that represents the multiple of the batch size. In the example from :ref:`Optimizing Performance by Limiting Batch Size section ` -- for batch size 4, the application should operate 4, 8, 12, 16, etc. requests. +- Use the requests that are grouped by the batch size together. For example, the first 4 requests are inferred, while the second group of the requests is being populated. Essentially, Automatic Batching shifts the asynchronicity from the individual requests to the groups of requests that constitute the batches. +- Balance the ``timeout`` value vs. the batch size. For example, in many cases, having a smaller ``timeout`` value/batch size may yield better performance than having a larger batch size with a ``timeout`` value that is not large enough to accommodate the full number of the required requests. +- When Automatic Batching is enabled, the ``timeout`` property of ``ov::CompiledModel`` can be changed anytime, even after the loading/compilation of the model. For example, setting the value to 0 disables Auto-batching effectively, as the collection of requests would be omitted. +- Carefully apply Auto-batching to the pipelines. For example, in the conventional "video-sources -> detection -> classification" flow, it is most beneficial to do Auto-batching over the inputs to the detection stage. The resulting number of detections is usually fluent, which makes Auto-batching less applicable for the classification stage. -If not enough inputs were collected, the `timeout` value makes the transparent execution fall back to the execution of individual requests. This value can be configured via the `AUTO_BATCH_TIMEOUT` property. -The timeout, which adds itself to the execution time of the requests, heavily penalizes the performance. To avoid this, when your parallel slack is bounded, provide OpenVINO with an additional hint. +Limitations ++++++++++++ -For example, when the application processes only 4 video streams, there is no need to use a batch larger than 4. The most future-proof way to communicate the limitations on the parallelism is to equip the performance hint with the optional `ov::hint::num_requests` configuration key set to 4. This will limit the batch size for the GPU and the number of inference streams for the CPU, hence each device uses `ov::hint::num_requests` while converting the hint to the actual device configuration options: +The following are limitations of the current AUTO Batching implementations: -@sphinxtabset +- The dynamic model is not supported by ``BATCH`` device. +- ``BATCH`` device can only support ``tput/ctput mode``. The ``latency/none mode`` is not supported. +- Supported are only models with ``batch dimension = 1``. +- The input/output tensor should come from ``inferRequest``, otherwise the user-created tensor will trigger a memory copying. +- The ``OPTIMAL_BATCH_SIZE`` should be greater than ``2``. In case it's not, user needs to specify a batch size which depends on model and device (CPU does not support this property). +- ``BATCH`` device supports GPU by default, while CPU will not trigger ``auto_batch`` in ``tput`` mode. +- ``AUTO_BATCH`` will bring much more compilation latency. +- Although it is less critical for the throughput-oriented scenarios, the load time with Auto-batching increases by almost double. +- Certain networks are not safely reshapable by the "batching" dimension (specified as ``N`` in the layout terms). Besides, if the batching dimension is not zeroth, Auto-batching will not be triggered "implicitly" by the throughput hint. +- The "explicit" notion, for example, ``BATCH:GPU``, using the relaxed dimensions tracking, often makes Auto-batching possible. For example, this method unlocks most **detection networks**. +- When *forcing* Auto-batching via the "explicit" device notion, make sure that you validate the results for correctness. +- Performance improvements happen at the cost of the growth of memory footprint. However, Auto-batching queries the available memory (especially for dGPU) and limits the selected batch size accordingly. -@sphinxtab{C++} -@snippet docs/snippets/ov_auto_batching.cpp hint_num_requests +Testing Performance with Benchmark_app +###################################### -@endsphinxtab +The ``benchmark_app`` sample, that has both :doc:`C++ ` and :doc:`Python ` versions, is the best way to evaluate the performance of Automatic Batching: -@sphinxtab{Python} +- The most straightforward way is using the performance hints: + + - benchmark_app **-hint tput** -d GPU -m 'path to your favorite model' +- You can also use the "explicit" device notion to override the strict rules of the implicit reshaping by the batch dimension: + + - benchmark_app **-hint none -d BATCH:GPU** -m 'path to your favorite model' +- or override the automatically deduced batch size as well: + + - $benchmark_app -hint none -d **BATCH:GPU(16)** -m 'path to your favorite model' + - This example also applies to CPU or any other device that generally supports batch execution. + - Keep in mind that some shell versions (e.g. ``bash``) may require adding quotes around complex device names, i.e. ``-d "BATCH:GPU(16)"`` in this example. -@snippet docs/snippets/ov_auto_batching.py hint_num_requests -@endsphinxtab +Note that Benchmark_app performs a warm-up run of a *single* request. As Auto-Batching requires significantly more requests to execute in batch, this warm-up run hits the default timeout value (1000 ms), as reported in the following example: -@endsphinxtabset +.. code-block:: sh + [ INFO ] First inference took 1000.18ms -For the *explicit* usage, you can limit the batch size by using `BATCH:GPU(4)`, where 4 is the number of requests running in parallel. +This value also exposed as the final execution statistics on the ``benchmark_app`` exit: + +.. code-block:: sh -## Other Performance Considerations + [ INFO ] Latency: + [ INFO ] Max: 1000.18 ms -To achieve the best performance with Automatic Batching, the application should: - - Operate inference requests of the number that represents the multiple of the batch size. In the example above, for batch size 4, the application should operate 4, 8, 12, 16, etc. requests. - - Use the requests that are grouped by the batch size together. For example, the first 4 requests are inferred, while the second group of the requests is being populated. Essentially, Automatic Batching shifts the asynchronicity from the individual requests to the groups of requests that constitute the batches. - - Balance the `timeout` value vs. the batch size. For example, in many cases, having a smaller `timeout` value/batch size may yield better performance than having a larger batch size with a `timeout` value that is not large enough to accommodate the full number of the required requests. - - When Automatic Batching is enabled, the `timeout` property of `ov::CompiledModel` can be changed anytime, even after the loading/compilation of the model. For example, setting the value to 0 disables Auto-batching effectively, as the collection of requests would be omitted. - - Carefully apply Auto-batching to the pipelines. For example, in the conventional "video-sources -> detection -> classification" flow, it is most beneficial to do Auto-batching over the inputs to the detection stage. The resulting number of detections is usually fluent, which makes Auto-batching less applicable for the classification stage. +This is NOT the actual latency of the batched execution, so you are recommended to refer to other metrics in the same log, for example, "Median" or "Average" execution. -The following are limitations of the current implementations: -- Although it is less critical for the throughput-oriented scenarios, the load time with Auto-batching increases by almost double. - - Certain networks are not safely reshapable by the "batching" dimension (specified as `N` in the layout terms). Besides, if the batching dimension is not zeroth, Auto-batching will not be triggered "implicitly" by the throughput hint. - - The "explicit" notion, for example, `BATCH:GPU`, using the relaxed dimensions tracking, often makes Auto-batching possible. For example, this method unlocks most **detection networks**. - - When *forcing* Auto-batching via the "explicit" device notion, make sure that you validate the results for correctness. - - Performance improvements happen at the cost of the growth of memory footprint. However, Auto-batching queries the available memory (especially for dGPU) and limits the selected batch size accordingly. - - -## Testing Performance with Benchmark_app -The `benchmark_app` sample, that has both [C++](../../samples/cpp/benchmark_app/README.md) and [Python](../../tools/benchmark_tool/README.md) versions, is the best way to evaluate the performance of Automatic Batching: - - The most straightforward way is using the performance hints: - - benchmark_app **-hint tput** -d GPU -m 'path to your favorite model' - - You can also use the "explicit" device notion to override the strict rules of the implicit reshaping by the batch dimension: - - benchmark_app **-hint none -d BATCH:GPU** -m 'path to your favorite model' - - or override the automatically deduced batch size as well: - - $benchmark_app -hint none -d **BATCH:GPU(16)** -m 'path to your favorite model' - - This example also applies to CPU or any other device that generally supports batch execution. - - Keep in mind that some shell versions (e.g. `bash`) may require adding quotes around complex device names, i.e. `-d "BATCH:GPU(16)"` in this example. +Additional Resources +#################### -Note that Benchmark_app performs a warm-up run of a *single* request. As Auto-Batching requires significantly more requests to execute in batch, this warm-up run hits the default timeout value (1000 ms), as reported in the following example: +* :doc:`Supported Devices ` - ``` - [ INFO ] First inference took 1000.18ms - ``` -This value also exposed as the final execution statistics on the `benchmark_app` exit: - ``` - [ INFO ] Latency: - [ INFO ] Max: 1000.18 ms - ``` -This is NOT the actual latency of the batched execution, so you are recommended to refer to other metrics in the same log, for example, "Median" or "Average" execution. -### Additional Resources +@endsphinxdirective -* [Supported Devices](supported_plugins/Supported_Devices.md) diff --git a/docs/OV_Runtime_UG/deployment/deployment-manager-tool.md b/docs/OV_Runtime_UG/deployment/deployment-manager-tool.md index b48525217f267b..5fdd5a2112ac57 100644 --- a/docs/OV_Runtime_UG/deployment/deployment-manager-tool.md +++ b/docs/OV_Runtime_UG/deployment/deployment-manager-tool.md @@ -1,202 +1,184 @@ # Deploying Your Application with Deployment Manager {#openvino_docs_install_guides_deployment_manager_tool} -The OpenVINO™ Deployment Manager is a Python command-line tool that creates a deployment package by assembling the model, OpenVINO IR files, your application, and associated dependencies into a runtime package for your target device. This tool is delivered within the Intel® Distribution of OpenVINO™ toolkit for Linux, Windows and macOS release packages. It is available in the `/tools/deployment_manager` directory after installation. +@sphinxdirective + +The OpenVINO™ Deployment Manager is a Python command-line tool that creates a deployment package by assembling the model, OpenVINO IR files, your application, and associated dependencies into a runtime package for your target device. This tool is delivered within the Intel® Distribution of OpenVINO™ toolkit for Linux, Windows and macOS release packages. It is available in the ``/tools/deployment_manager`` directory after installation. This article provides instructions on how to create a package with Deployment Manager and then deploy the package to your target systems. -## Prerequisites +Prerequisites +#################### To use the Deployment Manager tool, the following requirements need to be met: -* Intel® Distribution of OpenVINO™ toolkit is installed. See the [Installation Guide](../../install_guides/installing-openvino-overview.md) for instructions on different operating systems. + +* Intel® Distribution of OpenVINO™ toolkit is installed. See the :doc:`Installation Guide ` for instructions on different operating systems. * To run inference on a target device other than CPU, device drivers must be pre-installed: - * **For GPU**, see [Configurations for Intel® Processor Graphics (GPU)](../../install_guides/configurations-for-intel-gpu.md). - * **For GNA**, see [Intel® Gaussian & Neural Accelerator (GNA)](../../install_guides/configurations-for-intel-gna.md) -> **IMPORTANT**: The operating system on the target system must be the same as the development system on which you are creating the package. For example, if the target system is Ubuntu 18.04, the deployment package must be created from the OpenVINO™ toolkit installed on Ubuntu 18.04. + * **For GPU**, see :doc:`Configurations for Intel® Processor Graphics (GPU) `. + * **For GNA**, see :doc:`Intel® Gaussian & Neural Accelerator (GNA) ` + +.. important:: + + The operating system on the target system must be the same as the development system on which you are creating the package. For example, if the target system is Ubuntu 18.04, the deployment package must be created from the OpenVINO™ toolkit installed on Ubuntu 18.04. + +.. tip:: -> **TIP**: If your application requires additional dependencies, including the Microsoft Visual C++ Redistributable, use the ['--user_data' option](https://docs.openvino.ai/latest/openvino_docs_install_guides_deployment_manager_tool.html#run-standard-cli-mode) to add them to the deployment archive. Install these dependencies on the target host before running inference. + If your application requires additional dependencies, including the Microsoft Visual C++ Redistributable, use the `'--user_data' option <#running-deployment-manager-in-standard-cli-mode>`__ to add them to the deployment archive. Install these dependencies on the target host before running inference. -## Creating Deployment Package Using Deployment Manager +Creating Deployment Package Using Deployment Manager +#################################################### To create a deployment package that includes inference-related components of OpenVINO™ toolkit, you can run the Deployment Manager tool in either interactive or standard CLI mode . -### Running Deployment Manager in Interactive Mode +Running Deployment Manager in Interactive Mode +++++++++++++++++++++++++++++++++++++++++++++++ -@sphinxdirective +.. dropdown:: Click to expand/collapse -.. raw:: html + The interactive mode provides a user-friendly command-line interface that guides through the process with text prompts. -
+ To launch the Deployment Manager in interactive mode, open a new terminal window, go to the Deployment Manager tool directory, and run the tool script without parameters: -@endsphinxdirective + .. tab:: Linux -The interactive mode provides a user-friendly command-line interface that guides through the process with text prompts. + .. code-block:: sh -To launch the Deployment Manager in interactive mode, open a new terminal window, go to the Deployment Manager tool directory, and run the tool script without parameters: - -@sphinxdirective - -.. tab:: Linux - - .. code-block:: sh - - cd /tools/deployment_manager - - ./deployment_manager.py - -.. tab:: Windows - - .. code-block:: bat - - cd \deployment_tools\tools\deployment_manager - .\deployment_manager.py - -.. tab:: macOS - - .. code-block:: sh - - cd /tools/deployment_manager - ./deployment_manager.py - -@endsphinxdirective + cd /tools/deployment_manager -The target device selection dialog is displayed: - -![Deployment Manager selection dialog](../img/selection_dialog.png) + ./deployment_manager.py -Use the options provided on the screen to complete the selection of the target devices, and press **Enter** to proceed to the package generation dialog. To interrupt the generation process and exit the program, type **q** and press **Enter**. + .. tab:: Windows -Once the selection is accepted, the package generation dialog will appear: - -![Deployment Manager configuration dialog](../img/configuration_dialog.png) + .. code-block:: bat -The target devices selected in the previous step appear on the screen. To go back and change the selection, type **b** and press **Enter**. Use the default settings, or use the following options to configure the generation process: - -* `o. Change output directory` (optional): the path to the output directory. By default, it is set to your home directory. + cd \deployment_tools\tools\deployment_manager + .\deployment_manager.py -* `u. Provide (or change) path to folder with user data` (optional): the path to a directory with user data (OpenVINO IR, model, dataset, etc.) files and subdirectories required for inference, which will be added to the deployment archive. By default, it is set to `None`, which means that copying the user data to the target system need to be done separately. + .. tab:: macOS -* `t. Change archive name` (optional): the deployment archive name without extension. By default, it is set to `openvino_deployment_package`. - -After all the parameters are set, type **g** and press **Enter** to generate the package for the selected target devices. To interrupt the generation process and exit the program, type **q** and press **Enter**. + .. code-block:: sh -Once the script has successfully completed, the deployment package is generated in the specified output directory. + cd /tools/deployment_manager + ./deployment_manager.py -@sphinxdirective -.. raw:: html + The target device selection dialog is displayed: -
+ .. image:: _static/images/selection_dialog.png + :alt: Deployment Manager selection dialog -@endsphinxdirective + Use the options provided on the screen to complete the selection of the target devices, and press **Enter** to proceed to the package generation dialog. To interrupt the generation process and exit the program, type **q** and press **Enter**. -### Running Deployment Manager in Standard CLI Mode + Once the selection is accepted, the package generation dialog will appear: -@sphinxdirective + .. image:: _static/images/configuration_dialog.png + :alt: Deployment Manager configuration dialog -.. raw:: html + The target devices selected in the previous step appear on the screen. To go back and change the selection, type **b** and press **Enter**. Use the default settings, or use the following options to configure the generation process: -
+ * ``o. Change output directory`` (optional): the path to the output directory. By default, it is set to your home directory. -@endsphinxdirective + * ``u. Provide (or change) path to folder with user data`` (optional): the path to a directory with user data (OpenVINO IR, model, dataset, etc.) files and subdirectories required for inference, which will be added to the deployment archive. By default, it is set to ``None``, which means that copying the user data to the target system need to be done separately. -You can also run the Deployment Manager tool in the standard CLI mode. In this mode, specify the target devices and other parameters as command-line arguments of the Deployment Manager Python script. This mode facilitates integrating the tool in an automation pipeline. + * ``t. Change archive name`` (optional): the deployment archive name without extension. By default, it is set to ``openvino_deployment_package``. -To launch the Deployment Manager tool in the standard mode: open a new terminal window, go to the Deployment Manager tool directory, and run the tool command with the following syntax: + After all the parameters are set, type **g** and press **Enter** to generate the package for the selected target devices. To interrupt the generation process and exit the program, type **q** and press **Enter**. -@sphinxdirective + Once the script has successfully completed, the deployment package is generated in the specified output directory. -.. tab:: Linux - .. code-block:: sh +Running Deployment Manager in Standard CLI Mode ++++++++++++++++++++++++++++++++++++++++++++++++ - cd /tools/deployment_manager - ./deployment_manager.py <--targets> [--output_dir] [--archive_name] [--user_data] -.. tab:: Windows +.. dropdown:: Click to expand/collapse - .. code-block:: bat + You can also run the Deployment Manager tool in the standard CLI mode. In this mode, specify the target devices and other parameters as command-line arguments of the Deployment Manager Python script. This mode facilitates integrating the tool in an automation pipeline. - cd \tools\deployment_manager - .\deployment_manager.py <--targets> [--output_dir] [--archive_name] [--user_data] + To launch the Deployment Manager tool in the standard mode: open a new terminal window, go to the Deployment Manager tool directory, and run the tool command with the following syntax: -.. tab:: macOS + .. tab:: Linux - .. code-block:: sh + .. code-block:: sh - cd /tools/deployment_manager - ./deployment_manager.py <--targets> [--output_dir] [--archive_name] [--user_data] + cd /tools/deployment_manager + ./deployment_manager.py <--targets> [--output_dir] [--archive_name] [--user_data] -@endsphinxdirective + .. tab:: Windows -The following options are available: + .. code-block:: bat -* `<--targets>` (required): the list of target devices to run inference. To specify more than one target, separate them with spaces, for example, `--targets cpu gpu`. -To get a list of currently available targets, run the program with the `-h` option. + cd \tools\deployment_manager + .\deployment_manager.py <--targets> [--output_dir] [--archive_name] [--user_data] -* `[--output_dir]` (optional): the path to the output directory. By default, it is set to your home directory. + .. tab:: macOS -* `[--archive_name]` (optional): a deployment archive name without extension. By default, it is set to `openvino_deployment_package`. + .. code-block:: sh -* `[--user_data]` (optional): the path to a directory with user data (OpenVINO IR, model, dataset, etc.) files and subdirectories required for inference, which will be added to the deployment archive. By default, it is set to `None`, which means copying the user data to the target system need to be performed separately. + cd /tools/deployment_manager + ./deployment_manager.py <--targets> [--output_dir] [--archive_name] [--user_data] -Once the script has successfully completed, the deployment package is generated in the output directory specified. -@sphinxdirective + The following options are available: -.. raw:: html + * ``<--targets>`` (required): the list of target devices to run inference. To specify more than one target, separate them with spaces, for example, ``--targets cpu gpu``. + To get a list of currently available targets, run the program with the ``-h`` option. -
+ * ``[--output_dir]`` (optional): the path to the output directory. By default, it is set to your home directory. -@endsphinxdirective + * ``[--archive_name]`` (optional): a deployment archive name without extension. By default, it is set to ``openvino_deployment_package``. + + * ``[--user_data]`` (optional): the path to a directory with user data (OpenVINO IR, model, dataset, etc.) files and subdirectories required for inference, which will be added to the deployment archive. By default, it is set to ``None``, which means copying the user data to the target system need to be performed separately. -## Deploying Package on Target Systems + Once the script has successfully completed, the deployment package is generated in the output directory specified. -Once the Deployment Manager has successfully completed, the `.tar.gz` (on Linux or macOS) or `.zip` (on Windows) package is generated in the specified output directory. + +Deploying Package on Target Systems +################################### + +Once the Deployment Manager has successfully completed, the ``.tar.gz`` (on Linux or macOS) or ``.zip`` (on Windows) package is generated in the specified output directory. To deploy the OpenVINO Runtime components from the development machine to the target system, perform the following steps: 1. Copy the generated archive to the target system by using your preferred method. -2. Extract the archive to the destination directory on the target system. If the name of your archive is different from the default one shown below, replace `openvino_deployment_package` with your specified name. -@sphinxdirective - -.. tab:: Linux +2. Extract the archive to the destination directory on the target system. If the name of your archive is different from the default one shown below, replace ``openvino_deployment_package`` with your specified name. - .. code-block:: sh + .. tab:: Linux - tar xf openvino_deployment_package.tar.gz -C + .. code-block:: sh -.. tab:: Windows + tar xf openvino_deployment_package.tar.gz -C - .. code-block:: bat + .. tab:: Windows - Use the archiver of your choice to unzip the file. + .. code-block:: bat -.. tab:: macOS + Use the archiver of your choice to unzip the file. - .. code-block:: sh + .. tab:: macOS - tar xf openvino_deployment_package.tar.gz -C + .. code-block:: sh -@endsphinxdirective + tar xf openvino_deployment_package.tar.gz -C Now, the package is extracted to the destination directory. The following files and subdirectories are created: - - * `setupvars.sh` — a copy of `setupvars.sh`. - * `runtime` — contains the OpenVINO runtime binary files. - * `install_dependencies` — a snapshot of the `install_dependencies` directory from the OpenVINO installation directory. - * `` — the directory with the user data (OpenVINO IR, model, dataset, etc.) specified while configuring the package. - -3. On a target Linux system, to run inference install additional dependencies by running the `install_openvino_dependencies.sh` script: - ```sh - cd /openvino/install_dependencies - sudo -E ./install_openvino_dependencies.sh - ``` + + * ``setupvars.sh`` — a copy of ``setupvars.sh``. + * ``runtime`` — contains the OpenVINO runtime binary files. + * ``install_dependencies`` — a snapshot of the ``install_dependencies`` directory from the OpenVINO installation directory. + * ```` — the directory with the user data (OpenVINO IR, model, dataset, etc.) specified while configuring the package. + +3. On a target Linux system, to run inference install additional dependencies by running the ``install_openvino_dependencies.sh`` script: + + .. code-block: sh + + cd /openvino/install_dependencies + sudo -E ./install_openvino_dependencies.sh + 4. Set up the environment variables: -@sphinxdirective .. tab:: Linux @@ -219,7 +201,7 @@ To deploy the OpenVINO Runtime components from the development machine to the ta cd /openvino/ source ./setupvars.sh -@endsphinxdirective - Now, you have finished the deployment of the OpenVINO Runtime components to the target system. + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/deployment/deployment_intro.md b/docs/OV_Runtime_UG/deployment/deployment_intro.md index df629a51e97574..fc9f4581c3792d 100644 --- a/docs/OV_Runtime_UG/deployment/deployment_intro.md +++ b/docs/OV_Runtime_UG/deployment/deployment_intro.md @@ -11,47 +11,69 @@ Deploy Application with Deployment Manager Local Distribution Libraries -@endsphinxdirective -> **NOTE**: Note that [running inference in OpenVINO Runtime](../openvino_intro.md) is the most basic form of deployment. Before moving forward, make sure you know how to create a proper Inference configuration and [develop your application properly](../integrate_with_your_application.md) +.. note:: + Note that :doc:`running inference in OpenVINO Runtime ` is the most basic form of deployment. Before moving forward, make sure you know how to create a proper Inference configuration and :doc:`develop your application properly `. -## Local Deployment Options +Local Deployment Options +######################## - Set a dependency on the existing prebuilt packages, also called "centralized distribution": - - using Debian / RPM packages - a recommended way for Linux operating systems; - - using PIP package manager on PyPI - the default approach for Python-based applications; - - using Docker images - if the application should be deployed as a Docker image, use a pre-built OpenVINO™ Runtime Docker image as a base image in the Dockerfile for the application container image. For more information about OpenVINO Docker images, refer to [Installing OpenVINO on Linux from Docker](../../install_guides/installing-openvino-docker-linux.md) and [Installing OpenVINO on Windows from Docker](../../install_guides/installing-openvino-docker-windows.md). -Furthermore, to customize your OpenVINO Docker image, use the [Docker CI Framework](https://github.com/openvinotoolkit/docker_ci) to generate a Dockerfile and built the image. + + - using Debian / RPM packages - a recommended way for Linux operating systems; + - using PIP package manager on PyPI - the default approach for Python-based applications; + - using Docker images - if the application should be deployed as a Docker image, use a pre-built OpenVINO™ Runtime Docker image as a base image in the Dockerfile for the application container image. For more information about OpenVINO Docker images, refer to :doc:`Installing OpenVINO on Linux from Docker ` and :doc:`Installing OpenVINO on Windows from Docker `. + +Furthermore, to customize your OpenVINO Docker image, use the `Docker CI Framework ` to generate a Dockerfile and built the image. + - Grab a necessary functionality of OpenVINO together with your application, also called "local distribution": - - using [OpenVINO Deployment Manager](deployment-manager-tool.md) - providing a convenient way for creating a distribution package; - - using the advanced [local distribution](local-distribution.md) approach; - - using [a static version of OpenVINO Runtime linked to the final app](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/static_libaries.md). + + - using :doc:`OpenVINO Deployment Manager ` - providing a convenient way for creating a distribution package; + - using the advanced :doc:`local distribution ` approach; + - using `a static version of OpenVINO Runtime linked to the final app `__. The table below shows which distribution type can be used for what target operating system: -| Distribution type | Operating systems | -|------- ---------- | ----------------- | -| Debian packages | Ubuntu 18.04 long-term support (LTS), 64-bit; Ubuntu 20.04 long-term support (LTS), 64-bit | -| RMP packages | Red Hat Enterprise Linux 8, 64-bit | -| Docker images | Ubuntu 18.04 long-term support (LTS), 64-bit; Ubuntu 20.04 long-term support (LTS), 64-bit; Red Hat Enterprise Linux 8, 64-bit; Windows Server Core base LTSC 2019, 64-bit; Windows 10, version 20H2, 64-bit | -| PyPI (PIP package manager) | See [https://pypi.org/project/openvino/](https://pypi.org/project/openvino/) | -| [OpenVINO Deployment Manager](deployment-manager-tool.md) | All operating systems | -| [Local distribution](local-distribution.md) | All operating systems | -| [Build OpenVINO statically and link to the final app](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/static_libaries.md) | All operating systems | +.. list-table:: + :header-rows: 1 + + * - Distribution type + - Operating systems + * - Debian packages + - Ubuntu 18.04 long-term support (LTS), 64-bit; Ubuntu 20.04 long-term support (LTS), 64-bit + * - RMP packages + - Red Hat Enterprise Linux 8, 64-bit + * - Docker images + - Ubuntu 18.04 long-term support (LTS), 64-bit; Ubuntu 20.04 long-term support (LTS), 64-bit; Red Hat Enterprise Linux 8, 64-bit; Windows Server Core base LTSC 2019, 64-bit; Windows 10, version 20H2, 64-bit + * - PyPI (PIP package manager) + - See https://pypi.org/project/openvino + * - :doc:`OpenVINO Deployment Manager ` + - All operating systems + * - :doc:`Libraries for Local Distribution ` + - All operating systems + * - `Build OpenVINO statically and link to the final app `__ + - All operating systems -## Granularity of Major Distribution Types -The granularity of OpenVINO packages may vary for different distribution types. For example, the PyPI distribution of OpenVINO has a [single 'openvino' package](https://pypi.org/project/openvino/) that contains all the runtime libraries and plugins, while a [local distribution](local-distribution.md) is a more configurable type providing higher granularity. Below are important details of the set of libraries included in the OpenVINO Runtime package: +Granularity of Major Distribution Types +####################################### -![](../../img/deployment_simplified.svg) +The granularity of OpenVINO packages may vary for different distribution types. For example, the PyPI distribution of OpenVINO has a `single 'openvino' package `__ that contains all the runtime libraries and plugins, while a :doc:`local distribution ` is a more configurable type providing higher granularity. Below are important details of the set of libraries included in the OpenVINO Runtime package: -- The main library `openvino` is used by users' C++ applications to link against with. The library provides all OpenVINO Runtime public APIs, including both API 2.0 and the previous Inference Engine and nGraph APIs. For C language applications, `openvino_c` is additionally required for distribution. -- The "optional" plugin libraries like `openvino_intel_cpu_plugin` (matching the `openvino_.+_plugin` pattern) are used to provide inference capabilities on specific devices or additional capabilities like [Hetero Execution](../hetero_execution.md) and [Multi-Device Execution](../multi_device.md). -- The "optional" plugin libraries like `openvino_ir_frontend` (matching `openvino_.+_frontend`) are used to provide capabilities to read models of different file formats such as OpenVINO IR, TensorFlow, ONNX, and PaddlePaddle. +.. image:: _static/images/deployment_simplified.svg + + +- The main library ``openvino`` is used by users' C++ applications to link against with. The library provides all OpenVINO Runtime public APIs, including both API 2.0 and the previous Inference Engine and nGraph APIs. For C language applications, ``openvino_c`` is additionally required for distribution. +- The "optional" plugin libraries like ``openvino_intel_cpu_plugin`` (matching the ``openvino_.+_plugin`` pattern) are used to provide inference capabilities on specific devices or additional capabilities like :doc:`Hetero Execution ` and :doc:`Multi-Device Execution `. +- The "optional" plugin libraries like ``openvino_ir_frontend`` (matching ``openvino_.+_frontend``) are used to provide capabilities to read models of different file formats such as OpenVINO IR, TensorFlow, ONNX, and PaddlePaddle. Here the term "optional" means that if the application does not use the capability enabled by the plugin, the plugin library or a package with the plugin is not needed in the final distribution. -Building a local distribution will require more detailed information, and you will find it in the dedicated [Libraries for Local Distribution](local-distribution.md) article. +Building a local distribution will require more detailed information, and you will find it in the dedicated :doc:`Libraries for Local Distribution ` article. + +.. note:: + + Depending on your target OpenVINO devices, the following configurations might be needed for deployed machines: :doc:`Configurations for GPU `, :doc:`Configurations for GNA `. -> **NOTE**: Depending on your target OpenVINO devices, the following configurations might be needed for deployed machines: [Configurations for GPU](../../install_guides/configurations-for-intel-gpu.md), [Configurations for GNA](../../install_guides/configurations-for-intel-gna.md). +@endsphinxdirective \ No newline at end of file diff --git a/docs/OV_Runtime_UG/deployment/local-distribution.md b/docs/OV_Runtime_UG/deployment/local-distribution.md index 1b4e02143ad1dc..cd68ac4bdbf085 100644 --- a/docs/OV_Runtime_UG/deployment/local-distribution.md +++ b/docs/OV_Runtime_UG/deployment/local-distribution.md @@ -1,155 +1,163 @@ # Libraries for Local Distribution {#openvino_docs_deploy_local_distribution} +@sphinxdirective + With a local distribution, each C or C++ application/installer will have its own copies of OpenVINO Runtime binaries. However, OpenVINO has a scalable plugin-based architecture, which means that some components can be loaded in runtime only when they are really needed. Therefore, it is important to understand which minimal set of libraries is really needed to deploy the application. This guide helps you to achieve that goal. +Local distribution is also appropriate for OpenVINO binaries built from sources using `Build instructions `__, +but the guide below supposes OpenVINO Runtime is built dynamically. For case of `Static OpenVINO Runtime `__ select the required OpenVINO capabilities on CMake configuration stage using `CMake Options for Custom Compilation `__, the build and link the OpenVINO components into the final application. -Local dsitribution is also appropriate for OpenVINO binaries built from sources using [Build instructions](https://github.com/openvinotoolkit/openvino/wiki#how-to-build), but the guide below supposes OpenVINO Runtime is built dynamically. For case of [Static OpenVINO Runtime](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/static_libaries.md) select the required OpenVINO capabilities on CMake configuration stage using [CMake Options for Custom Compilation](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/cmake_options_for_custom_comiplation.md), the build and link the OpenVINO components into the final application. +.. note:: -> **NOTE**: The steps below are operating system independent and refer to a library file name without any prefixes (like `lib` on Unix systems) or suffixes (like `.dll` on Windows OS). Do not put `.lib` files on Windows OS to the distribution, because such files are needed only on a linker stage. + The steps below are operating system independent and refer to a library file name without any prefixes (like ``lib`` on Unix systems) or suffixes (like ``.dll`` on Windows OS). Do not put ``.lib`` files on Windows OS to the distribution, because such files are needed only on a linker stage. -## Library Requirements for C++ and C Languages -Independent on the language used to write the application, the `openvino` library must always be put to the final distribution, since it's a core library which orchestrates with all the inference and frontend plugins. In Intel® Distribution of OpenVINO™ toolkit, `openvino` depends on the TBB libraries which are used by OpenVINO Runtime to optimally saturate the devices with computations, so it must be put to the distribution package. +Library Requirements for C++ and C Languages +############################################ -If your application is written with C language, you need to put the `openvino_c` library additionally. +Independent on the language used to write the application, the ``openvino`` library must always be put to the final distribution, since it's a core library which orchestrates with all the inference and frontend plugins. In Intel® Distribution of OpenVINO™ toolkit, ``openvino`` depends on the TBB libraries which are used by OpenVINO Runtime to optimally saturate the devices with computations, so it must be put to the distribution package. -The `plugins.xml` file with information about inference devices must also be taken as a support file for `openvino`. +If your application is written with C language, you need to put the ``openvino_c`` library additionally. +The ``plugins.xml`` file with information about inference devices must also be taken as a support file for ``openvino``. -## Libraries for Pluggable Components + +Libraries for Pluggable Components +################################## The picture below presents dependencies between the OpenVINO Runtime core and pluggable libraries: -![](../../img/deployment_full.svg) +.. image:: _static/images/deployment_full.svg -### Libraries for Compute Devices +Libraries for Compute Devices ++++++++++++++++++++++++++++++ For each inference device, OpenVINO Runtime has its own plugin library: -- `openvino_intel_cpu_plugin` for [Intel® CPU devices](../supported_plugins/CPU.md). -- `openvino_intel_gpu_plugin` for [Intel® GPU devices](../supported_plugins/GPU.md). -- `openvino_intel_gna_plugin` for [Intel® GNA devices](../supported_plugins/GNA.md). -- `openvino_arm_cpu_plugin` for [ARM CPU devices](../supported_plugins/ARM_CPU.md). + +- ``openvino_intel_cpu_plugin`` for :doc:`Intel® CPU devices `. +- ``openvino_intel_gpu_plugin`` for :doc:`Intel® GPU devices `. +- ``openvino_intel_gna_plugin`` for :doc:`Intel® GNA devices `. +- ``openvino_arm_cpu_plugin`` for :doc:`ARM CPU devices `. Depending on what devices are used in the app, the appropriate libraries need to be put to the distribution package. As it is shown on the picture above, some plugin libraries may have OS-specific dependencies which are either backend libraries or additional supports files with firmware, etc. Refer to the table below for details: -@sphinxdirective +.. dropdown:: Windows OS: -.. raw:: html + .. list-table:: + :header-rows: 1 -
+ * - Device + - Dependency + * - CPU + - ``-`` + * - GPU + - ``OpenCL.dll``, ``cache.json`` + * - GNA + - ``gna.dll`` + * - Arm® CPU + - ``-`` -@endsphinxdirective -| Device | Dependency | -|-------------|------------| -| CPU | `-` | -| GPU | `OpenCL.dll`, `cache.json` | -| GNA | `gna.dll` | -| Arm® CPU | `-` | +.. dropdown:: Linux OS: -@sphinxdirective + .. list-table:: + :header-rows: 1 -.. raw:: html + * - Device + - Dependency + * - CPU + - ``-`` + * - GPU + - ``libOpenCL.so``, ``cache.json`` + * - GNA + - ``gna.dll`` + * - Arm® CPU + - ``-`` -
-@endsphinxdirective -@sphinxdirective +.. dropdown:: MacOS: -.. raw:: html + .. list-table:: + :header-rows: 1 -
+ * - Device + - Dependency + * - CPU + - ``-`` + * - Arm® CPU + - ``-`` -@endsphinxdirective -| Device | Dependency | -|-------------|-------------| -| CPU | `-` | -| GPU | `libOpenCL.so`, `cache.json` | -| GNA | `gna.dll` | -| Arm® CPU | `-` | +Libraries for Execution Modes ++++++++++++++++++++++++++++++ -@sphinxdirective +The ``HETERO``, ``MULTI``, ``BATCH`` and ``AUTO`` execution modes can also be used explicitly or implicitly by the application. Use the following recommendation scheme to decide whether to put the appropriate libraries to the distribution package: -.. raw:: html +- If :doc:`AUTO ` is used explicitly in the application or `ov::Core::compile_model `__ is used without specifying a device, put ``openvino_auto_plugin`` to the distribution. -
+ .. note:: -@endsphinxdirective -@sphinxdirective - -.. raw:: html + Automatic Device Selection relies on :doc:`[inference device plugins `. If you are not sure about what inference devices are available on target system, put all the inference plugin libraries to the distribution. If `ov::device::priorities `__ is used for `AUTO` to specify a limited device list, grab the corresponding device plugins only. -
+- If :doc:`MULTI ` is used explicitly, put ``openvino_auto_plugin`` to the distribution. +- If :doc:`HETERO ` is either used explicitly or `ov::hint::performance_mode `__ is used with GPU, put ``openvino_hetero_plugin`` to the distribution. +- If :doc:`BATCH ` is either used explicitly or ``ov::hint::performance_mode`` is used with GPU, put ``openvino_batch_plugin`` to the distribution. -@endsphinxdirective +Frontend Libraries for Reading Models ++++++++++++++++++++++++++++++++++++++ -| Device | Dependency | -|-------------|-------------| -| CPU | `-` | -| Arm® CPU | `-` | - -@sphinxdirective - -.. raw:: html - -
- -@endsphinxdirective - -### Libraries for Execution Modes - -The `HETERO`, `MULTI`, `BATCH` and `AUTO` execution modes can also be used explicitly or implicitly by the application. Use the following recommendation scheme to decide whether to put the appropriate libraries to the distribution package: -- If [AUTO](../auto_device_selection.md) is used explicitly in the application or `ov::Core::compile_model` is used without specifying a device, put `openvino_auto_plugin` to the distribution. - > **NOTE**: Automatic Device Selection relies on [inference device plugins](../supported_plugins/Device_Plugins.md). If you are not sure about what inference devices are available on target system, put all the inference plugin libraries to the distribution. If `ov::device::priorities` is used for `AUTO` to specify a limited device list, grab the corresponding device plugins only. +OpenVINO Runtime uses frontend libraries dynamically to read models in different formats: -- If [MULTI](../multi_device.md) is used explicitly, put `openvino_auto_plugin` to the distribution. -- If [HETERO](../hetero_execution.md) is either used explicitly or `ov::hint::performance_mode` is used with GPU, put `openvino_hetero_plugin` to the distribution. -- If [BATCH](../automatic_batching.md) is either used explicitly or `ov::hint::performance_mode` is used with GPU, put `openvino_batch_plugin` to the distribution. +- ``openvino_ir_frontend`` is used to read OpenVINO IR. +- ``openvino_tensorflow_frontend`` is used to read TensorFlow file format. +- ``openvino_onnx_frontend`` is used to read ONNX file format. +- ``openvino_paddle_frontend`` is used to read Paddle file format. -### Frontend Libraries for Reading Models +Depending on the model format types that are used in the application in `ov::Core::read_model `__, pick up the appropriate libraries. -OpenVINO Runtime uses frontend libraries dynamically to read models in different formats: -- `openvino_ir_frontend` is used to read OpenVINO IR. -- `openvino_tensorflow_frontend` is used to read TensorFlow file format. -- `openvino_onnx_frontend` is used to read ONNX file format. -- `openvino_paddle_frontend` is used to read Paddle file format. +.. note:: -Depending on the model format types that are used in the application in `ov::Core::read_model`, pick up the appropriate libraries. + To optimize the size of final distribution package, you are recommended to convert models to OpenVINO IR by using :doc:`Model Optimizer `. This way you don't have to keep TensorFlow, ONNX, PaddlePaddle, and other frontend libraries in the distribution package. -> **NOTE**: To optimize the size of final distribution package, you are recommended to convert models to OpenVINO IR by using [Model Optimizer](../../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). This way you don't have to keep TensorFlow, ONNX, PaddlePaddle, and other frontend libraries in the distribution package. +(Legacy) Preprocessing via G-API +++++++++++++++++++++++++++++++++ -### (Legacy) Preprocessing via G-API +.. note:: -> **NOTE**: [G-API](../../gapi/gapi_intro.md) preprocessing is a legacy functionality, use [preprocessing capabilities from OpenVINO 2.0](../preprocessing_overview.md) which do not require any additional libraries. + :doc:`G-API ` preprocessing is a legacy functionality, use :doc:`preprocessing capabilities from OpenVINO 2.0 ` which do not require any additional libraries. -If the application uses `InferenceEngine::PreProcessInfo::setColorFormat` or `InferenceEngine::PreProcessInfo::setResizeAlgorithm` methods, OpenVINO Runtime dynamically loads `openvino_gapi_preproc` plugin to perform preprocessing via G-API. +If the application uses `InferenceEngine::PreProcessInfo::setColorFormat `__ or `InferenceEngine::PreProcessInfo::setResizeAlgorithm `__ methods, OpenVINO Runtime dynamically loads `openvino_gapi_preproc` plugin to perform preprocessing via G-API. -## Examples +Examples +#################### **CPU + OpenVINO IR in C application** In this example, the application is written in C language, performs inference on CPU, and reads models stored as the OpenVINO IR format. The following libraries are used: -- The `openvino_c` library is a main dependency of the application. It links against this library. -- The `openvino` library is used as a private dependency for `openvino_c` and is also used in the deployment. -- `openvino_intel_cpu_plugin` is used for inference. -- `openvino_ir_frontend` is used to read source models. +- The ``openvino_c`` library is a main dependency of the application. It links against this library. +- The ``openvino`` library is used as a private dependency for ``openvino_c`` and is also used in the deployment. +- ``openvino_intel_cpu_plugin`` is used for inference. +- ``openvino_ir_frontend`` is used to read source models. **MULTI execution on GPU and CPU in `tput` mode** -In this example, the application is written in C++, performs inference [simultaneously on GPU and CPU devices](../multi_device.md) with the `ov::hint::PerformanceMode::THROUGHPUT` property set, and reads models stored in the ONNX format. The following libraries are used: -- The `openvino` library is a main dependency of the application. It links against this library. -- `openvino_intel_gpu_plugin` and `openvino_intel_cpu_plugin` are used for inference. -- `openvino_auto_plugin` is used for Multi-Device Execution. -- `openvino_auto_batch_plugin` can be also put to the distribution to improve the saturation of [Intel® GPU](../supported_plugins/GPU.md) device. If there is no such plugin, [Automatic Batching](../automatic_batching.md) is turned off. -- `openvino_onnx_frontend` is used to read source models. +In this example, the application is written in C++, performs inference :doc:`simultaneously on GPU and CPU devices ` with the `ov::hint::PerformanceMode::THROUGHPUT `__ property set, and reads models stored in the ONNX format. The following libraries are used: + +- The ``openvino`` library is a main dependency of the application. It links against this library. +- ``openvino_intel_gpu_plugin`` and ``openvino_intel_cpu_plugin`` are used for inference. +- ``openvino_auto_plugin`` is used for Multi-Device Execution. +- ``openvino_auto_batch_plugin`` can be also put to the distribution to improve the saturation of :doc:`Intel® GPU ` device. If there is no such plugin, :doc:`Automatic Batching ` is turned off. +- ``openvino_onnx_frontend`` is used to read source models. **Auto-Device Selection between GPU and CPU** -In this example, the application is written in C++, performs inference with the [Automatic Device Selection](../auto_device_selection.md) mode, limiting device list to GPU and CPU, and reads models [created using C++ code](../model_representation.md). The following libraries are used: -- The `openvino` library is a main dependency of the application. It links against this library. -- `openvino_auto_plugin` is used to enable Automatic Device Selection. -- `openvino_intel_gpu_plugin` and `openvino_intel_cpu_plugin` are used for inference. AUTO selects between CPU and GPU devices according to their physical existence on the deployed machine. -- No frontend library is needed because `ov::Model` is created in code. +In this example, the application is written in C++, performs inference with the :doc:`Automatic Device Selection ` mode, limiting device list to GPU and CPU, and reads models :doc:`created using C++ code `. The following libraries are used: + +- The ``openvino`` library is a main dependency of the application. It links against this library. +- ``openvino_auto_plugin`` is used to enable Automatic Device Selection. +- ``openvino_intel_gpu_plugin`` and ``openvino_intel_cpu_plugin`` are used for inference. AUTO selects between CPU and GPU devices according to their physical existence on the deployed machine. +- No frontend library is needed because ``ov::Model`` is created in code. + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/deployment_migration.md b/docs/OV_Runtime_UG/migration_ov_2_0/deployment_migration.md index f5d9a1c4213ca1..46d8a693094cc9 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/deployment_migration.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/deployment_migration.md @@ -1,151 +1,161 @@ # Installation & Deployment {#openvino_2_0_deployment} +@sphinxdirective + One of the main concepts for OpenVINO™ API 2.0 is being "easy to use", which includes: + * Simplification of migration from different frameworks to OpenVINO. -* Organization of OpenVINO. +* Organization of OpenVINO. * Usage of development tools. * Development and deployment of OpenVINO-based applications. + To accomplish that, the 2022.1 release OpenVINO introduced significant changes to the installation and deployment processes. This guide will walk you through these changes. -## The Installer Package Contains OpenVINO™ Runtime Only +The Installer Package Contains OpenVINO™ Runtime Only +##################################################### -Since OpenVINO 2022.1, development tools have been distributed only via [PyPI](https://pypi.org/project/openvino-dev/), and are no longer included in the OpenVINO installer package. For a list of these components, refer to the [installation overview](../../install_guides/installing-openvino-overview.md) guide. Benefits of this approach include: +Since OpenVINO 2022.1, development tools have been distributed only via `PyPI `__, and are no longer included in the OpenVINO installer package. For a list of these components, refer to the :doc:`installation overview ` guide. Benefits of this approach include: -* simplification of the user experience - in previous versions, installation and usage of OpenVINO Development Tools differed from one distribution type to another (the OpenVINO installer vs. PyPI), +* simplification of the user experience - in previous versions, installation and usage of OpenVINO Development Tools differed from one distribution type to another (the OpenVINO installer vs. PyPI), * ensuring that dependencies are handled properly via the PIP package manager, and support virtual environments of development tools. The structure of the OpenVINO 2022.1 installer package has been organized as follows: -- The `runtime` folder includes headers, libraries and CMake interfaces. -- The `tools` folder contains [the compile tool](../../../tools/compile_tool/README.md), [deployment manager](../../OV_Runtime_UG/deployment/deployment-manager-tool.md), and a set of `requirements.txt` files with links to the corresponding versions of the `openvino-dev` package. -- The `python` folder contains the Python version for OpenVINO Runtime. +* The ``runtime`` folder includes headers, libraries and CMake interfaces. +* The ``tools`` folder contains :doc:`the compile tool `, :doc:`deployment manager `, and a set of ``requirements.txt`` files with links to the corresponding versions of the ``openvino-dev`` package. +* The ``python`` folder contains the Python version for OpenVINO Runtime. -## Installing OpenVINO Development Tools via PyPI +Installing OpenVINO Development Tools via PyPI +############################################## Since OpenVINO Development Tools is no longer in the installer package, the installation process has also changed. This section describes it through a comparison with previous versions. -### For Versions Prior to 2022.1 +For Versions Prior to 2022.1 +++++++++++++++++++++++++++++ + +In previous versions, OpenVINO Development Tools was a part of the main package. After the package was installed, to convert models (for example, TensorFlow), you needed to install additional dependencies by using the requirement files, such as ``requirements_tf.txt``, install Post-Training Optimization tool and Accuracy Checker tool via the ``setup.py`` scripts, and then use the ``setupvars`` scripts to make the tools available to the following command: -In previous versions, OpenVINO Development Tools was a part of the main package. After the package was installed, to convert models (for example, TensorFlow), you needed to install additional dependencies by using the requirement files, such as `requirements_tf.txt`, install Post-Training Optimization tool and Accuracy Checker tool via the `setup.py` scripts, and then use the `setupvars` scripts to make the tools available to the following command: +.. code-block:: sh -```sh -$ mo.py -h -``` + $ mo.py -h -### For 2022.1 and After -In OpenVINO 2022.1 and later, you can install the development tools only from a [PyPI](https://pypi.org/project/openvino-dev/) repository, using the following command (taking TensorFlow as an example): +For 2022.1 and After +++++++++++++++++++++ -```sh -$ python3 -m pip install -r /tools/requirements_tf.txt -``` +In OpenVINO 2022.1 and later, you can install the development tools only from a `PyPI `__ repository, using the following command (taking TensorFlow as an example): -This will install all the development tools and additional components necessary to work with TensorFlow via the `openvino-dev` package (see **Step 4. Install the Package** on the [PyPI page](https://pypi.org/project/openvino-dev/) for parameters of other frameworks). +.. code-block:: sh + + $ python3 -m pip install -r /tools/requirements_tf.txt + + +This will install all the development tools and additional components necessary to work with TensorFlow via the ``openvino-dev`` package (see **Step 4. Install the Package** on the `PyPI page `__ for parameters of other frameworks). Then, the tools can be used by commands like: -```sh -$ mo -h -$ pot -h -``` +.. code-block:: sh + + $ mo -h + $ pot -h -Installation of any other dependencies is not required. For more details on the installation steps, see the [Install OpenVINO Development Tools](../../install_guides/installing-model-dev-tools.md). -## Interface Changes for Building C/C++ Applications +Installation of any other dependencies is not required. For more details on the installation steps, see the :doc:`Install OpenVINO Development Tools `. + +Interface Changes for Building C/C++ Applications +################################################# The new OpenVINO Runtime with its API 2.0 has also brought some changes for building C/C++ applications. -### CMake Interface +CMake Interface +++++++++++++++++++++ The CMake interface has been changed as follows: **With Inference Engine of previous versions**: -```cmake -find_package(InferenceEngine REQUIRED) -find_package(ngraph REQUIRED) -add_executable(ie_ngraph_app main.cpp) -target_link_libraries(ie_ngraph_app PRIVATE ${InferenceEngine_LIBRARIES} ${NGRAPH_LIBRARIES}) -``` +.. code-block:: cmake + + find_package(InferenceEngine REQUIRED) + find_package(ngraph REQUIRED) + add_executable(ie_ngraph_app main.cpp) + target_link_libraries(ie_ngraph_app PRIVATE ${InferenceEngine_LIBRARIES} ${NGRAPH_LIBRARIES}) + **With OpenVINO Runtime 2022.1 (API 2.0)**: -```cmake -find_package(OpenVINO REQUIRED) -add_executable(ov_app main.cpp) -target_link_libraries(ov_app PRIVATE openvino::runtime) +.. code-block:: cmake + + find_package(OpenVINO REQUIRED) + add_executable(ov_app main.cpp) + target_link_libraries(ov_app PRIVATE openvino::runtime) -add_executable(ov_c_app main.c) -target_link_libraries(ov_c_app PRIVATE openvino::runtime::c) -``` + add_executable(ov_c_app main.c) + target_link_libraries(ov_c_app PRIVATE openvino::runtime::c) -### Native Interfaces + +Native Interfaces +++++++++++++++++++++ It is possible to build applications without the CMake interface by using: MSVC IDE, UNIX makefiles, and any other interface, which has been changed as shown here: **With Inference Engine of previous versions**: -@sphinxdirective - .. tab:: Include dirs - .. code-block:: sh - - /deployment_tools/inference_engine/include - /deployment_tools/ngraph/include + .. code-block:: sh + + /deployment_tools/inference_engine/include + /deployment_tools/ngraph/include .. tab:: Path to libs - .. code-block:: sh + .. code-block:: sh - /deployment_tools/inference_engine/lib/intel64/Release - /deployment_tools/ngraph/lib/ + /deployment_tools/inference_engine/lib/intel64/Release + /deployment_tools/ngraph/lib/ .. tab:: Shared libs - .. code-block:: sh + .. code-block:: sh - // UNIX systems - inference_engine.so ngraph.so + // UNIX systems + inference_engine.so ngraph.so - // Windows - inference_engine.dll ngraph.dll + // Windows + inference_engine.dll ngraph.dll .. tab:: (Windows) .lib files - .. code-block:: sh - - ngraph.lib - inference_engine.lib + .. code-block:: sh -@endsphinxdirective + ngraph.lib + inference_engine.lib **With OpenVINO Runtime 2022.1 (API 2.0)**: -@sphinxdirective - .. tab:: Include dirs - .. code-block:: sh + .. code-block:: sh - /runtime/include + /runtime/include .. tab:: Path to libs - .. code-block:: sh + .. code-block:: sh - /runtime/lib/intel64/Release + /runtime/lib/intel64/Release .. tab:: Shared libs - .. code-block:: sh + .. code-block:: sh - // UNIX systems - openvino.so + // UNIX systems + openvino.so - // Windows - openvino.dll + // Windows + openvino.dll .. tab:: (Windows) .lib files @@ -153,49 +163,55 @@ It is possible to build applications without the CMake interface by using: MSVC openvino.lib -@endsphinxdirective -## Clearer Library Structure for Deployment +Clearer Library Structure for Deployment +######################################## -OpenVINO 2022.1 introduced a reorganization of the libraries, to make deployment easier. In the previous versions, it was required to use several libraries to perform deployment steps. Now you can just use `openvino` or `openvino_c` based on your developing language, with the necessary plugins to complete your task. For example, `openvino_intel_cpu_plugin` and `openvino_ir_frontend` plugins will enable loading OpenVINO IRs and performing inference on the CPU device (for more details, see the [Local distribution with OpenVINO](../deployment/local-distribution.md)). +OpenVINO 2022.1 introduced a reorganization of the libraries, to make deployment easier. In the previous versions, it was required to use several libraries to perform deployment steps. Now you can just use ``openvino`` or ``openvino_c`` based on your developing language, with the necessary plugins to complete your task. For example, ``openvino_intel_cpu_plugin`` and ``openvino_ir_frontend`` plugins will enable loading OpenVINO IRs and performing inference on the CPU device (for more details, see the :doc:`Local distribution with OpenVINO `). Below are detailed comparisons of the library structure between OpenVINO 2022.1 and the previous versions: -* Starting with 2022.1 release, a single core library with all the functionalities (`openvino` for C++ Runtime, `openvino_c` for Inference Engine API C interface) is used, instead of the previous core libraries which contained `inference_engine`, `ngraph`, `inference_engine_transformations` and `inference_engine_lp_transformations`. -* The optional `inference_engine_preproc` preprocessing library (if `InferenceEngine::PreProcessInfo::setColorFormat` or `InferenceEngine::PreProcessInfo::setResizeAlgorithm` is used) has been renamed to `openvino_gapi_preproc` and deprecated in 2022.1. For more details, see the [Preprocessing capabilities of OpenVINO API 2.0](preprocessing.md). +* Starting with 2022.1 release, a single core library with all the functionalities (``openvino`` for C++ Runtime, ``openvino_c`` for Inference Engine API C interface) is used, instead of the previous core libraries which contained ``inference_engine``, ``ngraph``, ``inference_engine_transformations`` and ``inference_engine_lp_transformations``. +* The optional ``inference_engine_preproc`` preprocessing library (if `InferenceEngine::PreProcessInfo::setColorFormat `__ or `InferenceEngine::PreProcessInfo::setResizeAlgorithm `__ is used) has been renamed to ``openvino_gapi_preproc`` and deprecated in 2022.1. For more details, see the :doc:`Preprocessing capabilities of OpenVINO API 2.0 `. + * The libraries of plugins have been renamed as follows: - * `openvino_intel_cpu_plugin` is used for [CPU](../supported_plugins/CPU.md) device instead of `MKLDNNPlugin`. - * `openvino_intel_gpu_plugin` is used for [GPU](../supported_plugins/GPU.md) device instead of `clDNNPlugin`. - * `openvino_auto_plugin` is used for [Auto-Device Plugin](../auto_device_selection.md). + + * ``openvino_intel_cpu_plugin`` is used for :doc:`CPU ` device instead of ``MKLDNNPlugin``. + * ``openvino_intel_gpu_plugin`` is used for :doc:`GPU ` device instead of ``clDNNPlugin``. + * ``openvino_auto_plugin`` is used for :doc:`Auto-Device Plugin `. + * The plugins for reading and converting models have been changed as follows: - * `openvino_ir_frontend` is used to read IRs instead of `inference_engine_ir_reader`. - * `openvino_onnx_frontend` is used to read ONNX models instead of `inference_engine_onnx_reader` (with its dependencies). - * `openvino_paddle_frontend` is added in 2022.1 to read PaddlePaddle models. + + * ``openvino_ir_frontend`` is used to read IRs instead of ``inference_engine_ir_reader``. + * ``openvino_onnx_frontend`` is used to read ONNX models instead of ``inference_engine_onnx_reader`` (with its dependencies). + * ``openvino_paddle_frontend`` is added in 2022.1 to read PaddlePaddle models. + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/intro.md b/docs/OV_Runtime_UG/migration_ov_2_0/intro.md index 290610326492a8..80438604a56eb1 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/intro.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/intro.md @@ -12,83 +12,101 @@ openvino_2_0_preprocessing openvino_2_0_model_creation -@endsphinxdirective -This guide introduces the new OpenVINO™ API: API 2.0, as well as the new OpenVINO IR model format: IR v11. Here, you will find comparisons of their "old" and "new" versions. +This guide introduces the new OpenVINO™ API: API 2.0, as well as the new OpenVINO IR model format: IR v11. Here, you will find comparisons of their "old" and "new" versions. -### Introduction of API 2.0 +Introduction of API 2.0 +####################### Versions of OpenVINO prior to 2022.1 required changes in the application logic when migrating an app from other frameworks, such as TensorFlow, ONNX Runtime, PyTorch, PaddlePaddle, etc. The changes were required because: -- Model Optimizer changed input precisions for some inputs. For example, neural language processing models with `I64` inputs were changed to include `I32` ones. -- Model Optimizer changed layouts for TensorFlow models (see the [Layouts in OpenVINO](../layout_overview.md)). It lead to unusual requirement of using the input data with a different layout than that of the framework: -![tf_openvino] -- Inference Engine API (`InferenceEngine::CNNNetwork`) applied some conversion rules for input and output precisions due to limitations in device plugins. +- Model Optimizer changed input precisions for some inputs. For example, neural language processing models with ``I64`` inputs were changed to include ``I32`` ones. +- Model Optimizer changed layouts for TensorFlow models (see the :doc:`Layouts in OpenVINO `). It lead to unusual requirement of using the input data with a different layout than that of the framework: + +.. image:: _static/images/tf_openvino.svg + :alt: tf_openvino + +- Inference Engine API (`InferenceEngine::CNNNetwork `__) applied some conversion rules for input and output precisions due to limitations in device plugins. - Users needed to specify input shapes during model conversions in Model Optimizer, and work with static shapes in the application. -OpenVINO™ 2022.1 has introduced API 2.0 (also called OpenVINO API v2) to align the logic of working with models as it is done in their origin frameworks - no layout and precision changes, operating with tensor names and indices to address inputs and outputs. OpenVINO Runtime has combined Inference Engine API used for inference and nGraph API targeted to work with models and operations. API 2.0 has a common structure, naming convention styles, namespaces, and removes duplicated structures. For more details, see the [Changes to Inference Pipeline in OpenVINO API v2](common_inference_pipeline.md). +OpenVINO™ 2022.1 has introduced API 2.0 (also called OpenVINO API v2) to align the logic of working with models as it is done in their origin frameworks - no layout and precision changes, operating with tensor names and indices to address inputs and outputs. OpenVINO Runtime has combined Inference Engine API used for inference and nGraph API targeted to work with models and operations. API 2.0 has a common structure, naming convention styles, namespaces, and removes duplicated structures. For more details, see the :doc:`Changes to Inference Pipeline in OpenVINO API v2 `. + +.. note:: + + Your existing applications will continue to work with OpenVINO Runtime 2022.1, as normal. Although, migration to API 2.0 is strongly recommended. This will allow you to use additional features, such as :doc:`Preprocessing ` and :doc:`Dynamic shapes support `. -> **NOTE**: Your existing applications will continue to work with OpenVINO Runtime 2022.1, as normal. Although, migration to API 2.0 is strongly recommended. This will allow you to use additional features, such as [Preprocessing](../preprocessing_overview.md) and [Dynamic shapes support](../ov_dynamic_shapes.md). -### The New OpenVINO IR v11 +The New OpenVINO IR v11 +####################### -To support these features, OpenVINO has introduced OpenVINO IR v11, which is now the default version for Model Optimizer. The model represented in OpenVINO IR v11 fully matches the original model in the original framework format in terms of inputs and outputs. It is also not required to specify input shapes during conversion, which results in OpenVINO IR v11 containing `-1` to denote undefined dimensions. For more details on how to fully utilize this feature, see [Working with dynamic shapes](../ov_dynamic_shapes.md). For information on how to reshape to static shapes in application, see [Changing input shapes](../ShapeInference.md). +To support these features, OpenVINO has introduced OpenVINO IR v11, which is now the default version for Model Optimizer. The model represented in OpenVINO IR v11 fully matches the original model in the original framework format in terms of inputs and outputs. It is also not required to specify input shapes during conversion, which results in OpenVINO IR v11 containing ``-1`` to denote undefined dimensions. For more details on how to fully utilize this feature, see :doc:`Working with dynamic shapes `. For information on how to reshape to static shapes in application, see :doc:`Changing input shapes `. OpenVINO IR v11 is fully compatible with applications written with the Inference Engine API used by older versions of OpenVINO. This backward compatibility is allowed thanks to additional runtime information included in OpenVINO IR v11. This means that when OpenVINO IR v11 is read by an application based on Inference Engine, it is internally converted to OpenVINO IR v10. OpenVINO IR v11 is supported by all OpenVINO Development tools including Post-Training Optimization Tool, Benchmark app, etc. -### Backward Compatibility for OpenVINO IR v10 +Backward Compatibility for OpenVINO IR v10 +########################################## -API 2.0 also supports backward compatibility for models of OpenVINO IR v10. If you have OpenVINO IR v10 files, they can also be fed to OpenVINO Runtime. For more details, see the [migration steps](common_inference_pipeline.md). +API 2.0 also supports backward compatibility for models of OpenVINO IR v10. If you have OpenVINO IR v10 files, they can also be fed to OpenVINO Runtime. For more details, see the :doc:`migration steps `. Some of the OpenVINO Development Tools also support both OpenVINO IR v10 and v11 as an input: -- Accuracy checker uses API 2.0 for model accuracy measurement by default. It also supports switching to the old API by using the `--use_new_api False` command-line parameter. Both launchers accept OpenVINO IR v10 and v11, but in some cases configuration files should be updated. For more details, see the [Accuracy Checker documentation](https://github.com/openvinotoolkit/open_model_zoo/blob/master/tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/openvino_launcher_readme.md). -- [Compile tool](../../../tools/compile_tool/README.md) compiles the model to be used in API 2.0 by default. To use the resulting compiled blob under the Inference Engine API, the additional `ov_api_1_0` option should be passed. -However, Post-Training Optimization Tool of OpenVINO 2022.1 does not support OpenVINO IR v10. They require the latest version of Model Optimizer to generate OpenVINO IR v11 files. +- Accuracy checker uses API 2.0 for model accuracy measurement by default. It also supports switching to the old API by using the ``--use_new_api False`` command-line parameter. Both launchers accept OpenVINO IR v10 and v11, but in some cases configuration files should be updated. For more details, see the `Accuracy Checker documentation `__. +- :doc:`Compile tool ` compiles the model to be used in API 2.0 by default. To use the resulting compiled blob under the Inference Engine API, the additional ``ov_api_1_0`` option should be passed. -> **NOTE**: To quantize your OpenVINO IR v10 models to run with OpenVINO 2022.1, download and use Post-Training Optimization Tool of OpenVINO 2021.4. +However, Post-Training Optimization Tool of OpenVINO 2022.1 does not support OpenVINO IR v10. They require the latest version of Model Optimizer to generate OpenVINO IR v11 files. -@sphinxdirective +.. note:: -.. _differences_api20_ie: + To quantize your OpenVINO IR v10 models to run with OpenVINO 2022.1, download and use Post-Training Optimization Tool of OpenVINO 2021.4. -@endsphinxdirective +.. _differences_api20_ie: -### Differences in API 2.0 and Inference Engine API Behaviors +Differences in API 2.0 and Inference Engine API Behaviors +######################################################### Inference Engine and nGraph APIs do not become deprecated with the introduction of the new API, and they can still be used in applications. However, it is highly recommended to migrate to API 2.0, as it offers more features (further extended in future releases), such as: -- [Working with dynamic shapes](../ov_dynamic_shapes.md), which increases performance when working with compatible models such as NLP (Neural Language Processing) and super-resolution models. -- [Preprocessing of the model](../preprocessing_overview.md), which adds preprocessing operations to inference models and fully occupies the accelerator, freeing CPU resources. + +- :doc:`Working with dynamic shapes `, which increases performance when working with compatible models such as NLP (Neural Language Processing) and super-resolution models. +- :doc:`Preprocessing of the model `, which adds preprocessing operations to inference models and fully occupies the accelerator, freeing CPU resources. To understand the differences between Inference Engine API and API 2.0, see the definitions of two types of behaviors first: + - **Old behavior** of OpenVINO assumes that: + - Model Optimizer can change input element types and order of dimensions (layouts) for the model from the original framework. - Inference Engine can override input and output element types. - - Inference Engine API uses operation names to address inputs and outputs (e.g. InferenceEngine::InferRequest::GetBlob). + - Inference Engine API uses operation names to address inputs and outputs (e.g. `InferenceEngine::InferRequest::GetBlob `__). - Inference Engine API does not support compiling of models with dynamic input shapes. + - **New behavior** implemented in 2022.1 assumes full model alignment with the framework: + - Model Optimizer preserves input element types and order of dimensions (layouts), and stores tensor names from the original models. - - OpenVINO Runtime 2022.1 reads models in any format (OpenVINO IR v10, OpenVINO IR v11, TensorFlow (check [TensorFlow Frontend Capabilities and Limitations](../../resources/tensorflow_frontend.md)), ONNX, PaddlePaddle, etc.). + - OpenVINO Runtime 2022.1 reads models in any format (OpenVINO IR v10, OpenVINO IR v11, TensorFlow (check :doc:`TensorFlow Frontend Capabilities and Limitations `), ONNX, PaddlePaddle, etc.). - API 2.0 uses tensor names for addressing, which is the standard approach among the compatible model frameworks. - API 2.0 can also address input and output tensors by the index. Some model formats like ONNX are sensitive to the input and output order, which is preserved by OpenVINO 2022.1. The table below demonstrates which behavior, **old** or **new**, is used for models based on the two APIs. -| API | OpenVINO IR v10 | OpenVINO IR v11 | ONNX Files | Models Created in Code | -|-------------------------------|------------------|------------------|------------|------------------------| -|Inference Engine / nGraph APIs | Old | Old | Old | Old | -|API 2.0 | Old | New | New | New | ++--------------------------------+-----------------+-----------------+-----------------+------------------------+ +| API | OpenVINO IR v10 | OpenVINO IR v11 | ONNX Files | Models Created in Code | ++================================+=================+=================+=================+========================+ +| Inference Engine / nGraph APIs | Old | Old | Old | Old | ++--------------------------------+-----------------+-----------------+-----------------+------------------------+ +| API 2.0 | Old | New | New | New | ++--------------------------------+-----------------+-----------------+-----------------+------------------------+ -### More Information +More Information +#################### See the following pages to understand how to migrate Inference Engine-based applications to API 2.0: - - [Installation & Deployment](deployment_migration.md) - - [OpenVINO™ Common Inference pipeline](common_inference_pipeline.md) - - [Preprocess your model](./preprocessing.md) - - [Configure device](./configure_devices.md) - - [OpenVINO™ Model Creation](graph_construction.md) -[tf_openvino]: ../../img/tf_openvino.svg +- :doc:`Installation & Deployment ` +- :doc:`OpenVINO™ Common Inference pipeline ` +- :doc:`Preprocess your model ` +- :doc:`Configure device ` +- :doc:`OpenVINO™ Model Creation ` + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/multi_device.md b/docs/OV_Runtime_UG/multi_device.md index 01d03ef100bd3f..e1b6492d59e5a2 100644 --- a/docs/OV_Runtime_UG/multi_device.md +++ b/docs/OV_Runtime_UG/multi_device.md @@ -7,13 +7,13 @@ To run inference on multiple devices, you can choose either of the following way - Use the :ref:`CUMULATIVE_THROUGHPUT option ` of the Automatic Device Selection mode. This way, you can use all available devices in the system without the need to specify them. - Use the Multi-Device execution mode. This page will explain how it works and how to use it. -@endsphinxdirective - -## How MULTI Works +How MULTI Works +#################### The Multi-Device execution mode, or MULTI for short, acts as a "virtual" or a "proxy" device, which does not bind to a specific type of hardware. Instead, it assigns available computing devices to particular inference requests, which are then executed in parallel. The potential gains from using Multi-Device execution are: + * improved throughput from using multiple devices at once, * increase in performance stability due to multiple devices sharing inference workload. @@ -22,31 +22,29 @@ Importantly, the Multi-Device mode does not change the application logic, so it Note that the performance increase in this mode comes from utilizing multiple devices at once. This means that you need to provide the devices with enough inference requests to keep them busy, otherwise you will not benefit much from using MULTI. -## Using the Multi-Device Mode +Using the Multi-Device Mode +########################### Following the OpenVINO™ naming convention, the Multi-Device mode is assigned the label of “MULTI.” The only configuration option available for it is a prioritized list of devices to use: -@sphinxdirective -+---------------------------+---------------------------------+------------------------------------------------------------+ -| Property | Property values | Description | -+===========================+=================================+============================================================+ -| | | MULTI: | | Specifies the devices available for selection. | -| | | comma-separated, no spaces | | The device sequence will be taken as priority | -+---------------------------+---------------------------------+ | from high to low. | -| ov::device::priorities | | device names | | Priorities can be set directly as a string. | -| | | comma-separated, no spaces | | -+---------------------------+---------------------------------+------------------------------------------------------------+ ++----------------------------+---------------------------------+------------------------------------------------------------+ +| Property | Property values | Description | ++============================+=================================+============================================================+ +| | | MULTI: | | Specifies the devices available for selection. | +| | | comma-separated, no spaces | | The device sequence will be taken as priority | ++----------------------------+---------------------------------+ | from high to low. | +| ``ov::device::priorities`` | | device names | | Priorities can be set directly as a string. | +| | | comma-separated, no spaces | | ++----------------------------+---------------------------------+------------------------------------------------------------+ -@endsphinxdirective Specifying the device list explicitly is required by MULTI, as it defines the devices available for inference and sets their priorities. Importantly, the list may also specify the number of requests for MULTI to keep for each device, as described below. -Note that OpenVINO™ Runtime enables you to use “GPU” as an alias for “GPU.0” in function calls. More details on enumerating devices can be found in [Working with devices](supported_plugins/Device_Plugins.md). +Note that OpenVINO™ Runtime enables you to use “GPU” as an alias for “GPU.0” in function calls. More details on enumerating devices can be found in :doc:`Working with devices `. The following commands are accepted by the API: -@sphinxdirective .. tab:: C++ @@ -60,11 +58,9 @@ The following commands are accepted by the API: :language: python :fragment: [MULTI_0] -@endsphinxdirective Notice that MULTI allows you to **change device priorities on the fly**. You can alter the order, exclude a device, and bring an excluded device back. Still, it does not allow adding new devices. -@sphinxdirective .. tab:: C++ @@ -78,19 +74,17 @@ Notice that MULTI allows you to **change device priorities on the fly**. You can :language: python :fragment: [MULTI_1] -@endsphinxdirective - +One more thing you can define is the **number of requests to allocate for each device**. You can do it simply by adding the number to each device in parentheses, like this: ``"MULTI:CPU(2),GPU(2)"``. However, this method is not recommended as it is not performance-portable. The suggested approach is to configure individual devices and query the resulting number of requests to be used at the application level, as described in `Configuring Individual Devices and Creating MULTI On Top <#configuring-individual-devices-and-creating-the-multi-device-on-top>`__. -One more thing you can define is the **number of requests to allocate for each device**. You can do it simply by adding the number to each device in parentheses, like this: `"MULTI:CPU(2),GPU(2)"`. However, this method is not recommended as it is not performance-portable. The suggested approach is to configure individual devices and query the resulting number of requests to be used at the application level, as described in [Configuring Individual Devices and Creating MULTI On Top](#config-multi-on-top). +To check what devices are present in the system, you can use the Device API. For information on how to do it, check :doc:`Query device properties and configuration `. -To check what devices are present in the system, you can use the Device API. For information on how to do it, check [Query device properties and configuration](supported_plugins/config_properties.md). +Configuring Individual Devices and Creating the Multi-Device On Top ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -### Configuring Individual Devices and Creating the Multi-Device On Top As mentioned previously, executing inference with MULTI may be set up by configuring individual devices before creating the "MULTI" device on top. It may be considered for performance reasons. -@sphinxdirective .. tab:: C++ @@ -104,17 +98,15 @@ As mentioned previously, executing inference with MULTI may be set up by configu :language: python :fragment: [MULTI_4] -@endsphinxdirective Alternatively, you can combine all the individual device settings into a single config file and load it for MULTI to parse. See the code example in the next section. +Querying the Optimal Number of Inference Requests ++++++++++++++++++++++++++++++++++++++++++++++++++ - -### Querying the Optimal Number of Inference Requests When using MULTI, you don't need to sum over included devices yourself, you can query the optimal number of requests directly, -using the [configure devices](supported_plugins/config_properties.md) property: +using the :doc:`configure devices ` property: -@sphinxdirective .. tab:: C++ @@ -122,56 +114,52 @@ using the [configure devices](supported_plugins/config_properties.md) property: :language: cpp :fragment: [part5] -@endsphinxdirective - - -## Using the Multi-Device with OpenVINO Samples and Benchmarking Performance +Using the Multi-Device with OpenVINO Samples and Benchmarking Performance +######################################################################### To see how the Multi-Device execution is used in practice and test its performance, take a look at OpenVINO's Benchmark Application which presents the optimal performance of the plugin without the need for additional settings, like the number of requests or CPU threads. Here is an example command to evaluate performance of CPU + GPU: -```sh -./benchmark_app –d MULTI:CPU,GPU –m -i -niter 1000 -``` +.. code-block:: sh + + ./benchmark_app –d MULTI:CPU,GPU –m -i -niter 1000 + + +For more information, refer to the :doc:`C++ ` or :doc:`Python ` version instructions. -For more information, refer to the [C++](../../samples/cpp/benchmark_app/README.md) or [Python](../../tools/benchmark_tool/README.md) version instructions. -@sphinxdirective .. note:: You can keep using the FP16 IR without converting it to FP32, even if some of the listed devices do not support it. The conversion will be done automatically for you. - No demos are yet fully optimized for MULTI, by means of supporting the ov::optimal_number_of_infer_requests property, using the GPU streams/throttling, and so on. -@endsphinxdirective + No demos are yet fully optimized for MULTI, by means of supporting the ``ov::optimal_number_of_infer_requests`` property, using the GPU streams/throttling, and so on. + +Performance Considerations for the Multi-Device Execution +######################################################### -## Performance Considerations for the Multi-Device Execution For best performance when using the MULTI execution mode you should consider a few recommendations: -- MULTI usually performs best when the fastest device is specified first in the device candidate list. -This is particularly important when the request-level parallelism is not sufficient -(e.g. the number of requests is not enough to saturate all devices). -- Just like with any throughput-oriented execution mode, it is highly recommended to query the optimal number of inference requests -directly from the instance of the `ov:compiled_model`. Refer to the code of the previously mentioned `benchmark_app` for more details. -- Execution on certain device combinations, for example CPU+GPU, performs better with certain knobs. Refer to the `benchmark_app` code for details. One specific example is disabling GPU driver polling, which in turn requires multiple GPU streams to balance out slower -communication of inference completion from the device to the host. -- The MULTI logic always attempts to save on copying data between device-agnostic and user-facing inference requests, -and device-specific 'worker' requests that are being actually scheduled behind the scene. -To facilitate the copy savings, it is recommended to run the requests in the order in which they were created. + +- MULTI usually performs best when the fastest device is specified first in the device candidate list. This is particularly important when the request-level parallelism is not sufficient (e.g. the number of requests is not enough to saturate all devices). +- Just like with any throughput-oriented execution mode, it is highly recommended to query the optimal number of inference requests directly from the instance of the ``ov:compiled_model``. Refer to the code of the previously mentioned ``benchmark_app`` for more details. +- Execution on certain device combinations, for example CPU+GPU, performs better with certain knobs. Refer to the ``benchmark_app`` code for details. One specific example is disabling GPU driver polling, which in turn requires multiple GPU streams to balance out slower communication of inference completion from the device to the host. +- The MULTI logic always attempts to save on copying data between device-agnostic and user-facing inference requests, and device-specific 'worker' requests that are being actually scheduled behind the scene. To facilitate the copy savings, it is recommended to run the requests in the order in which they were created. - While performance of accelerators combines well with MULTI, the CPU+GPU execution may introduce certain performance issues. It is due to the devices sharing some resources, like power or bandwidth. Enabling the GPU throttling hint, which saves a CPU thread for CPU inference, is an example of a recommended solution addressing this issue. +Additional Resources +#################### -## Additional Resources +- :doc:`Supported Devices ` +- :doc:`Automatic Device Selection ` -- [Supported Devices](supported_plugins/Supported_Devices.md) -- [Automatic Device Selection](./auto_device_selection.md) -@sphinxdirective .. raw:: html -@endsphinxdirective -> **NOTE**: This video is currently available only for C++, but many of the same concepts apply to Python. +.. note:: This video is currently available only for C++, but many of the same concepts apply to Python. + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/performance_hints.md b/docs/OV_Runtime_UG/performance_hints.md index c66bb552b473ac..fcf1d930b861fb 100644 --- a/docs/OV_Runtime_UG/performance_hints.md +++ b/docs/OV_Runtime_UG/performance_hints.md @@ -17,7 +17,6 @@ As discussed in the :doc:`Optimization Guide `__ and `ov::hint::PerformanceMode::LATENCY `__. -A special `ov::hint::PerformanceMode::UNDEFINED `__ hint acts the same as specifying no hint. For more information on conducting performance measurements with the ``benchmark_app``, refer to the last section in this document. diff --git a/docs/OV_Runtime_UG/supported_plugins/CPU.md b/docs/OV_Runtime_UG/supported_plugins/CPU.md index df17fd10f0d57c..14b569e3e4249c 100644 --- a/docs/OV_Runtime_UG/supported_plugins/CPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/CPU.md @@ -105,14 +105,14 @@ to query ``ov::device::capabilities`` property, which should contain ``BF16`` in :fragment: [part0] -If the model has been converted to ``bf16``, the ``ov::inference_precision`` is set to ``ov::element::bf16`` and can be checked via +If the model has been converted to ``bf16``, the ``ov::hint::inference_precision`` is set to ``ov::element::bf16`` and can be checked via the ``ov::CompiledModel::get_property`` call. The code below demonstrates how to get the element type: .. doxygensnippet:: snippets/cpu/Bfloat16Inference1.cpp :language: py :fragment: [part1] -To infer the model in ``f32`` precision instead of ``bf16`` on targets with native ``bf16`` support, set the ``ov::inference_precision`` to ``ov::element::f32``. +To infer the model in ``f32`` precision instead of ``bf16`` on targets with native ``bf16`` support, set the ``ov::hint::inference_precision`` to ``ov::element::f32``. .. tab-set:: @@ -134,11 +134,11 @@ To infer the model in ``f32`` precision instead of ``bf16`` on targets with nati The ``Bfloat16`` software simulation mode is available on CPUs with Intel® AVX-512 instruction set that do not support the native ``avx512_bf16`` instruction. This mode is used for development purposes and it does not guarantee good performance. -To enable the simulation, the ``ov::inference_precision`` has to be explicitly set to ``ov::element::bf16``. +To enable the simulation, the ``ov::hint::inference_precision`` has to be explicitly set to ``ov::element::bf16``. .. note:: - If ``ov::inference_precision`` is set to ``ov::element::bf16`` on a CPU without native bfloat16 support or bfloat16 simulation mode, an exception is thrown. + If ``ov::hint::inference_precision`` is set to ``ov::element::bf16`` on a CPU without native bfloat16 support or bfloat16 simulation mode, an exception is thrown. .. note:: @@ -292,7 +292,7 @@ Read-write Properties All parameters must be set before calling ``ov::Core::compile_model()`` in order to take effect or passed as additional argument to ``ov::Core::compile_model()`` - ``ov::enable_profiling`` -- ``ov::inference_precision`` +- ``ov::hint::inference_precision`` - ``ov::hint::performance_mode`` - ``ov::hint::num_request`` - ``ov::num_streams`` diff --git a/docs/OV_Runtime_UG/supported_plugins/GNA.md b/docs/OV_Runtime_UG/supported_plugins/GNA.md index 7faace9e172d45..1b07d5de1bf815 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GNA.md +++ b/docs/OV_Runtime_UG/supported_plugins/GNA.md @@ -140,7 +140,7 @@ quantization hints based on statistics for the provided dataset. * Accuracy (i16 weights) * Performance (i8 weights) -For POT quantized models, the ``ov::inference_precision`` property has no effect except in cases described in the +For POT quantized models, the ``ov::hint::inference_precision`` property has no effect except in cases described in the :ref:`Model and Operation Limitations section <#model-and-operation-limitations>`. @@ -268,7 +268,7 @@ In order to take effect, the following parameters must be set before model compi - ov::cache_dir - ov::enable_profiling -- ov::inference_precision +- ov::hint::inference_precision - ov::hint::num_requests - ov::intel_gna::compile_target - ov::intel_gna::firmware_model_image_path @@ -354,7 +354,7 @@ Support for 2D Convolutions using POT For POT to successfully work with the models including GNA3.0 2D convolutions, the following requirements must be met: * All convolution parameters are natively supported by HW (see tables above). -* The runtime precision is explicitly set by the ``ov::inference_precision`` property as ``i8`` for the models produced by +* The runtime precision is explicitly set by the ``ov::hint::inference_precision`` property as ``i8`` for the models produced by the ``performance mode`` of POT, and as ``i16`` for the models produced by the ``accuracy mode`` of POT. diff --git a/docs/OV_Runtime_UG/supported_plugins/GPU.md b/docs/OV_Runtime_UG/supported_plugins/GPU.md index d2e5bce1d37052..3be45dc12eba85 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/GPU.md @@ -327,7 +327,7 @@ All parameters must be set before calling ``ov::Core::compile_model()`` in order - ov::hint::performance_mode - ov::hint::execution_mode - ov::hint::num_requests -- ov::inference_precision +- ov::hint::inference_precision - ov::num_streams - ov::compilation_num_threads - ov::device::id diff --git a/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md b/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md index dc73deb70965d0..0eac844e4c7701 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md +++ b/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md @@ -45,46 +45,85 @@ To create the ``ov::RemoteContext`` object for user context, explicitly provide of ``ov::RemoteContext`` derived classes. -.. tab:: Linux +.. tab:: Linux/C++ .. tab:: Create from cl_context .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp :language: cpp - :fragment: context_from_cl_context + :fragment: [context_from_cl_context] .. tab:: Create from cl_queue .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp :language: cpp - :fragment: context_from_cl_queue + :fragment: [context_from_cl_queue] .. tab:: Create from VADisplay .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp :language: cpp - :fragment: context_from_va_display + :fragment: [context_from_va_display] -.. tab:: Windows +.. tab:: Windows/C++ .. tab:: Create from cl_context .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp :language: cpp - :fragment: context_from_cl_context + :fragment: [context_from_cl_context] .. tab:: Create from cl_queue .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp :language: cpp - :fragment: context_from_cl_queue + :fragment: [context_from_cl_queue] .. tab:: Create from ID3D11Device .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp :language: cpp - :fragment: context_from_d3d_device + :fragment: [context_from_d3d_device] +.. tab:: Linux/C + + .. tab:: Create from cl_context + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [context_from_cl_context] + + .. tab:: Create from cl_queue + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [context_from_cl_queue] + + .. tab:: Create from VADisplay + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [context_from_va_display] + +.. tab:: Windows/C + + .. tab:: Create from cl_context + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [context_from_cl_context] + + .. tab:: Create from cl_queue + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [context_from_cl_queue] + + .. tab:: Create from ID3D11Device + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [context_from_d3d_device] Getting RemoteContext from the Plugin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -96,19 +135,33 @@ Once the plugin options have been changed, the internal context is replaced by t To request the current default context of the plugin, use one of the following methods: +.. tab:: C++ -.. tab:: Get context from Core + .. tab:: Get context from Core - .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: cpp - :fragment: default_context_from_core + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp + :language: cpp + :fragment: [default_context_from_core] -.. tab:: Get context from compiled model + .. tab:: Get context from compiled model - .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: cpp - :fragment: default_context_from_model + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp + :language: cpp + :fragment: [default_context_from_model] + +.. tab:: C + + .. tab:: Get context from Core + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [default_context_from_core] + .. tab:: Get context from compiled model + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [default_context_from_model] Memory Sharing Between Application and GPU Plugin ########################################################### @@ -116,70 +169,140 @@ Memory Sharing Between Application and GPU Plugin The classes that implement the ``ov::RemoteTensor`` interface are the wrappers for native API memory handles (which can be obtained from them at any time). -To create a shared tensor from a native memory handle, use dedicated ``create_tensor``or ``create_tensor_nv12`` methods +To create a shared tensor from a native memory handle, use dedicated ``create_tensor`` or ``create_tensor_nv12`` methods of the ``ov::RemoteContext`` sub-classes. ``ov::intel_gpu::ocl::ClContext`` has multiple overloads of ``create_tensor`` methods which allow to wrap pre-allocated native handles with the ``ov::RemoteTensor`` -object or request plugin to allocate specific device memory. For more details, see the code snippets below: +object or request plugin to allocate specific device memory. There also provides C APIs to do the same things with C++ APIs. +For more details, see the code snippets below: .. tab-set:: - .. tab-item:: Wrap native handles + .. tab-item:: Wrap native handles/C++ + :sync: wrap-native-handles .. tab-set:: .. tab-item:: USM pointer + :sync: usm-pointer .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: wrap_usm_pointer + :language: cpp + :fragment: [wrap_usm_pointer] .. tab-item:: cl_mem + :sync: cl_mem .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: wrap_cl_mem + :language: cpp + :fragment: [wrap_cl_mem] .. tab-item:: cl::Buffer + :sync: buffer .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: wrap_cl_buffer + :language: cpp + :fragment: [wrap_cl_buffer] .. tab-item:: cl::Image2D + :sync: image2D .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: wrap_cl_image + :language: cpp + :fragment: [wrap_cl_image] .. tab-item:: biplanar NV12 surface + :sync: biplanar .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: wrap_nv12_surface + :language: cpp + :fragment: [wrap_nv12_surface] - .. tab-item:: Allocate device memory + .. tab-item:: Allocate device memory/C++ + :sync: allocate-device-memory .. tab-set:: - + .. tab-item:: USM host memory + :sync: usm-host-memory .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: allocate_usm_host + :language: cpp + :fragment: [allocate_usm_host] .. tab-item:: USM device memory + :sync: usm-device-memory .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: allocate_usm_device + :language: cpp + :fragment: [allocate_usm_device] .. tab-item:: cl::Buffer .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp - :language: sh - :fragment: allocate_cl_buffer + :language: cpp + :fragment: [allocate_cl_buffer] + +.. tab-set:: + + .. tab-item:: Wrap native handles/C + :sync: wrap-native-handles + + .. tab-set:: + + .. tab-item:: USM pointer + :sync: usm-pointer + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [wrap_usm_pointer] + + .. tab-item:: cl_mem + :sync: cl_mem + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [wrap_cl_mem] + + .. tab-item:: cl::Buffer + :sync: buffer + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [wrap_cl_buffer] + + .. tab-item:: cl::Image2D + :sync: image2D + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [wrap_cl_image] + + .. tab-item:: biplanar NV12 surface + :sync: biplanar + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [create_nv12_surface] + + .. tab-item:: Allocate device memory/C + :sync: allocate-device-memory + + .. tab-set:: + + .. tab-item:: USM host memory + :sync: usm-host-memory + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [allocate_usm_host] + + .. tab-item:: USM device memory + :sync: usm-device-memory + + .. doxygensnippet:: docs/snippets/gpu/remote_objects_creation_c.cpp + :language: c + :fragment: [allocate_usm_device] The ``ov::intel_gpu::ocl::D3DContext`` and ``ov::intel_gpu::ocl::VAContext`` classes are derived from ``ov::intel_gpu::ocl::ClContext``. Therefore, they provide the functionality described above and extend it @@ -202,9 +325,17 @@ should be added before model compilation: .. tab:: two-plane - .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp - :language: cpp - :fragment: [init_preproc] + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp + :language: cpp + :fragment: [init_preproc] + + .. tab:: C + + .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes_c.cpp + :language: c + :fragment: [init_preproc] .. tab:: single-plane @@ -228,21 +359,29 @@ inputs need to be set via the ``ov::InferRequest::set_tensors`` method with vect .. tab:: two-plane - .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp - :language: cpp - :fragment: single_batch + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp + :language: cpp + :fragment: [single_batch] + + .. tab:: C + + .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes_c.cpp + :language: c + :fragment: [single_batch] .. tab:: single-plane .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp :language: cpp - :fragment: single_batch + :fragment: [single_batch] .. tab:: NV12 to Grey .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp :language: cpp - :fragment: single_batch + :fragment: [single_batch] .. tab:: Multiple Batches @@ -250,19 +389,19 @@ inputs need to be set via the ``ov::InferRequest::set_tensors`` method with vect .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp :language: cpp - :fragment: batched_case + :fragment: [batched_case] .. tab:: single-plane .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp :language: cpp - :fragment: batched_case + :fragment: [batched_case] .. tab:: NV12 to Grey .. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp :language: cpp - :fragment: batched_case + :fragment: [batched_case] I420 color format can be processed in a similar way @@ -283,7 +422,7 @@ on waiting for the completion of inference. The pseudo-code may look as follows: .. doxygensnippet:: docs/snippets/gpu/queue_sharing.cpp :language: cpp - :fragment: queue_sharing + :fragment: [queue_sharing] Limitations @@ -326,20 +465,27 @@ To see pseudo-code of usage examples, refer to the sections below. .. doxygensnippet:: docs/snippets/gpu/context_sharing.cpp :language: cpp - :fragment: context_sharing_get_from_ov + :fragment: [context_sharing_get_from_ov] .. dropdown:: Running GPU Plugin Inference within User-Supplied Shared Context .. doxygensnippet:: docs/snippets/gpu/context_sharing.cpp :language: cpp - :fragment: context_sharing_user_handle + :fragment: [context_sharing_user_handle] .. dropdown:: Direct Consuming of the NV12 VAAPI Video Decoder Surface on Linux - .. doxygensnippet:: docs/snippets/gpu/context_sharing_va.cpp - :language: cpp - :fragment: context_sharing_va + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/gpu/context_sharing_va.cpp + :language: cpp + :fragment: [context_sharing_va] + + .. tab:: C + .. doxygensnippet:: docs/snippets/gpu/context_sharing_va_c.cpp + :language: c + :fragment: [context_sharing_va] See Also ####################################### diff --git a/docs/_static/images/BASIC_FLOW_IE_C.svg b/docs/_static/images/BASIC_FLOW_IE_C.svg index 25310487735e23..6b8ad0ef282518 100644 --- a/docs/_static/images/BASIC_FLOW_IE_C.svg +++ b/docs/_static/images/BASIC_FLOW_IE_C.svg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a50bd923023c4cec3781476f9c8ced19150b5fd2c2c143166604b51655e13768 -size 50623 +oid sha256:ccc7704d2a27f7491729767443f3d2bdd0ccc930f16fde631a7f9c67d158297a +size 71369 diff --git a/docs/MO_DG/img/BASIC_FLOW_MO_simplified.svg b/docs/_static/images/BASIC_FLOW_MO_simplified.svg similarity index 100% rename from docs/MO_DG/img/BASIC_FLOW_MO_simplified.svg rename to docs/_static/images/BASIC_FLOW_MO_simplified.svg diff --git a/docs/img/DEVELOPMENT_FLOW_V3_crunch.svg b/docs/_static/images/DEVELOPMENT_FLOW_V3_crunch.svg similarity index 100% rename from docs/img/DEVELOPMENT_FLOW_V3_crunch.svg rename to docs/_static/images/DEVELOPMENT_FLOW_V3_crunch.svg diff --git a/docs/_static/images/DeepSpeech-0.8.2.png b/docs/_static/images/DeepSpeech-0.8.2.png new file mode 100644 index 00000000000000..ddab04ac34ac29 --- /dev/null +++ b/docs/_static/images/DeepSpeech-0.8.2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdff3768930f683b81ca466be4f947af3172933a702cd38201a254df27a68556 +size 62498 diff --git a/docs/_static/images/FaceNet.svg b/docs/_static/images/FaceNet.svg new file mode 100644 index 00000000000000..89927473829898 --- /dev/null +++ b/docs/_static/images/FaceNet.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2720b6d3b5e680978a91379c8c37366285299aab31aa139ad9abea8334aae34 +size 57687 diff --git a/docs/_static/images/NCF_start.svg b/docs/_static/images/NCF_start.svg new file mode 100644 index 00000000000000..345d9561721cff --- /dev/null +++ b/docs/_static/images/NCF_start.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0389fe34562993b1285f1994dbc878e9547a841c903bf204074ed2219b6bc7 +size 323210 diff --git a/docs/img/WHAT_TO_USE.svg b/docs/_static/images/WHAT_TO_USE.svg similarity index 100% rename from docs/img/WHAT_TO_USE.svg rename to docs/_static/images/WHAT_TO_USE.svg diff --git a/docs/img/batch_device.svg b/docs/_static/images/batch_device.svg similarity index 100% rename from docs/img/batch_device.svg rename to docs/_static/images/batch_device.svg diff --git a/docs/OV_Runtime_UG/img/configuration_dialog.png b/docs/_static/images/configuration_dialog.png similarity index 100% rename from docs/OV_Runtime_UG/img/configuration_dialog.png rename to docs/_static/images/configuration_dialog.png diff --git a/docs/img/cpu_execution_conventional_approach.svg b/docs/_static/images/cpu_execution_conventional_approach.svg similarity index 100% rename from docs/img/cpu_execution_conventional_approach.svg rename to docs/_static/images/cpu_execution_conventional_approach.svg diff --git a/docs/img/cpu_execution_streams.svg b/docs/_static/images/cpu_execution_streams.svg similarity index 100% rename from docs/img/cpu_execution_streams.svg rename to docs/_static/images/cpu_execution_streams.svg diff --git a/docs/img/cpu_execution_streams_2.svg b/docs/_static/images/cpu_execution_streams_2.svg similarity index 100% rename from docs/img/cpu_execution_streams_2.svg rename to docs/_static/images/cpu_execution_streams_2.svg diff --git a/docs/img/deployment_full.svg b/docs/_static/images/deployment_full.svg similarity index 100% rename from docs/img/deployment_full.svg rename to docs/_static/images/deployment_full.svg diff --git a/docs/img/deployment_simplified.svg b/docs/_static/images/deployment_simplified.svg similarity index 100% rename from docs/img/deployment_simplified.svg rename to docs/_static/images/deployment_simplified.svg diff --git a/docs/_static/images/dog.png b/docs/_static/images/dog.png new file mode 100644 index 00000000000000..77da0c3beaaeff --- /dev/null +++ b/docs/_static/images/dog.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d49a45dd5cbace188cfdc23a2aeff441c2187f1610c9720b91ea6fbbb232387 +size 150582 diff --git a/docs/MO_DG/img/inception_v1_first_block.svg b/docs/_static/images/inception_v1_first_block.svg similarity index 100% rename from docs/MO_DG/img/inception_v1_first_block.svg rename to docs/_static/images/inception_v1_first_block.svg diff --git a/docs/MO_DG/img/inception_v1_std_input.svg b/docs/_static/images/inception_v1_std_input.svg similarity index 100% rename from docs/MO_DG/img/inception_v1_std_input.svg rename to docs/_static/images/inception_v1_std_input.svg diff --git a/docs/MO_DG/img/inception_v1_std_output.svg b/docs/_static/images/inception_v1_std_output.svg similarity index 100% rename from docs/MO_DG/img/inception_v1_std_output.svg rename to docs/_static/images/inception_v1_std_output.svg diff --git a/docs/img/large_batch_approach.svg b/docs/_static/images/large_batch_approach.svg similarity index 100% rename from docs/img/large_batch_approach.svg rename to docs/_static/images/large_batch_approach.svg diff --git a/docs/_static/images/lm_1b.svg b/docs/_static/images/lm_1b.svg new file mode 100644 index 00000000000000..df8075c041624f --- /dev/null +++ b/docs/_static/images/lm_1b.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:062fa64afa0cc43c4a2c2c0442e499b6176c837857222af30bad2fa7c9515420 +size 95508 diff --git a/docs/img/nncf_workflow.svg b/docs/_static/images/nncf_workflow.svg similarity index 100% rename from docs/img/nncf_workflow.svg rename to docs/_static/images/nncf_workflow.svg diff --git a/docs/img/quantization_picture.svg b/docs/_static/images/quantization_picture.svg similarity index 100% rename from docs/img/quantization_picture.svg rename to docs/_static/images/quantization_picture.svg diff --git a/tools/pot/docs/range_supervision/images/img_combined_2.png b/docs/_static/images/range_supervision/img_combined_2.png similarity index 100% rename from tools/pot/docs/range_supervision/images/img_combined_2.png rename to docs/_static/images/range_supervision/img_combined_2.png diff --git a/tools/pot/docs/range_supervision/images/scheme3.svg b/docs/_static/images/range_supervision/scheme3.svg similarity index 100% rename from tools/pot/docs/range_supervision/images/scheme3.svg rename to docs/_static/images/range_supervision/scheme3.svg diff --git a/docs/OV_Runtime_UG/img/selection_dialog.png b/docs/_static/images/selection_dialog.png similarity index 100% rename from docs/OV_Runtime_UG/img/selection_dialog.png rename to docs/_static/images/selection_dialog.png diff --git a/docs/img/tf_openvino.svg b/docs/_static/images/tf_openvino.svg similarity index 100% rename from docs/img/tf_openvino.svg rename to docs/_static/images/tf_openvino.svg diff --git a/tools/pot/docs/images/workflow_simple.svg b/docs/_static/images/workflow_simple.svg similarity index 100% rename from tools/pot/docs/images/workflow_simple.svg rename to docs/_static/images/workflow_simple.svg diff --git a/docs/_static/js/custom.js b/docs/_static/js/custom.js index 0e67c39943533a..f9f8c9a9000d4f 100644 --- a/docs/_static/js/custom.js +++ b/docs/_static/js/custom.js @@ -40,10 +40,6 @@ $(document).ready(function () { init_switchers(); handleSwitcherParam(); initViewerJS(); - var TABLE_SORT = window.TABLE_SORT; - if (TABLE_SORT) { - addTableSort(); - } addLegalNotice(); updateSearchForm(); initBenchmarkPickers(); // included with the new benchmarks page @@ -134,76 +130,6 @@ function createVersions() { } - -function addTableSort() { - var tables = $('table.table'); - tables.each(function () { - var table = $(this); - var headings = table.find('th'); - headings.each(function () { - var th = $(this); - var index = th.index(); - var sortBtn = $(''); - th.addClass('sort-header'); - th.click(function () { - var counter = 0; - sortBtn.addClass('sort-active'); - sortBy = sortBtn.data('sortby'); - var trs = table.find('tbody tr'); - sortBtn.toggleClass('ascending'); - trs.sort(function (item1, item2) { - - if (sortBtn.hasClass('ascending')) { - var text1 = $(item1).find('td').eq(index).text(); - var text2 = $(item2).find('td').eq(index).text(); - } - else { - var text1 = $(item2).find('td').eq(index).text(); - var text2 = $(item1).find('td').eq(index).text(); - } - // try converting to num - var _text1 = parseFloat(text1); - var _text2 = parseFloat(text2); - - if (!isNaN(_text1) && !isNaN(_text2)) { - text1 = _text1; - text2 = _text2; - } - if (text1 > text2) { - return 1; - } - else if (text1 < text2) { - return -1; - } - else { - return 0; - } - }).map(function () { - var row = $(this); - if (counter % 2 === 0) { - row.removeClass('row-odd'); - row.addClass('row-even'); - } - else { - row.removeClass('row-even'); - row.addClass('row-odd'); - } - counter++; - table.find('tbody').append(row); - }); - - headings.each(function () { - if ($(this).index() !== index) { - $(this).find('.sort-btn').removeClass('ascending'); - $(this).find('.sort-btn').removeClass('sort-active'); - } - }); - }); - th.find('p').append(sortBtn); - }); - }); -} - function initViewerJS() { try { var images = $('main img[src*="_images"]'); diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6c09dd1d15908..ed3c4c79fcfa4d 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -18,10 +18,11 @@ {% block docs_navbar %} {{ super() }}
-

OpenVINO 2022.1 introduces a new version of OpenVINO API (API 2.0). For more information on the changes and transition steps, see the transition guide

+

OpenVINO 2022.1 has introduced OpenVINO API 2.0. For more information on transition steps from the previous API, see the transition guide

{% endblock %} + diff --git a/docs/api/ie_python_api/api.rst b/docs/api/ie_python_api/api.rst index c06cc2b2fb8c80..5faa85f4e05b8e 100644 --- a/docs/api/ie_python_api/api.rst +++ b/docs/api/ie_python_api/api.rst @@ -73,6 +73,12 @@ OpenVINO Python API openvino.runtime.opset10 +.. autosummary:: + :toctree: _autosummary + :template: custom-module-template.rst + + openvino.runtime.opset11 + .. autosummary:: :toctree: _autosummary :template: custom-module-template.rst @@ -95,4 +101,4 @@ OpenVINO Python API :maxdepth: 2 :hidden: - compatibility \ No newline at end of file + compatibility diff --git a/docs/dev/cmake_options_for_custom_comiplation.md b/docs/dev/cmake_options_for_custom_comiplation.md index 50d3e933c4bb9c..15c77a8bc17c37 100644 --- a/docs/dev/cmake_options_for_custom_comiplation.md +++ b/docs/dev/cmake_options_for_custom_comiplation.md @@ -96,6 +96,8 @@ This document provides description and default values for CMake options that can > * Hybrid architecture to separate Performance / Efficiency cores and schedule tasks in the optimal way. > **Note:** if you build OpenVINO runtime with [oneTBB] support where TBBBind 2.5 is automatically loaded by TBB in runtime, then set `ENABLE_TBBBIND_2_5` to `OFF` +> * make sure libtbbbind.so is in the same folder as libtbb.so. For example, oneTBB package on Ubuntu 22.04 has libtbbbind missed. https://bugs.launchpad.net/ubuntu/+source/onetbb/+bug/2006898 +> * oneTBB relies on higher version hwloc to recognize hybrid CPU core types correctly, on some machines, they require higher hwloc version to work correctly. Check if hwloc-info --version returns hwloc version >= 2.7.0, Ubuntu 20.04 with hwloc 2.1.0 * `ENABLE_SSE42` enables SSE4.2 optimizations: * `ON` is default for x86 platforms; not available for other platforms. @@ -132,6 +134,7 @@ cmake ... In this case OpenVINO CMake scripts take `TBBROOT` environment variable into account and provided TBB will be used. **Note:** if you are building TBB from source files, please install TBB after and use `TBBROOT` to point to installation root. +**Note:** reference to oneTBB Note in [Options affecting binary size](#options-affecting-binary-size) ## Test capabilities @@ -158,6 +161,9 @@ In this case OpenVINO CMake scripts take `TBBROOT` environment variable into acc * `ENABLE_INTEGRITYCHECK` builds DLLs with [/INTEGRITYCHECK] flag: * `OFF` is default. * Available on MSVC compiler only. +* `ENABLE_QSPECTRE` builds with [/Qspectre] flag: + * `OFF` is default. + * Available on MSVC compiler only. ## Additional Resources @@ -172,6 +178,7 @@ In this case OpenVINO CMake scripts take `TBBROOT` environment variable into acc [precompiled headers]:https://cmake.org/cmake/help/git-stage/command/target_precompile_headers.html [unity build]:https://cmake.org/cmake/help/latest/prop_tgt/UNITY_BUILD.html [/INTEGRITYCHECK]:https://docs.microsoft.com/en-us/cpp/build/reference/integritycheck-require-signature-check?view=msvc-160 +[/Qspectre]:https://learn.microsoft.com/en-us/cpp/build/reference/qspectre?view=msvc-170 [Intel TBB]:https://software.intel.com/content/www/us/en/develop/tools/threading-building-blocks.html [Python]:https://www.python.org/ [Java]:https://www.java.com/ru/ @@ -189,4 +196,4 @@ In this case OpenVINO CMake scripts take `TBBROOT` environment variable into acc [TensorFlow Lite]:https://www.tensorflow.org/lite [PyTorch]:https://www.tensorflow.org/lite [FlatBuffers]:https://google.github.io/flatbuffers/ -[oneTBB]:https://github.com/oneapi-src/oneTBB \ No newline at end of file +[oneTBB]:https://github.com/oneapi-src/oneTBB diff --git a/docs/dev/installing.md b/docs/dev/installing.md index 34297285e5adcb..c180e92dc2cf3f 100644 --- a/docs/dev/installing.md +++ b/docs/dev/installing.md @@ -148,12 +148,12 @@ omz_downloader --name googlenet-v1 --output_dir %USERPROFILE%\Documents\models Linux and macOS: ```sh mkdir ~/ir -mo --input_model ~/models/public/googlenet-v1/googlenet-v1.caffemodel --data_type FP16 --output_dir ~/ir +mo --input_model ~/models/public/googlenet-v1/googlenet-v1.caffemodel --compress_to_fp16 --output_dir ~/ir ``` Windows: ```bat mkdir %USERPROFILE%\Documents\ir -mo --input_model %USERPROFILE%\Documents\models\public\googlenet-v1\googlenet-v1.caffemodel --data_type FP16 --output_dir %USERPROFILE%\Documents\ir +mo --input_model %USERPROFILE%\Documents\models\public\googlenet-v1\googlenet-v1.caffemodel --compress_to_fp16 --output_dir %USERPROFILE%\Documents\ir ``` 5. Run Inference on the Sample @@ -175,16 +175,18 @@ The following commands run the Image Classification Code Sample using the [`dog. Linux and macOS: ```sh -cd ~/inference_engine_cpp_samples_build/intel64/Release +cd ~/openvino_cpp_samples_build//Release ./classification_sample_async -i ~/Downloads/dog.bmp -m ~/ir/googlenet-v1.xml -d CPU ``` +where the is the output of ``uname -m``, for example, ``intel64``, ``armhf``, or ``aarch64``. Windows: ```bat -cd %USERPROFILE%\Documents\Intel\OpenVINO\inference_engine_samples_build\intel64\Release +cd %USERPROFILE%\Documents\Intel\OpenVINO\openvino_cpp_samples_build\\Release .\classification_sample_async.exe -i %USERPROFILE%\Downloads\dog.bmp -m %USERPROFILE%\Documents\ir\googlenet-v1.xml -d CPU ``` +where the is either ``intel64`` or ``aarch64`` depending on the platform architecture. When the sample application is complete, you see the label and confidence data for the top 10 categories on the display: diff --git a/docs/doxygen-xfail.txt b/docs/doxygen-xfail.txt index f3f1abd334e2f3..bfb5159f63158e 100644 --- a/docs/doxygen-xfail.txt +++ b/docs/doxygen-xfail.txt @@ -27,6 +27,7 @@ openvino_docs_OV_UG_lpt_gathertransformation.rst openvino_docs_OV_UG_lpt_linopsequencefusion.rst openvino_docs_OV_UG_lpt_mvntransformation.rst openvino_docs_OV_UG_lpt_markupavgpoolprecisionpreserved.rst +openvino_docs_OV_UG_lpt_markupbias.rst openvino_docs_OV_UG_lpt_markupcanbequantized.rst openvino_docs_OV_UG_lpt_markuppertensorquantization.rst openvino_docs_OV_UG_lpt_markupprecisions.rst diff --git a/docs/get_started/get_started_demos.md b/docs/get_started/get_started_demos.md index a3ac90983db68a..aef7c866dd2370 100644 --- a/docs/get_started/get_started_demos.md +++ b/docs/get_started/get_started_demos.md @@ -1,42 +1,54 @@ # Get Started with C++ Samples {#openvino_docs_get_started_get_started_demos} +@sphinxdirective + The guide presents a basic workflow for building and running C++ code samples in OpenVINO. Note that these steps will not work with the Python samples. -To get started, you must first install OpenVINO Runtime, install OpenVINO Development tools, and build the sample applications. See the Prerequisites section for instructions. +To get started, you must first install OpenVINO Runtime, install OpenVINO Development tools, and build the sample applications. See the :ref:`Prerequisites ` section for instructions. Once the prerequisites have been installed, perform the following steps: -1. Use Model Downloader to download a suitable model. -2. Convert the model with Model Optimizer. -3. Download media files to run inference. -4. Run inference with the Image Classification sample application and see the results. +1. :ref:`Use Model Downloader to download a suitable model `. +2. :ref:`Convert the model with Model Optimizer `. +3. :ref:`Download media files to run inference `. +4. :ref:`Run inference with the Image Classification sample application and see the results `. -## Prerequisites +.. _prerequisites-samples: -### Install OpenVINO Runtime +Prerequisites +############# + +Install OpenVINO Runtime +++++++++++++++++++++++++ To use sample applications, install OpenVINO Runtime via one of the following distribution channels (other distributions do not include sample files): -* Archive files (recommended) - [Linux](@ref openvino_docs_install_guides_installing_openvino_from_archive_linux) | [Windows](@ref openvino_docs_install_guides_installing_openvino_from_archive_windows) | [macOS](@ref openvino_docs_install_guides_installing_openvino_from_archive_macos) -* [APT](@ref openvino_docs_install_guides_installing_openvino_apt) or [YUM](@ref openvino_docs_install_guides_installing_openvino_yum) for Linux -* Docker image - [Linux](@ref openvino_docs_install_guides_installing_openvino_docker_linux) | [Windows](@ref openvino_docs_install_guides_installing_openvino_docker_windows) -* [Build from source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md) +* Archive files (recommended) - :doc:`Linux ` | :doc:`Windows ` | :doc:`macOS ` +* :doc:`APT ` or :doc:`YUM ` for Linux +* Docker image - :doc:`Linux ` | :doc:`Windows ` +* `Build from source `__ + +Make sure that you also `install OpenCV `__ , as it's required for running sample applications. -Make sure that you also [install OpenCV](https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO), as it's required for running sample applications. +Install OpenVINO Development Tools +++++++++++++++++++++++++++++++++++ -### Install OpenVINO Development Tools +To install OpenVINO Development Tools, follow the :doc:`instructions for C++ developers on the Install OpenVINO Development Tools page `. This guide uses the ``googlenet-v1`` model from the Caffe framework, therefore, when you get to Step 4 of the installation, run the following command to install OpenVINO with the Caffe requirements: -To install OpenVINO Development Tools, follow the [instructions for C++ developers on the Install OpenVINO Development Tools page](../install_guides/installing-model-dev-tools.md#cpp_developers). This guide uses the `googlenet-v1` model from the Caffe framework, therefore, when you get to Step 4 of the installation, run the following command to install OpenVINO with the Caffe requirements: +.. code-block:: sh -``` sh pip install openvino-dev[caffe] -``` -### Build Samples -To build OpenVINO samples, follow the build instructions for your operating system on the [OpenVINO Samples](../OV_Runtime_UG/Samples_Overview.md) page. The build will take about 5-10 minutes, depending on your system. +Build Samples ++++++++++++++ -## Step 1: Download the Models +To build OpenVINO samples, follow the build instructions for your operating system on the :doc:`OpenVINO Samples ` page. The build will take about 5-10 minutes, depending on your system. + +.. _download-models: + +Step 1: Download the Models +########################### You must have a model that is specific for your inference task. Example model types are: @@ -46,34 +58,37 @@ You must have a model that is specific for your inference task. Example model ty You can use one of the following options to find a model suitable for OpenVINO: -- Download public or Intel pre-trained models from [Open Model Zoo](@ref model_zoo) using [Model Downloader tool](@ref omz_tools_downloader) +- Download public or Intel pre-trained models from :doc:`Open Model Zoo ` using :doc:`Model Downloader tool ` - Download from GitHub, Caffe Zoo, TensorFlow Zoo, etc. - Train your own model with machine learning tools This guide uses OpenVINO Model Downloader to get pre-trained models. You can use one of the following commands to find a model with this method: * List the models available in the downloader. - ``` sh + + .. code-block:: sh + omz_info_dumper --print_all - ``` -* Use `grep` to list models that have a specific name pattern (e.g. `ssd-mobilenet`, `yolo`). Replace `` with the name of the model. - ``` sh +* Use ``grep`` to list models that have a specific name pattern (e.g. ``ssd-mobilenet``, ``yolo``). Replace ```` with the name of the model. + + .. code-block:: sh + omz_info_dumper --print_all | grep - ``` -* Use Model Downloader to download models. Replace `` with the directory to download the model to and `` with the name of the model. - ``` sh +* Use Model Downloader to download models. Replace ```` with the directory to download the model to and ```` with the name of the model. + + .. code-block:: sh + omz_downloader --name --output_dir - ``` This guide used the following model to run the Image Classification Sample: - |Model Name | Code Sample or Demo App | - |-----------------------------------------------|------------------------------------------| - |`googlenet-v1` | Image Classification Sample | - -@sphinxdirective ++------------------+-----------------------------+ +| Model Name | Code Sample or Demo App | ++==================+=============================+ +| ``googlenet-v1`` | Image Classification Sample | ++------------------+-----------------------------+ .. dropdown:: Click to view how to download the GoogleNet v1 Caffe model @@ -147,25 +162,24 @@ This guide used the following model to run the Image Classification Sample: ========= Replacing text in /Users/username/models/public/googlenet-v1/googlenet-v1.prototxt ========= -@endsphinxdirective +.. _convert-models-to-intermediate-representation: -## Step 2: Convert the Model with Model Optimizer +Step 2: Convert the Model with Model Optimizer +############################################## In this step, your trained models are ready to run through the Model Optimizer to convert them to the IR (Intermediate Representation) format. For most model types, this is required before using OpenVINO Runtime with the model. -Models in the IR format always include an `.xml` and `.bin` file and may also include other files such as `.json` or `.mapping`. Make sure you have these files together in a single directory so OpenVINO Runtime can find them. +Models in the IR format always include an ``.xml`` and ``.bin`` file and may also include other files such as ``.json`` or ``.mapping``. Make sure you have these files together in a single directory so OpenVINO Runtime can find them. -REQUIRED: `model_name.xml` -REQUIRED: `model_name.bin` -OPTIONAL: `model_name.json`, `model_name.mapping`, etc. +REQUIRED: ``model_name.xml`` +REQUIRED: ``model_name.bin`` +OPTIONAL: ``model_name.json``, ``model_name.mapping``, etc. This tutorial uses the public GoogleNet v1 Caffe model to run the Image Classification Sample. See the example in the Download Models section of this page to learn how to download this model. The googlenet-v1 model is downloaded in the Caffe format. You must use Model Optimizer to convert the model to IR. -Create an `` directory to contain the model's Intermediate Representation (IR). - -@sphinxdirective +Create an ```` directory to contain the model's Intermediate Representation (IR). .. tab:: Linux @@ -185,21 +199,18 @@ Create an `` directory to contain the model's Intermediate Representatio mkdir ~/ir -@endsphinxdirective - -To save disk space for your IR file, you can apply [weights compression to FP16](../MO_DG/prepare_model/FP16_Compression.md). To generate an IR with FP16 weights, run Model Optimizer with the `--compress_to_fp16` option. +To save disk space for your IR file, you can apply :doc:`weights compression to FP16 `. To generate an IR with FP16 weights, run Model Optimizer with the ``--compress_to_fp16`` option. Generic Model Optimizer script: -``` sh +.. code-block:: sh + mo --input_model / -``` -The IR files produced by the script are written to the `` directory. -The command with most placeholders filled in and FP16 precision: +The IR files produced by the script are written to the ```` directory. -@sphinxdirective +The command with most placeholders filled in and FP16 precision: .. tab:: Linux @@ -219,95 +230,93 @@ The command with most placeholders filled in and FP16 precision: mo --input_model ~/models/public/googlenet-v1/googlenet-v1.caffemodel --compress_to_fp16 --output_dir ~/ir -@endsphinxdirective +.. _download-media: -## Step 3: Download a Video or a Photo as Media +Step 3: Download a Video or a Photo as Media +############################################ -Most of the samples require you to provide an image or a video as the input to run the model on. You can get them from sites like [Pexels](https://pexels.com) or [Google Images](https://images.google.com). +Most of the samples require you to provide an image or a video as the input to run the model on. You can get them from sites like `Pexels `__ or `Google Images `__ . As an alternative, OpenVINO also provides several sample images and videos for you to run code samples and demo applications: - - [Sample images and video](https://storage.openvinotoolkit.org/data/test_data/) - - [Sample videos](https://github.com/intel-iot-devkit/sample-videos) +- `Sample images and video `__ +- `Sample videos `__ -## Step 4: Run Inference on a Sample +.. _run-image-classification: + +Step 4: Run Inference on a Sample +################################## To run the **Image Classification** code sample with an input image using the IR model: 1. Set up the OpenVINO environment variables: -@sphinxdirective -.. tab:: Linux - .. code-block:: sh - - source /setupvars.sh - -.. tab:: Windows - - .. code-block:: bat - - \setupvars.bat - -.. tab:: macOS - - .. code-block:: sh - - source /setupvars.sh - -@endsphinxdirective + .. tab:: Linux + + .. code-block:: sh + + source /setupvars.sh + + .. tab:: Windows + + .. code-block:: bat + + \setupvars.bat + + .. tab:: macOS + + .. code-block:: sh + + source /setupvars.sh 2. Go to the code samples release directory created when you built the samples earlier: -@sphinxdirective -.. tab:: Linux - - .. code-block:: sh - - cd ~/openvino_cpp_samples_build/intel64/Release - -.. tab:: Windows - - .. code-block:: bat - - cd %USERPROFILE%\Documents\Intel\OpenVINO\openvino_samples_build\intel64\Release - -.. tab:: macOS - - .. code-block:: sh - - cd ~/openvino_cpp_samples_build/intel64/Release - -@endsphinxdirective + .. tab:: Linux + + .. code-block:: sh + + cd ~/openvino_cpp_samples_build/intel64/Release + + .. tab:: Windows + + .. code-block:: bat + + cd %USERPROFILE%\Documents\Intel\OpenVINO\openvino_samples_build\intel64\Release + + .. tab:: macOS + + .. code-block:: sh + + cd ~/openvino_cpp_samples_build/intel64/Release + 3. Run the code sample executable, specifying the input media file, the IR for your model, and a target device for performing inference: -@sphinxdirective - -.. tab:: Linux - - .. code-block:: sh - - classification_sample_async -i -m -d - -.. tab:: Windows - - .. code-block:: bat - classification_sample_async.exe -i -m -d - -.. tab:: macOS - - .. code-block:: sh - - classification_sample_async -i -m -d - -@endsphinxdirective - -### Examples + .. tab:: Linux + + .. code-block:: sh + + classification_sample_async -i -m -d + + .. tab:: Windows + + .. code-block:: bat + + classification_sample_async.exe -i -m -d + + .. tab:: macOS + + .. code-block:: sh + + classification_sample_async -i -m -d + +Examples +++++++++ -#### Running Inference on CPU +Running Inference on CPU +------------------------ -The following command shows how to run the Image Classification Code Sample using the [dog.bmp](https://storage.openvinotoolkit.org/data/test_data/images/224x224/dog.bmp) file as an input image, the model in IR format from the `ir` directory, and the CPU as the target hardware: +The following command shows how to run the Image Classification Code Sample using the `dog.bmp `__ file as an input image, the model in IR format from the ``ir`` directory, and the CPU as the target hardware: -@sphinxdirective .. tab:: Linux .. code-block:: sh @@ -326,15 +335,11 @@ The following command shows how to run the Image Classification Code Sample usin ./classification_sample_async -i ~/Downloads/dog.bmp -m ~/ir/googlenet-v1.xml -d CPU -@endsphinxdirective - When the sample application is complete, you are given the label and confidence for the top 10 categories. The input image and sample output of the inference results is shown below: - +.. image:: _static/images/dog.png -@sphinxdirective - - .. code-block:: sh +.. code-block:: sh Top 10 results: @@ -353,15 +358,15 @@ When the sample application is complete, you are given the label and confidence 216 0.0057589 clumber, clumber spaniel 154 0.0052615 Pekinese, Pekingese, Peke -@endsphinxdirective - The following example shows how to run the same sample using GPU as the target device. -#### Running Inference on GPU +Running Inference on GPU +------------------------ - > **NOTE**: Running inference on Intel® Processor Graphics (GPU) requires [additional hardware configuration steps](../install_guides/configurations-for-intel-gpu.md), as described earlier on this page. Running on GPU is not compatible with macOS. +.. note:: + + Running inference on Intel® Processor Graphics (GPU) requires :doc:`additional hardware configuration steps `, as described earlier on this page. Running on GPU is not compatible with macOS. -@sphinxdirective .. tab:: Linux .. code-block:: sh @@ -374,11 +379,13 @@ The following example shows how to run the same sample using GPU as the target d .\classification_sample_async.exe -i %USERPROFILE%\Downloads\dog.bmp -m %USERPROFILE%\Documents\ir\googlenet-v1.xml -d GPU -@endsphinxdirective +Other Demos and Samples +####################### + +See the :doc:`Samples ` page for more sample applications. Each sample page explains how the application works and shows how to run it. Use the samples as a starting point that can be adapted for your own application. -## Other Demos and Samples +OpenVINO also provides demo applications for using off-the-shelf models from :doc:`Open Model Zoo `. Visit :doc:`Open Model Zoo Demos ` if you'd like to see even more examples of how to run model inference with the OpenVINO API. -See the [Samples](../OV_Runtime_UG/Samples_Overview.md) page for more sample applications. Each sample page explains how the application works and shows how to run it. Use the samples as a starting point that can be adapted for your own application. +@endsphinxdirective -OpenVINO also provides demo applications for using off-the-shelf models from [Open Model Zoo](@ref model_zoo). Visit [Open Model Zoo Demos](@ref omz_demos) if you'd like to see even more examples of how to run model inference with the OpenVINO API. diff --git a/docs/install_guides/installing-openvino-apt.md b/docs/install_guides/installing-openvino-apt.md index eb47d2aa236848..aae1f0afc10aab 100644 --- a/docs/install_guides/installing-openvino-apt.md +++ b/docs/install_guides/installing-openvino-apt.md @@ -1,57 +1,60 @@ # Install Intel® Distribution of OpenVINO™ Toolkit for Linux Using APT Repository {#openvino_docs_install_guides_installing_openvino_apt} + + @sphinxdirective -With the OpenVINO™ 2022.3 release, you can install OpenVINO Runtime on Linux using the APT repository. OpenVINO™ Development Tools can be installed via PyPI only. See :ref:`Installing Additional Components ` for more information. +With the OpenVINO™ 2022.3 release, you can install OpenVINO Runtime on Linux using the APT repository. +OpenVINO™ Development Tools can be installed via PyPI only. +See `Installing Additional Components `__ for more information. -See the `Release Notes `_ for more information on updates in the latest release. +See the `Release Notes `__ +for more information on updates in the latest release. -Installing OpenVINO Runtime from APT is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the :doc:`Install OpenVINO from PyPI ` page for instructions on how to install OpenVINO Runtime for Python using PyPI. +Installing OpenVINO Runtime from APT is recommended for C++ developers. If you are working with Python, +the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the +:doc:`Install OpenVINO from PyPI ` page for instructions +on how to install OpenVINO Runtime for Python using PyPI. .. warning:: - By downloading and using this container and the included software, you agree to the terms and conditions of the `software license agreements `_. + By downloading and using this container and the included software, you agree to the terms and + conditions of the + `software license agreements `__. -@endsphinxdirective +Prerequisites +####################################### -## Prerequisites - -@sphinxdirective .. tab:: System Requirements | Full requirement listing is available in: - | `System Requirements Page `_ + | `System Requirements Page `__ .. tab:: Processor Notes Processor graphics are not included in all processors. - See `Product Specifications`_ for information about your processor. - - .. _Product Specifications: https://ark.intel.com/ + See `Product Specifications `__ for information about your processor. .. tab:: Software Requirements - * `CMake 3.13 or higher, 64-bit `_ + * `CMake 3.13 or higher, 64-bit `__ * GCC 7.5.0 (for Ubuntu 18.04) or GCC 9.3.0 (for Ubuntu 20.04) - * `Python 3.7 - 3.10, 64-bit `_ - - -.. _install runtime apt: + * `Python 3.7 - 3.10, 64-bit `__ -@endsphinxdirective +Installing OpenVINO Runtime +####################################### -## Installing OpenVINO Runtime +Step 1: Set Up the OpenVINO Toolkit APT Repository ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -### Step 1: Set Up the OpenVINO Toolkit APT Repository -@sphinxdirective -#. Install the GPG key for the repository +1. Install the GPG key for the repository - a. Download the `GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB `_ + a. Download the `GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB `__ You can also use the following command: @@ -73,7 +76,7 @@ Installing OpenVINO Runtime from APT is recommended for C++ developers. If you a sudo apt-get install gnupg -#. Add the repository via the following command: +2. Add the repository via the following command: .. tab:: Ubuntu 18 @@ -88,27 +91,26 @@ Installing OpenVINO Runtime from APT is recommended for C++ developers. If you a echo "deb https://apt.repos.intel.com/openvino/2022 focal main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2022.list -#. Update the list of packages via the update command: +3. Update the list of packages via the update command: .. code-block:: sh sudo apt update -#. Verify that the APT repository is properly set up. Run the apt-cache command to see a list of all available OpenVINO packages and components: +4. Verify that the APT repository is properly set up. Run the apt-cache command to see a list of all available OpenVINO packages and components: .. code-block:: sh apt-cache search openvino -@endsphinxdirective - -### Step 2: Install OpenVINO Runtime Using the APT Package Manager -#### Install OpenVINO Runtime +Step 2: Install OpenVINO Runtime Using the APT Package Manager +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -@sphinxdirective +Install OpenVINO Runtime +------------------------ .. tab:: The Latest Version @@ -141,13 +143,12 @@ Installing OpenVINO Runtime from APT is recommended for C++ developers. If you a .. note:: - You can use ``--no-install-recommends`` option to install only required packages. Keep in mind that the build tools must be installed **separately** if you want to compile the samples. + You can use ``--no-install-recommends`` option to install only required packages. + Keep in mind that the build tools must be installed **separately** if you want to compile the samples. -@endsphinxdirective -#### Check for Installed Packages and Versions - -@sphinxdirective +Check for Installed Packages and Versions +----------------------------------------- Run the following command: @@ -155,32 +156,26 @@ Run the following command: apt list --installed | grep openvino -.. _intall additional components apt: - -@endsphinxdirective +Step 3 (Optional): Install Additional Components +++++++++++++++++++++++++++++++++++++++++++++++++ -### Step 3 (Optional): Install Additional Components - -@sphinxdirective - -OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you installed OpenVINO Runtime using APT, OpenVINO Development Tools must be installed separately. +OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. +It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. +If you installed OpenVINO Runtime using APT, OpenVINO Development Tools must be installed separately. See the **For C++ Developers** section on the :doc:`Install OpenVINO Development Tools ` page for instructions. -@endsphinxdirective -### Step 4 (Optional): Configure Inference on Non-CPU Devices - -@sphinxdirective +Step 4 (Optional): Configure Inference on Non-CPU Devices ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -To enable the toolkit components to use processor graphics (GPU) on your system, follow the steps in :doc:`GPU Setup Guide `. +To enable the toolkit components to use processor graphics (GPU) on your system, follow the steps +in :doc:`GPU Setup Guide `. -@endsphinxdirective -### Step 5: Build Samples - -@sphinxdirective +Step 5: Build Samples ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ To build the C++ or C sample applications for Linux, run the ``build_samples.sh`` script: @@ -198,11 +193,10 @@ To build the C++ or C sample applications for Linux, run the ``build_samples.sh` For more information, refer to :ref:`Build the Sample Applications on Linux `. -@endsphinxdirective -### Uninstalling OpenVINO Runtime -@sphinxdirective +Uninstalling OpenVINO Runtime +####################################### To uninstall OpenVINO Runtime via APT, run the following command based on your needs: @@ -224,16 +218,15 @@ To uninstall OpenVINO Runtime via APT, run the following command based on your n sudo apt autoremove openvino-2022.3.0 -@endsphinxdirective +What's Next? +####################################### -## What's Next? +Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! +Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials: -@sphinxdirective - -Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials: - -* Try the `C++ Quick Start Example `_ for step-by-step instructions on building and running a basic image classification C++ application. +* Try the `C++ Quick Start Example `_ for step-by-step + instructions on building and running a basic image classification C++ application. .. image:: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :width: 400 @@ -243,7 +236,7 @@ Now that you've installed OpenVINO Runtime, you're ready to run your own machine * `Basic object detection with the Hello Reshape SSD C++ sample `_ * `Automatic speech recognition C++ sample `_ -You can also try the following things: +You can also try the following: * Learn more about :doc:`OpenVINO Workflow `. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation `. @@ -252,8 +245,12 @@ You can also try the following things: * See sample applications in :doc:`OpenVINO toolkit Samples Overview `. * Take a glance at the OpenVINO product home page: https://software.intel.com/en-us/openvino-toolkit. -@endsphinxdirective -## Additional Resources +Additional Resources +####################################### + +- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) + + -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file +@endsphinxdirective \ No newline at end of file diff --git a/docs/install_guides/installing-openvino-brew.md b/docs/install_guides/installing-openvino-brew.md index 557e38872e84c2..1bbf98042015a2 100644 --- a/docs/install_guides/installing-openvino-brew.md +++ b/docs/install_guides/installing-openvino-brew.md @@ -2,58 +2,52 @@ @sphinxdirective -With the OpenVINO™ 2022.3 release, you can install OpenVINO Runtime on macOS and Linux via `Homebrew `_. OpenVINO™ Development Tools can be installed via PyPI only. See :ref:`Installing Additional Components ` for more information. +With the OpenVINO™ 2022.3 release, you can install OpenVINO Runtime on macOS and Linux via `Homebrew `_. OpenVINO™ Development Tools can be installed via PyPI only. See `Installing Additional Components <#optional-installing-additional-components>`__ for more information. -See the `Release Notes `_ for more information on updates in the latest release. +See the `Release Notes `__ for more information on updates in the latest release. Installing OpenVINO Runtime from Homebrew is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the :doc:`Install OpenVINO from PyPI ` page for instructions on how to install OpenVINO Runtime for Python using PyPI. -.. note:: +.. note:: Only CPU is supported for inference if you install OpenVINO via HomeBrew. -.. warning:: +.. warning:: - By downloading and using this container and the included software, you agree to the terms and conditions of the `software license agreements `_. + By downloading and using this container and the included software, you agree to the terms and conditions of the `software license agreements `__. -@endsphinxdirective - -## Prerequisites - -### System Requirements -@sphinxdirective - -Full requirement listing is available on the `System Requirements Page `_ +Prerequisites +#################### -@endsphinxdirective +System Requirements +++++++++++++++++++++ -### Software Requirements +Full requirement listing is available on the `System Requirements Page `__ -@sphinxdirective +Software Requirements ++++++++++++++++++++++ .. tab:: macOS * `Homebrew `_ - * `CMake 3.13 or higher `_ (choose "macOS 10.13 or later"). Add `/Applications/CMake.app/Contents/bin` to path (for default installation). - * `Python 3.7 - 3.10 `_ (choose 3.7 - 3.10). Install and add it to path. - * Apple Xcode Command Line Tools. In the terminal, run `xcode-select --install` from any directory to install it. + * `CMake 3.13 or higher `__ (choose "macOS 10.13 or later"). Add ``/Applications/CMake.app/Contents/bin`` to path (for default installation). + * `Python 3.7 - 3.10 `__ (choose 3.7 - 3.10). Install and add it to path. + * Apple Xcode Command Line Tools. In the terminal, run ``xcode-select --install`` from any directory to install it. * (Optional) Apple Xcode IDE (not required for OpenVINO™, but useful for development) .. tab:: Linux * `Homebrew `_ - * `CMake 3.13 or higher, 64-bit `_ + * `CMake 3.13 or higher, 64-bit `__ * GCC 7.5.0 (for Ubuntu 18.04) or GCC 9.3.0 (for Ubuntu 20.04) - * `Python 3.7 - 3.10, 64-bit `_ - -@endsphinxdirective + * `Python 3.7 - 3.10, 64-bit `__ -## Installing OpenVINO Runtime -@sphinxdirective +Installing OpenVINO Runtime +########################### -1. Make sure that you have installed HomeBrew on your system. If not, follow the instructions on `the Homebrew website `_ to install and configure it. +1. Make sure that you have installed HomeBrew on your system. If not, follow the instructions on `the Homebrew website `__ to install and configure it. 2. Open a command prompt terminal window, and run the following command to install OpenVINO Runtime: @@ -61,15 +55,11 @@ Full requirement listing is available on the `System Requirements Page `_. -@endsphinxdirective - -## Uninstalling OpenVINO +Uninstalling OpenVINO +##################### To uninstall OpenVINO via HomeBrew, use the following command: -```sh -brew uninstall openvino -``` -## What's Next? +.. code-block:: sh -@sphinxdirective + brew uninstall openvino + + +What's Next? +#################### -Now that you've installed OpenVINO Runtime, you can try the following things: +Now that you've installed OpenVINO Runtime, you can try the following things: * Learn more about :doc:`OpenVINO Workflow `. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation `. @@ -99,8 +89,9 @@ Now that you've installed OpenVINO Runtime, you can try the following things: * See sample applications in :doc:`OpenVINO toolkit Samples Overview `. * Take a glance at the OpenVINO product home page: https://software.intel.com/en-us/openvino-toolkit. -@endsphinxdirective +Additional Resources +#################### -## Additional Resources +* `OpenVINO Installation Selector Tool `__ -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file +@endsphinxdirective diff --git a/docs/install_guides/installing-openvino-docker-linux.md b/docs/install_guides/installing-openvino-docker-linux.md index 745b5db27b8a0a..828977d3b15b34 100644 --- a/docs/install_guides/installing-openvino-docker-linux.md +++ b/docs/install_guides/installing-openvino-docker-linux.md @@ -1,178 +1,227 @@ # Install Intel® Distribution of OpenVINO™ toolkit for Linux from a Docker Image {#openvino_docs_install_guides_installing_openvino_docker_linux} + +@sphinxdirective + This guide provides steps on creating a Docker image with Intel® Distribution of OpenVINO™ toolkit for Linux and using the image on different devices. -## System Requirements +System Requirements +################### -@sphinxdirective .. tab:: Target Operating Systems with Python Versions - +----------------------------------------------+-------------------------+ - | Operating System | Included Python Version | - +==============================================+=========================+ - | Ubuntu 18.04 long-term support (LTS), 64-bit | 3.8 | - +----------------------------------------------+-------------------------+ - | Ubuntu 20.04 long-term support (LTS), 64-bit | 3.8 | - +----------------------------------------------+-------------------------+ - | Red Hat Enterprise Linux 8, 64-bit | 3.8 | - +----------------------------------------------+-------------------------+ + +----------------------------------------------+-------------------------+ + | Operating System | Included Python Version | + +==============================================+=========================+ + | Ubuntu 18.04 long-term support (LTS), 64-bit | 3.8 | + +----------------------------------------------+-------------------------+ + | Ubuntu 20.04 long-term support (LTS), 64-bit | 3.8 | + +----------------------------------------------+-------------------------+ + | Red Hat Enterprise Linux 8, 64-bit | 3.8 | + +----------------------------------------------+-------------------------+ .. tab:: Host Operating Systems - * Linux - * Windows Subsystem for Linux 2 (WSL2) on CPU or GPU - * macOS on CPU only - - To launch a Linux image on WSL2 when trying to run inferences on a GPU, make sure that the following requirements are met: + * Linux + * Windows Subsystem for Linux 2 (WSL2) on CPU or GPU + * macOS on CPU only + + To launch a Linux image on WSL2 when trying to run inferences on a GPU, make sure that the following requirements are met: + + * Only Windows 10 with 21H2 update or above installed and Windows 11 are supported. + * Intel GPU driver on Windows host with version 30.0.100.9684 or above need be installed. For more details, + `this article at intel.com `__ . + * From 2022.1 release, the Docker images contain preinstalled recommended version of OpenCL Runtime with WSL2 support. - - Only Windows 10 with 21H2 update or above installed and Windows 11 are supported. - - Intel GPU driver on Windows host with version 30.0.100.9684 or above need be installed. Please see `this article`_ for more details. - - From 2022.1 release, the Docker images contain preinstalled recommended version of OpenCL Runtime with WSL2 support. - - .. _this article: https://www.intel.com/content/www/us/en/artificial-intelligence/harness-the-power-of-intel-igpu-on-your-machine.html#articleparagraph_983312434 -@endsphinxdirective +Installation +############# -## Installation Flow +* Use a prebuilt image: + + 1. `Get a prebuilt image from provided sources `__ + 2. `Run the image on different devices `__ + 3. `Run samples in the Docker image `__ + +* If you want to customize your image, you can also build a Docker image manually: + + 1. `Prepare a Dockerfile `__ + 2. `Configure the Docker image `__ + 3. `Run the image on different devices `__ + 4. `Run samples in the Docker image `__ -There are two ways to install OpenVINO with Docker. You can choose either of them according to your needs: -* Use a prebuilt image. Do the following steps: - 1. Get a prebuilt image from provided sources. - 2. Run the image on different devices. - 3. (Optional) Run samples in the Docker image. -* If you want to customize your image, you can also build a Docker image manually by using the following steps: - 1. Prepare a Dockerfile. - 2. Configure the Docker image. - 3. Run the image on different devices. - 4. (Optional) Run samples in the Docker image. -## Getting a Prebuilt Image from Provided Sources +Getting a Prebuilt Image from Provided Sources +++++++++++++++++++++++++++++++++++++++++++++++ You can find prebuilt images on: -- [Docker Hub](https://hub.docker.com/u/openvino) -- [Red Hat Quay.io](https://quay.io/organization/openvino) -- [Red Hat Ecosystem Catalog (runtime image)](https://catalog.redhat.com/software/containers/intel/openvino-runtime/606ff4d7ecb5241699188fb3) -- [Red Hat Ecosystem Catalog (development image)](https://catalog.redhat.com/software/containers/intel/openvino-dev/613a450dc9bc35f21dc4a1f7) -- [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/intel_corporation.openvino) +- `Docker Hub `__ +- `Red Hat Quay.io `__ +- `Red Hat Ecosystem Catalog (runtime image) `__ +- `Red Hat Ecosystem Catalog (development image) `__ +- `Azure Marketplace `__ -## Preparing a Dockerfile +Preparing a Dockerfile +++++++++++++++++++++++ -You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles) or generate a Dockerfile with your settings via [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) which can generate a Dockerfile, build, test and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. -You can also try our [Tutorials](https://github.com/openvinotoolkit/docker_ci/tree/master/docs/tutorials) which demonstrate the usage of Docker containers with OpenVINO. +You can use the `available Dockerfiles on GitHub `__ +or generate a Dockerfile with your settings via `DockerHub CI Framework `__ +which can generate a Dockerfile, build, test and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. +You can also try our `Tutorials `__ +which demonstrate the usage of Docker containers with OpenVINO. -## Configuring the Image for Different Devices +Configuring the Image for Different Devices ++++++++++++++++++++++++++++++++++++++++++++ -If you want to run inferences on a CPU no extra configuration is needed. Go to Running the image on different devices for the next step. +If you want to run inference on a CPU no extra configuration is needed. +Go to `Run the image on different devices `__ for the next step. -### Configuring Docker Image for GPU +Configuring Docker Image for GPU +-------------------------------- -By default, the distributed Docker image for OpenVINO has the recommended version of Intel® Graphics Compute Runtime for oneAPI Level Zero and OpenCL Driver for the operating system installed inside. If you want to build an image with a custom version of OpenCL Runtime included, you need to modify the Dockerfile using the lines below (the 19.41.14441 version is used as an example) and build the image manually: +By default, the distributed Docker image for OpenVINO has the recommended version of +Intel® Graphics Compute Runtime for oneAPI Level Zero and OpenCL Driver for the operating +system installed inside. If you want to build an image with a custom version of OpenCL Runtime included, +you need to modify the Dockerfile using the lines below (the 19.41.14441 version is used as an example) and build the image manually: **Ubuntu 18.04/20.04**: -```sh -WORKDIR /tmp/opencl -RUN useradd -ms /bin/bash -G video,users openvino && \ - chown openvino -R /home/openvino - -RUN apt-get update && \ - apt-get install -y --no-install-recommends ocl-icd-libopencl1 && \ - rm -rf /var/lib/apt/lists/* && \ - curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-gmmlib_19.3.2_amd64.deb" --output "intel-gmmlib_19.3.2_amd64.deb" && \ - curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-igc-core_1.0.2597_amd64.deb" --output "intel-igc-core_1.0.2597_amd64.deb" && \ - curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-igc-opencl_1.0.2597_amd64.deb" --output "intel-igc-opencl_1.0.2597_amd64.deb" && \ - curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-opencl_19.41.14441_amd64.deb" --output "intel-opencl_19.41.14441_amd64.deb" && \ - curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-ocloc_19.41.14441_amd64.deb" --output "intel-ocloc_19.04.12237_amd64.deb" && \ - dpkg -i /tmp/opencl/*.deb && \ - ldconfig && \ - rm /tmp/opencl -``` +.. code-block:: sh + + WORKDIR /tmp/opencl + RUN useradd -ms /bin/bash -G video,users openvino && \ + chown openvino -R /home/openvino + + RUN apt-get update && \ + apt-get install -y --no-install-recommends ocl-icd-libopencl1 && \ + rm -rf /var/lib/apt/lists/* && \ + curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-gmmlib_19.3.2_amd64.deb" --output "intel-gmmlib_19.3.2_amd64.deb" && \ + curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-igc-core_1.0.2597_amd64.deb" --output "intel-igc-core_1.0.2597_amd64.deb" && \ + curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-igc-opencl_1.0.2597_amd64.deb" --output "intel-igc-opencl_1.0.2597_amd64.deb" && \ + curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-opencl_19.41.14441_amd64.deb" --output "intel-opencl_19.41.14441_amd64.deb" && \ + curl -L "https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-ocloc_19.41.14441_amd64.deb" --output "intel-ocloc_19.04.12237_amd64.deb" && \ + dpkg -i /tmp/opencl/*.deb && \ + ldconfig && \ + rm /tmp/opencl + **RHEL 8**: -```sh -WORKDIR /tmp/opencl -RUN useradd -ms /bin/bash -G video,users openvino && \ - chown openvino -R /home/openvino -RUN groupmod -g 44 video - -RUN yum update -y && yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - yum update -y && yum install -y ocl-icd ocl-icd-devel && \ - yum clean all && rm -rf /var/cache/yum && \ - curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-gmmlib-19.3.2-1.el7.x86_64.rpm/download -o intel-gmmlib-19.3.2-1.el7.x86_64.rpm && \ - curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-gmmlib-devel-19.3.2-1.el7.x86_64.rpm/download -o intel-gmmlib-devel-19.3.2-1.el7.x86_64.rpm && \ - curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-core-1.0.2597-1.el7.x86_64.rpm/download -o intel-igc-core-1.0.2597-1.el7.x86_64.rpm && \ - curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-opencl-1.0.2597-1.el7.x86_64.rpm/download -o intel-igc-opencl-1.0.2597-1.el7.x86_64.rpm && \ - curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-opencl-devel-1.0.2597-1.el7.x86_64.rpm/download -o intel-igc-opencl-devel-1.0.2597-1.el7.x86_64.rpm && \ - curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-opencl-19.41.14441-1.el7.x86_64.rpm/download -o intel-opencl-19.41.14441-1.el7.x86_64.rpm \ - rpm -ivh ${TEMP_DIR}/*.rpm && \ - ldconfig && \ - rm -rf ${TEMP_DIR} && \ - yum remove -y epel-release -``` - -## Running the Docker Image on Different Devices - -### Running the Image on CPU +.. code-block:: sh + + WORKDIR /tmp/opencl + RUN useradd -ms /bin/bash -G video,users openvino && \ + chown openvino -R /home/openvino + RUN groupmod -g 44 video + + RUN yum update -y && yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + yum update -y && yum install -y ocl-icd ocl-icd-devel && \ + yum clean all && rm -rf /var/cache/yum && \ + curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-gmmlib-19.3.2-1.el7.x86_64.rpm/download -o intel-gmmlib-19.3.2-1.el7.x86_64.rpm && \ + curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-gmmlib-devel-19.3.2-1.el7.x86_64.rpm/download -o intel-gmmlib-devel-19.3.2-1.el7.x86_64.rpm && \ + curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-core-1.0.2597-1.el7.x86_64.rpm/download -o intel-igc-core-1.0.2597-1.el7.x86_64.rpm && \ + curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-opencl-1.0.2597-1.el7.x86_64.rpm/download -o intel-igc-opencl-1.0.2597-1.el7.x86_64.rpm && \ + curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-opencl-devel-1.0.2597-1.el7.x86_64.rpm/download -o intel-igc-opencl-devel-1.0.2597-1.el7.x86_64.rpm && \ + curl -L https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-opencl-19.41.14441-1.el7.x86_64.rpm/download -o intel-opencl-19.41.14441-1.el7.x86_64.rpm \ + rpm -ivh ${TEMP_DIR}/*.rpm && \ + ldconfig && \ + rm -rf ${TEMP_DIR} && \ + yum remove -y epel-release + + +Running the Docker Image on Different Devices ++++++++++++++++++++++++++++++++++++++++++++++ + +Running the Image on CPU +------------------------- Run the Docker image with the following command: -``` -docker run -it --rm -``` -Note the following things: +.. code-block:: sh + + docker run -it --rm -- Kernel reports the same information for all containers as for native application, for example, CPU, memory information. -- All instructions that are available to host process available for process in container, including, for example, AVX2, AVX512. No restrictions. -- Docker does not use virtualization or emulation. The process in Docker is just a regular Linux process, but it is isolated from external world on kernel level. Performance loss is minor. +Note the following: -### Running the Image on GPU +- Kernel reports the same information for all containers as for native application, + for example, CPU, memory information. +- All instructions that are available to host process available for process in container, + including, for example, AVX2, AVX512. No restrictions. +- Docker does not use virtualization or emulation. The process in Docker is just a regular + Linux process, but it is isolated from external world on kernel level. Performance loss is minor. -> **NOTE**: Only Intel® integrated graphics are supported. -Note the following things: +Running the Image on GPU +------------------------- + +.. note:: + + Only Intel® integrated graphics are supported. + +Note the following: - GPU is not available in the container by default. You must attach it to the container. - Kernel driver must be installed on the host. -- In the container, non-root user must be in the `video` and `render` groups. To add a user to the render group, follow the [Configuration Guide for the Intel® Graphics Compute Runtime for OpenCL™ on Ubuntu 20.04](https://github.com/openvinotoolkit/docker_ci/blob/master/configure_gpu_ubuntu20.md). +- In the container, non-root user must be in the ``video`` and ``render`` groups. + To add a user to the render group, follow the + `Configuration Guide for the Intel® Graphics Compute Runtime for OpenCL™ on Ubuntu 20.04 `__. -To make GPU available in the container, attach the GPU to the container using `--device /dev/dri` option and run the container: +To make GPU available in the container, attach the GPU to the container using ``--device /dev/dri`` option and run the container: * Ubuntu 18 or RHEL 8: - ```sh - docker run -it --rm --device /dev/dri - ``` - > **NOTE**: If your host system is Ubuntu 20, follow the [Configuration Guide for the Intel® Graphics Compute Runtime for OpenCL™ on Ubuntu* 20.04](https://github.com/openvinotoolkit/docker_ci/blob/master/configure_gpu_ubuntu20.md). + + .. code-block:: sh + + docker run -it --rm --device /dev/dri + + .. note:: + + If your host system is Ubuntu 20, follow the + `Configuration Guide for the Intel® Graphics Compute Runtime for OpenCL™ on Ubuntu* 20.04 `__. * WSL2: - ```sh - docker run -it --rm --device /dev/dxg --volume /usr/lib/wsl:/usr/lib/wsl - ``` - > **NOTE**: To launch a Linux image on WSL2, make sure that the additional requirements in System Requirements are met. + + .. code-block:: sh + docker run -it --rm --device /dev/dxg --volume /usr/lib/wsl:/usr/lib/wsl -## Running Samples in Docker Image + .. note:: + + To launch a Linux image on WSL2, make sure that the additional `System Requirements `__ are met. -To run the `Hello Classification Sample` on a specific inference device, run the following commands: + +Running Samples in Docker Image +############################### + +To run the ``Hello Classification Sample`` on a specific inference device, run the following commands: **CPU**: -```sh -docker run -it --rm -/bin/bash -c "cd ~ && omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -O https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python3 /opt/intel/openvino/samples/python/hello_classification/hello_classification.py public/googlenet-v1/FP16/googlenet-v1.xml car_1.bmp CPU" -``` +.. code-block:: sh + + docker run -it --rm + /bin/bash -c "cd ~ && omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -O https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python3 /opt/intel/openvino/samples/python/hello_classification/hello_classification.py public/googlenet-v1/FP16/googlenet-v1.xml car_1.bmp CPU" **GPU**: -```sh -docker run -itu root:root --rm --device /dev/dri:/dev/dri -/bin/bash -c "omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -O https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python3 samples/python/hello_classification/hello_classification.py public/googlenet-v1/FP16/googlenet-v1.xml car_1.bmp GPU" -``` +.. code-block:: sh + + docker run -itu root:root --rm --device /dev/dri:/dev/dri + /bin/bash -c "omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -O https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python3 samples/python/hello_classification/hello_classification.py public/googlenet-v1/FP16/googlenet-v1.xml car_1.bmp GPU" + + +Additional Resources +############################### + +- `DockerHub CI Framework `__ for Intel® Distribution of OpenVINO™ toolkit. + The Framework can generate a Dockerfile, build, test, and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. + You can reuse available Dockerfiles, add your layer and customize the image of OpenVINO™ for your needs. +- `Intel® Distribution of OpenVINO™ toolkit home page `__ +- `OpenVINO Installation Selector Tool `__ + + +@endsphinxdirective -## Additional Resources -- [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) for Intel® Distribution of OpenVINO™ toolkit. The Framework can generate a Dockerfile, build, test, and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. You can reuse available Dockerfiles, add your layer and customize the image of OpenVINO™ for your needs. -- Intel® Distribution of OpenVINO™ toolkit home page: [https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit) -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file diff --git a/docs/install_guides/installing-openvino-docker-windows.md b/docs/install_guides/installing-openvino-docker-windows.md index b42034cb7bc9b2..e446b4e9ebaa53 100644 --- a/docs/install_guides/installing-openvino-docker-windows.md +++ b/docs/install_guides/installing-openvino-docker-windows.md @@ -1,97 +1,121 @@ # Install Intel® Distribution of OpenVINO™ toolkit for Windows from Docker Image {#openvino_docs_install_guides_installing_openvino_docker_windows} +@sphinxdirective + This guide provides steps for creating a Docker image with Intel® Distribution of OpenVINO™ toolkit for Windows and using the Docker image on different devices. -## System Requirements +.. _system-requirements-docker-windows: + +System Requirements +#################### -@sphinxdirective .. tab:: Target Operating System with Python Versions - +------------------------------------+--------------------------+ - | Operating System | Supported Python Version | - +====================================+==========================+ - | Windows Server Core base LTSC 2019 | 3.8 | - +------------------------------------+--------------------------+ - | Windows 10, version 20H2 | 3.8 | - +------------------------------------+--------------------------+ + +------------------------------------+--------------------------+ + | Operating System | Supported Python Version | + +====================================+==========================+ + | Windows Server Core base LTSC 2019 | 3.8 | + +------------------------------------+--------------------------+ + | Windows 10, version 20H2 | 3.8 | + +------------------------------------+--------------------------+ .. tab:: Host Operating Systems - * Windows 10, 64-bit Pro, Enterprise or Education (1607 Anniversary Update, Build 14393 or later) editions - * Windows Server 2016 or higher + * Windows 10, 64-bit Pro, Enterprise or Education (1607 Anniversary Update, Build 14393 or later) editions + * Windows Server 2016 or higher -@endsphinxdirective -### Additional Requirements for GPU +Additional Requirements for GPU ++++++++++++++++++++++++++++++++ To use GPU Acceleration in Windows containers, make sure that the following requirements for Windows host, OpenVINO and Docker are met: -- [Windows requirements](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/gpu-acceleration): +- `Windows requirements `__: + - The container host must be running Windows Server 2019 or Windows 10 of version 1809 or higher. - - The container base image must be `mcr.microsoft.com/windows:1809` or higher. Windows Server Core and Nano Server container images are not currently supported. + - The container base image must be ``mcr.microsoft.com/windows:1809`` or higher. Windows Server Core and Nano Server container images are not currently supported. - The container host must be running Docker Engine 19.03 or higher. - The container host must have GPU running display drivers of version WDDM 2.5 or higher. + - GPU requirement for OpenVINO: Intel Graphics Driver for Windows of version 15.65 or higher. -- [Docker isolation mode requirements](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container): +- `Docker isolation mode requirements `__: + - Windows host and container version tags must match. - - [Windows host and container isolation process support](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility). + - `Windows host and container isolation process support `__. -## Installation Flow +Installation Flow +#################### There are two ways to install OpenVINO with Docker. You can choose either of them according to your needs: + * Use a prebuilt image. Do the following steps: - 1. Get a prebuilt image from provided sources. - 2. Run the image on different devices. + + 1. `Get a prebuilt image from provided sources <#getting-a-prebuilt-image-from-provided-sources>`__. + 2. `Run the image on different devices <#running-the-docker-image-on-different-devices>`__. + * If you want to customize your image, you can also build a Docker image manually by using the following steps: - 1. Prepare a Dockerfile. - 2. Configure the Docker image. - 3. Run the image on different devices. -## Getting a Prebuilt Image from Provided Sources + 1. `Prepare a Dockerfile <#preparing-a-dockerfile>`__. + 2. `Configure the Docker image <#configuring-the-docker-image-for-different-devices>`__. + 3. `Run the image on different devices <#running-the-docker-image-on-different-devices>`__. + +Getting a Prebuilt Image from Provided Sources +############################################## You can find prebuilt images on: -- [Docker Hub](https://hub.docker.com/u/openvino) -- [Azure Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/intel_corporation.openvino) +- `Docker Hub `__ +- `Azure Marketplace `__ + +Preparing a Dockerfile +###################### + +You can use the `available Dockerfiles on GitHub `__ or generate a Dockerfile with your settings via `DockerHub CI Framework `__ which can generate a Dockerfile, build, test and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. -## Preparing a Dockerfile +Configuring the Docker Image for Different Devices +################################################## -You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles) or generate a Dockerfile with your settings via [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) which can generate a Dockerfile, build, test and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. +Installing Additional Dependencies for CPU +++++++++++++++++++++++++++++++++++++++++++ -## Configuring the Docker Image for Different Devices +Installing CMake +---------------- -### Installing Additional Dependencies for CPU +To add CMake to the image, add the following commands to the Dockerfile: -#### Installing CMake +.. code-block:: bat - To add CMake to the image, add the following commands to the Dockerfile: - ```bat RUN powershell.exe -Command ` Invoke-WebRequest -URI https://cmake.org/files/v3.14/cmake-3.14.7-win64-x64.msi -OutFile %TMP%\\cmake-3.14.7-win64-x64.msi ; ` Start-Process %TMP%\\cmake-3.14.7-win64-x64.msi -ArgumentList '/quiet /norestart' -Wait ; ` Remove-Item %TMP%\\cmake-3.14.7-win64-x64.msi -Force RUN SETX /M PATH "C:\Program Files\CMake\Bin;%PATH%" - ``` - In case of proxy issues, please add the `ARG HTTPS_PROXY` and `-Proxy %%HTTPS_PROXY%` settings to the `powershell.exe` command to the Dockerfile. Then build a Docker image: - ```bat + +In case of proxy issues, please add the ``ARG HTTPS_PROXY`` and ``-Proxy %%HTTPS_PROXY%`` settings to the ``powershell.exe`` command to the Dockerfile. Then build a Docker image: + +.. code-block:: bat + docker build . -t ` --build-arg HTTPS_PROXY= - ``` - -#### Installing Microsoft Visual Studio Build Tools - You can add Microsoft Visual Studio Build Tools to a Windows OS Docker image using the [offline](https://docs.microsoft.com/en-us/visualstudio/install/create-an-offline-installation-of-visual-studio?view=vs-2019) or [online](https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019) installers for Build Tools. - - Microsoft Visual Studio Build Tools are licensed as a supplement your existing Microsoft Visual Studio license. - - Any images built with these tools should be for your personal use or for use in your organization in accordance with your existing Visual Studio and Windows licenses. - To add MSBuild 2019 to the image, add the following commands to the Dockerfile: - ```bat - RUN powershell.exe -Command Invoke-WebRequest -URI https://aka.ms/vs/16/release/vs_buildtools.exe -OutFile %TMP%\\vs_buildtools.exe +Installing Microsoft Visual Studio Build Tools +---------------------------------------------- + +You can add Microsoft Visual Studio Build Tools to a Windows OS Docker image using the `offline `__ or `online `__ installers for Build Tools. + +Microsoft Visual Studio Build Tools are licensed as a supplement your existing Microsoft Visual Studio license. +Any images built with these tools should be for your personal use or for use in your organization in accordance with your existing Visual Studio and Windows licenses. + +To add MSBuild 2019 to the image, add the following commands to the Dockerfile: + +.. code-block:: bat + + RUN powershell.exe -Command Invoke-WebRequest -URI https://aka.ms/vs/16/release/vs_buildtools.exe -OutFile %TMP%\\vs_buildtools.exe + RUN %TMP%\\vs_buildtools.exe --quiet --norestart --wait --nocache ` --installPath "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools" ` --add Microsoft.VisualStudio.Workload.MSBuildTools ` @@ -101,75 +125,110 @@ You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoo --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 ` --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 ` --remove Microsoft.VisualStudio.Component.Windows81SDK || IF "%ERRORLEVEL%"=="3010" EXIT 0 && powershell set-executionpolicy remotesigned - ``` - In case of proxy issues, please use the [offline installer for Build Tools](https://docs.microsoft.com/en-us/visualstudio/install/create-an-offline-installation-of-visual-studio?view=vs-2019). -### Configuring the Image for GPU +In case of proxy issues, please use the `offline installer for Build Tools `__. + +Configuring the Image for GPU ++++++++++++++++++++++++++++++ + +.. note:: + + Since GPU is not supported in `prebuilt images <#getting-a-prebuilt-image-from-provided-sources>`__ or `default Dockerfiles `__, you must make sure the Additional Requirements for GPU in `System Requirements <#system-requirements>`__ are met, and do the following steps to build the image manually. -> **NOTE**: Since GPU is not supported in prebuilt images or [default Dockerfiles](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles), you must make sure the Additional Requirements for GPU in System Requirements are met, and do the following steps to build the image manually. +1. Reuse one of `available Dockerfiles `__. You can also use your own Dockerfile. +2. Check your `Windows host and container isolation process compatibility `__. +3. Find the appropriate Windows container base image on `DockerHub `__ and set up your host/container version in the ``FROM`` Dockerfile instruction. + + For example, in the ``openvino_c_dev_.dockerfile``, change: + + .. code-block:: bat + + FROM mcr.microsoft.com/windows/servercore:ltsc2019 AS ov_base -1. Reuse one of [available Dockerfiles](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles). You can also use your own Dockerfile. -2. Check your [Windows host and container isolation process compatibility](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility). -3. Find the appropriate Windows container base image on [DockerHub](https://hub.docker.com/_/microsoft-windows) and set up your host/container version in the `FROM` Dockerfile instruction. - For example, in the `openvino_c_dev_.dockerfile`, change: - ```bat - FROM mcr.microsoft.com/windows/servercore:ltsc2019 AS ov_base - ``` to: - ```bat - FROM mcr.microsoft.com/windows:20H2 - ``` + + .. code-block:: bat + + FROM mcr.microsoft.com/windows:20H2 + + 4. Build the Docker image by running the following command: - ```bat - docker build --build-arg package_url= -f -t . - ``` -5. Copy `OpenCL.dll` from your `C:\Windows\System32` host folder to any `temp` directory: - ```bat - mkdir C:\tmp - copy C:\Windows\System32\OpenCL.dll C:\tmp - ``` -## Running the Docker Image on Different Devices + .. code-block:: bat + + docker build --build-arg package_url= -f -t . + + +5. Copy ``OpenCL.dll`` from your ``C:\Windows\System32`` host folder to any ``temp`` directory: + + .. code-block:: bat + + mkdir C:\tmp + copy C:\Windows\System32\OpenCL.dll C:\tmp + -### Running the Image on CPU +Running the Docker Image on Different Devices +############################################# + +Running the Image on CPU +++++++++++++++++++++++++ To start the interactive session, run the following command: -```bat -docker run -it --rm -``` + +.. code-block:: bat + + docker run -it --rm + If you want to try some samples, run the image with the following command: -```bat -docker run -it --rm -cmd /S /C "omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -kO https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python samples\python\hello_classification\hello_classification.py public\googlenet-v1\FP16\googlenet-v1.xml car_1.bmp CPU" -``` -### Running the Image on GPU +.. code-block:: bat + + docker run -it --rm + cmd /S /C "omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -kO https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python samples\python\hello_classification\hello_classification.py public\googlenet-v1\FP16\googlenet-v1.xml car_1.bmp CPU" + + +Running the Image on GPU +++++++++++++++++++++++++ + +.. note:: -> **NOTE**: Since GPU is not supported in prebuilt images or [default Dockerfiles](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles), you must make sure the Additional Requirements for GPU in System Requirements are met, and configure and build the image manually before you can run inferences on a GPU. + Since GPU is not supported in `prebuilt images <#getting-a-prebuilt-image-from-provided-sources>`__ or `default Dockerfiles `__, you must make sure the Additional Requirements for GPU in `System Requirements <#system-requirements>`__ are met, and `configure and build the image manually <#configuring-the-image-for-gpu>`__ before you can run inferences on a GPU. 1. To try inference on a GPU, run the image with the following command: - ```bat - docker run -it --rm -u ContainerAdministrator --isolation process --device class/5B45201D-F2F2-4F3B-85BB-30FF1F953599 -v C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_518f2921ba495409:C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_518f2921ba495409 -v C:\tmp:C:\tmp - ``` + + .. code-block:: bat + + docker run -it --rm -u ContainerAdministrator --isolation process --device class/5B45201D-F2F2-4F3B-85BB-30FF1F953599 -v C:\Windows\System32\DriverStore\FileRepository\iigd_dch. inf_amd64_518f2921ba495409:C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_518f2921ba495409 -v C:\tmp:C:\tmp + + where - - `--device class/5B45201D-F2F2-4F3B-85BB-30FF1F953599` is a reserved interface class GUID for a GPU device. - - `C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_518f2921ba495409` is the path to OpenCL driver home directory. To find it on your PC, run the `C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_*` regular expression. - - `C:\tmp` is the folder with the copy of `OpenCL.dll` from your `C:\Windows\System32` host folder. -2. Copy `OpenCL.dll` to the `C:\Windows\System32` folder inside the container and set appropriate registry entry. Now you can run inference on a GPU device: - ```bat - copy C:\tmp\OpenCL.dll C:\Windows\System32\ && reg add "HKLM\SOFTWARE\Khronos\OpenCL\Vendors" /v "C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_518f2921ba495409\ocl\bin\x64\intelocl64.dll" /t REG_DWORD /d 0 - ``` - For example, run the `Hello Classification Python` sample with the following command: - ```bat - omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -kO https://storage.openvinotoolkit.org/data/test_data/images/car_1.bmp && python samples\python\hello_classification\hello_classification.py public\googlenet-v1\FP16\googlenet-v1.xml car_1.bmp GPU - ``` - - -## Additional Resources - -- [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) for Intel® Distribution of OpenVINO™ toolkit. The Framework can generate a Dockerfile, build, test, and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. You can reuse available Dockerfiles, add your layer and customize the image of OpenVINO™ for your needs. -- Intel® Distribution of OpenVINO™ toolkit home page: [https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit) -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file + + - ``--device class/5B45201D-F2F2-4F3B-85BB-30FF1F953599`` is a reserved interface class GUID for a GPU device. + - ``C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_518f2921ba495409`` is the path to OpenCL driver home directory. To find it on your PC, run the ``C:\Windows\System32\DriverStore\FileRepository\iigd_dch.inf_amd64_*`` regular expression. + - ``C:\tmp`` is the folder with the copy of ``OpenCL.dll`` from your ``C:\Windows\System32`` host folder. + +2. Copy ``OpenCL.dll`` to the ``C:\Windows\System32`` folder inside the container and set appropriate registry entry. Now you can run inference on a GPU device: + + .. code-block:: bat + + copy C:\tmp\OpenCL.dll C:\Windows\System32\ && reg add "HKLM\SOFTWARE\Khronos\OpenCL\Vendors" /v "C:\Windows\System32\DriverStore\FileRepository\iigd_dch. inf_amd64_518f2921ba495409\ocl\bin\x64\intelocl64.dll" /t REG_DWORD /d 0 + + + For example, run the ``Hello Classification Python`` sample with the following command: + + .. code-block:: bat + + omz_downloader --name googlenet-v1 --precisions FP16 && omz_converter --name googlenet-v1 --precision FP16 && curl -kO https://storage.openvinotoolkit.org/data/test_data/images/ car_1.bmp && python samples\python\hello_classification\hello_classification.py public\googlenet-v1\FP16\googlenet-v1.xml car_1.bmp GPU + + +Additional Resources +#################### + +- `DockerHub CI Framework `__ for Intel® Distribution of OpenVINO™ toolkit. The Framework can generate a Dockerfile, build, test, and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. You can reuse available Dockerfiles, add your layer and customize the image of OpenVINO™ for your needs. +- Intel® Distribution of OpenVINO™ toolkit home page: `https://software.intel.com/en-us/openvino-toolkit `__ +- `OpenVINO Installation Selector Tool `__ + +@endsphinxdirective diff --git a/docs/install_guides/installing-openvino-from-archive-linux.md b/docs/install_guides/installing-openvino-from-archive-linux.md index abb899a116d227..ad602dc0802d85 100644 --- a/docs/install_guides/installing-openvino-from-archive-linux.md +++ b/docs/install_guides/installing-openvino-from-archive-linux.md @@ -1,32 +1,36 @@ # Install OpenVINO™ Runtime on Linux from an Archive File {#openvino_docs_install_guides_installing_openvino_from_archive_linux} -With the OpenVINO™ 2022.3 release, you can download and use archive files to install OpenVINO Runtime. The archive files contain pre-built binaries and library files needed for OpenVINO Runtime, as well as code samples. -Installing OpenVINO Runtime from archive files is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. See the [Install OpenVINO from PyPI](installing-openvino-pip.md) page for instructions on how to install OpenVINO Runtime for Python using PyPI. +@sphinxdirective -> **NOTE**: Since the OpenVINO™ 2022.1 release, the following development tools: Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, Accuracy Checker, and Annotation Converter can be installed via [pypi.org](https://pypi.org/project/openvino-dev/) only. +Installing OpenVINO Runtime from archive files is recommended for C++ developers. It will contain code samples, +as well as pre-built binaries and library files needed for OpenVINO Runtime. If you work with Python, +the PyPI package may be a better choice. See the :doc:`Install OpenVINO from PyPI ` +page for instructions on how to install OpenVINO Runtime for Python using PyPI. -See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes) for more information on updates in the latest release. +.. note:: + The following development tools can be installed via `pypi.org `__ only: + Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, + Accuracy Checker, and Annotation Converter. + +See the `Release Notes ` for more information on updates in the latest release. -@sphinxdirective .. tab:: System Requirements | Full requirement listing is available in: - | `System Requirements Page `_ + | `System Requirements Page `__ .. tab:: Processor Notes - Processor graphics are not included in all processors. - See `Product Specifications`_ for information about your processor. - - .. _Product Specifications: https://ark.intel.com/ + | Processor graphics are not included in all processors. + | See `Product Specifications `__ for information about your processor. .. tab:: Software - * `CMake 3.13 or higher, 64-bit `_ - * `Python 3.7 - 3.10, 64-bit `_ + * `CMake 3.13 or higher, 64-bit `__ + * `Python 3.7 - 3.10, 64-bit `__ * GCC: .. tab:: Ubuntu 18.04 @@ -44,7 +48,7 @@ See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNo .. tab:: CENTOS 7 * GCC 8.3.1 - Use folloving instructions to install it: + Use the following instructions to install it: Install GCC 8.3.1 via devtoolset-8 .. code-block:: sh @@ -59,13 +63,13 @@ See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNo source /opt/rh/devtoolset-8/enable gcc -v -@endsphinxdirective -## Installing OpenVINO Runtime -### Step 1: Download and Install the OpenVINO Core Components +Installing OpenVINO Runtime +############################################################ -@sphinxdirective +Step 1: Download and Install the OpenVINO Core Components +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1. Open a command prompt terminal window. You can use the keyboard shortcut: Ctrl+Alt+T @@ -122,47 +126,72 @@ See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNo .. code-block:: sh - cd /opt/intel/openvino_2022.3.0/ + cd /opt/intel/openvino_2022.3.0 sudo -E ./install_dependencies/install_openvino_dependencies.sh 6. For simplicity, it is useful to create a symbolic link as below: .. code-block:: sh + cd /opt/intel sudo ln -s openvino_2022.3.0 openvino_2022 .. note:: - If you have already installed a previous release of OpenVINO 2022, a symbolic link to the ``openvino_2022`` folder may already exist. Unlink the previous link with ``sudo unlink openvino_2022``, and then re-run the command above. + If you have already installed a previous release of OpenVINO 2022, a symbolic link to the ``openvino_2022`` folder may already exist. + Unlink the previous link with ``sudo unlink openvino_2022``, and then re-run the command above. -@endsphinxdirective -Congratulations, you finished the installation! The `/opt/intel/openvino_2022` folder now contains the core components for OpenVINO. If you used a different path in Step 2, for example, `/home//Intel/`, OpenVINO is then installed in `/home//Intel/openvino_2022`. The path to the `openvino_2022` directory is also referred as `` throughout the OpenVINO documentation. +Congratulations, you have finished the installation! The ``/opt/intel/openvino_2022`` folder now contains +the core components for OpenVINO. If you used a different path in Step 2, for example, ``/home//Intel/``, +OpenVINO is now in ``/home//Intel/openvino_2022``. The path to the ``openvino_2022`` directory is +also referred as ```` throughout the OpenVINO documentation. + -### Step 2: Configure the Environment +Step 2: Configure the Environment +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -You must update several environment variables before you can compile and run OpenVINO applications. Open a terminal window and run the `setupvars.sh` script as shown below to temporarily set your environment variables. If your is not `/opt/intel/openvino_2022`, use the correct one instead. +You must update several environment variables before you can compile and run OpenVINO applications. +Open a terminal window and run the ``setupvars.sh`` script as shown below to temporarily set your environment variables. +If your is not ``/opt/intel/openvino_2022``, use the correct one instead. -```sh -source /opt/intel/openvino_2022/setupvars.sh -``` +.. code-block:: sh -If you have more than one OpenVINO version on your machine, you can easily switch its version by sourcing the `setupvars.sh` of your choice. + source /opt/intel/openvino_2022/setupvars.sh -> **NOTE**: The above command must be re-run every time you start a new terminal session. To set up Linux to automatically run the command every time a new terminal is opened, open `~/.bashrc` in your favorite editor and add `source /opt/intel/openvino_2022/setupvars.sh` after the last line. Next time when you open a terminal, you will see `[setupvars.sh] OpenVINO™ environment initialized`. Changing `.bashrc` is not recommended when you have multiple OpenVINO versions on your machine and want to switch among them. + +If you have more than one OpenVINO version installed on your system, you can easily switch versions by sourcing the `setupvars.sh` of your choice. + +.. note:: + + The above command must be re-run every time you start a new terminal session. + To set up Linux to automatically run the command every time a new terminal is opened, + open ``~/.bashrc`` in your favorite editor and add ``source /opt/intel/openvino_2022/setupvars.sh`` after the last line. + Next time when you open a terminal, you will see ``[setupvars.sh] OpenVINO™ environment initialized``. + Changing ``.bashrc`` is not recommended when you have multiple OpenVINO versions on your machine and want to switch among them. The environment variables are set. Continue to the next section if you want to download any additional components. -### Step 3 (Optional): Install Additional Components -OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you install OpenVINO Runtime using archive files, OpenVINO Development Tools must be installed separately. +Step 3 (Optional): Install Additional Components +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. +It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. +If you install OpenVINO Runtime using archive files, OpenVINO Development Tools must be installed separately. -See the [Install OpenVINO Development Tools](installing-model-dev-tools.md) page for step-by-step installation instructions. +See the :doc:`Install OpenVINO Development Tools ` +page for step-by-step installation instructions. -OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the [instructions on GitHub](https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO). +OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their +capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the +`instructions on GitHub `__. + +Step 4 (Optional): Configure Inference on Non-CPU Devices +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +OpenVINO Runtime has a plugin architecture that enables you to run inference on multiple devices +without rewriting your code. Supported devices include integrated GPUs, discrete GPUs and GNAs. +See the instructions below to set up OpenVINO on these devices. -### Step 4 (Optional): Configure Inference on Non-CPU Devices -OpenVINO Runtime has a plugin architecture that enables you to run inference on multiple devices without rewriting your code. Supported devices include integrated GPUs, discrete GPUs and GNAs. See the instructions below to set up OpenVINO on these devices. - -@sphinxdirective .. tab:: GPU To enable the toolkit components to use processor graphics (GPU) on your system, follow the steps in :ref:`GPU Setup Guide `. @@ -171,58 +200,63 @@ OpenVINO Runtime has a plugin architecture that enables you to run inference on To enable the toolkit components to use Intel® Gaussian & Neural Accelerator (GNA) on your system, follow the steps in :ref:`GNA Setup Guide `. -@endsphinxdirective -## What's Next? -Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials. -@sphinxdirective -.. tab:: Get started with Python +What's Next? +############################################################ + +Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! +Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials. + - Try the `Python Quick Start Example `_ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. +.. tab:: Get started with Python + + Try the `Python Quick Start Example `_ + to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. .. image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif :width: 400 - - Visit the :ref:`Tutorials ` page for more Jupyter Notebooks to get you started with OpenVINO, such as: + + Visit the :doc:`Tutorials ` page for more Jupyter Notebooks to get you started with OpenVINO, such as: * `OpenVINO Python API Tutorial `_ * `Basic image classification program with Hello Image Classification `_ * `Convert a PyTorch model and use it for image background removal `_ + .. tab:: Get started with C++ - Try the `C++ Quick Start Example `_ for step-by-step instructions on building and running a basic image classification C++ application. + Try the :doc:`C++ Quick Start Example ` for step-by-step instructions + on building and running a basic image classification C++ application. .. image:: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :width: 400 - Visit the :ref:`Samples ` page for other C++ example applications to get you started with OpenVINO, such as: + Visit the :doc:`Samples ` page for other C++ example applications to get you started with OpenVINO, such as: - * `Basic object detection with the Hello Reshape SSD C++ sample `_ - * `Automatic speech recognition C++ sample `_ + * `Basic object detection with the Hello Reshape SSD C++ sample `__ + * `Automatic speech recognition C++ sample `__ -@endsphinxdirective -## Uninstalling the Intel® Distribution of OpenVINO™ Toolkit -To uninstall the toolkit, follow the steps on the [Uninstalling page](uninstalling-openvino.md). +Uninstalling the Intel® Distribution of OpenVINO™ Toolkit +########################################################### -## Additional Resources +To uninstall the toolkit, follow the steps on the :doc:`Uninstalling page `. -@sphinxdirective - -* :ref:`Troubleshooting Guide for OpenVINO Installation & Configuration ` -* Converting models for use with OpenVINO™: :ref:`Model Optimizer User Guide ` -* Writing your own OpenVINO™ applications: :ref:`OpenVINO™ Runtime User Guide ` -* Sample applications: :ref:`OpenVINO™ Toolkit Samples Overview ` -* Pre-trained deep learning models: :ref:`Overview of OpenVINO™ Toolkit Pre-Trained Models ` -* IoT libraries and code samples in the GitHub repository: `Intel® IoT Developer Kit`_ -.. _Intel® IoT Developer Kit: https://github.com/intel-iot-devkit +Additional Resources +########################################################### -@endsphinxdirective +* :doc:`Troubleshooting Guide for OpenVINO Installation & Configuration ` +* Converting models for use with OpenVINO™: :doc:`Model Optimizer User Guide ` +* Writing your own OpenVINO™ applications: :doc:`OpenVINO™ Runtime User Guide ` +* Sample applications: :doc:`OpenVINO™ Toolkit Samples Overview ` +* Pre-trained deep learning models: :doc:`Overview of OpenVINO™ Toolkit Pre-Trained Models ` +* IoT libraries and code samples in the GitHub repository: `Intel® IoT Developer Kit `__ +* `OpenVINO Installation Selector Tool `__ -## Additional Resources -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file + +@endsphinxdirective + diff --git a/docs/install_guides/installing-openvino-from-archive-macos.md b/docs/install_guides/installing-openvino-from-archive-macos.md index 23d20d6e666130..2e3793bce92e3f 100644 --- a/docs/install_guides/installing-openvino-from-archive-macos.md +++ b/docs/install_guides/installing-openvino-from-archive-macos.md @@ -1,53 +1,59 @@ # Install OpenVINO™ Runtime on macOS from an Archive File {#openvino_docs_install_guides_installing_openvino_from_archive_macos} +@sphinxdirective + With the OpenVINO™ 2022.3 release, you can download and use archive files to install OpenVINO Runtime. The archive files contain pre-built binaries and library files needed for OpenVINO Runtime, as well as code samples. -Installing OpenVINO Runtime from archive files is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the [Install OpenVINO from PyPI](installing-openvino-pip.md) page for instructions on how to install OpenVINO Runtime for Python using PyPI. +Installing OpenVINO Runtime from archive files is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the :doc:`Install OpenVINO from PyPI ` page for instructions on how to install OpenVINO Runtime for Python using PyPI. -See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes) for more information on updates in the latest release. +See the `Release Notes `__ for more information on updates in the latest release. -> **NOTE**: Since the OpenVINO™ 2022.1 release, the following development tools: Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, Accuracy Checker, and Annotation Converter can be installed via [pypi.org](https://pypi.org/project/openvino-dev/) only. +.. note:: + + Since the OpenVINO™ 2022.1 release, the following development tools: Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, Accuracy Checker, and Annotation Converter can be installed via `pypi.org `__ only. -@sphinxdirective .. tab:: System Requirements | Full requirement listing is available in: - | `System Requirements Page `_ + | `System Requirements Page `__ .. tab:: Software Requirements - * `CMake 3.13 or higher `_ (choose "macOS 10.13 or later"). Add `/Applications/CMake.app/Contents/bin` to path (for default install). - * `Python 3.7 - 3.10 `_ (choose 3.7 - 3.10). Install and add to path. - * Apple Xcode Command Line Tools. In the terminal, run `xcode-select --install` from any directory + * `CMake 3.13 or higher `__ (choose "macOS 10.13 or later"). Add ``/Applications/CMake.app/Contents/bin`` to path (for default install). + * `Python 3.7 - 3.10 `__ (choose 3.7 - 3.10). Install and add to path. + * Apple Xcode Command Line Tools. In the terminal, run ``xcode-select --install`` from any directory * (Optional) Apple Xcode IDE (not required for OpenVINO™, but useful for development) -@endsphinxdirective -## Installing OpenVINO Runtime +Installing OpenVINO Runtime +########################### -### Step 1: Install OpenVINO Core Components +Step 1: Install OpenVINO Core Components +++++++++++++++++++++++++++++++++++++++++ -@sphinxdirective -1. Open a command prompt terminal window. -2. Create the `/opt/intel` folder for OpenVINO by using the following command. If the folder already exists, skip this command. +1. Open a command prompt terminal window. +2. Create the ``/opt/intel`` folder for OpenVINO by using the following command. If the folder already exists, skip this command. .. code-block:: sh sudo mkdir /opt/intel - - .. note:: - - The `/opt/intel` path is the recommended folder path for installing OpenVINO. You may use a different path if desired. -3. Browse to the current user's `Downloads` folder: + + .. note:: + + The ``/opt/intel`` path is the recommended folder path for installing OpenVINO. You may use a different path if desired. + + +3. Browse to the current user's ``Downloads`` folder: .. code-block:: sh cd /Downloads - -4. Download the `OpenVINO Runtime archive file for macOS `_, extract the files, rename the extracted folder and move it to the desired path: + + +4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: .. tab:: x86, 64-bit @@ -65,55 +71,62 @@ See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNo tar -xf openvino_2022.3.0.tgz sudo mv m_openvino_toolkit_macos_11_0_2022.3.0.9052.9752fafe8eb_arm64 /opt/intel/openvino_2022.3.0 + 5. For simplicity, it is useful to create a symbolic link as below: .. code-block:: sh sudo ln -s openvino_2022.3.0 openvino_2022 - - .. note:: - - If you have already installed a previous release of OpenVINO 2022, a symbolic link to the `openvino_2022` folder may already exist. Unlink the previous link with `sudo unlink openvino_2022`, and then re-run the command above. -@endsphinxdirective + .. note:: + + If you have already installed a previous release of OpenVINO 2022, a symbolic link to the ``openvino_2022`` folder may already exist. Unlink the previous link with ``sudo unlink openvino_2022``, and then re-run the command above. + -Congratulations, you finished the installation! The `/opt/intel/openvino_2022` folder now contains the core components for OpenVINO. If you used a different path in Step 2, you will find the `openvino_2022` folder there. The path to the `openvino_2022` directory is also referred as `` throughout the OpenVINO documentation. +Congratulations, you finished the installation! The ``/opt/intel/openvino_2022`` folder now contains the core components for OpenVINO. If you used a different path in Step 2, you will find the ``openvino_2022`` folder there. The path to the ``openvino_2022`` directory is also referred as ```` throughout the OpenVINO documentation. -### Step 2: Configure the Environment +Step 2: Configure the Environment ++++++++++++++++++++++++++++++++++ -You must update several environment variables before you can compile and run OpenVINO applications. Open a terminal window and run the `setupvars.sh` script as shown below to temporarily set your environment variables. If your is not `/opt/intel/openvino_2022`, use the correct one instead. +You must update several environment variables before you can compile and run OpenVINO applications. Open a terminal window and run the ``setupvars.sh`` script as shown below to temporarily set your environment variables. If your ```` is not ``/opt/intel/openvino_2022``, use the correct one instead. -```sh -source /opt/intel/openvino_2022/setupvars.sh -``` +.. code-block:: sh -If you have more than one OpenVINO™ version on your machine, you can easily switch its version by sourcing the `setupvars.sh` of your choice. + source /opt/intel/openvino_2022/setupvars.sh -> **NOTE**: The above command must be re-run every time you start a new terminal session. To set up macOS to automatically run the command every time a new terminal is opened, open `~/.zshrc` in your favorite editor and add `source /opt/intel/openvino_2022/setupvars.sh` after the last line. Next time when you open a terminal, you will see `[setupvars.sh] OpenVINO™ environment initialized`. Changing `~/.zshrc` is not recommended when you have multiple OpenVINO versions on your machine and want to switch among them. + +If you have more than one OpenVINO™ version on your machine, you can easily switch its version by sourcing the ``setupvars.sh`` of your choice. + +.. note:: + + The above command must be re-run every time you start a new terminal session. To set up macOS to automatically run the command every time a new terminal is opened, open ``~/.zshrc`` in your favorite editor and add ``source /opt/intel/openvino_2022/setupvars.sh`` after the last line. Next time when you open a terminal, you will see ``[setupvars.sh] OpenVINO™ environment initialized``. Changing ``~/.zshrc`` is not recommended when you have multiple OpenVINO versions on your machine and want to switch among them. The environment variables are set. Continue to the next section if you want to download any additional components. -### Step 3 (Optional): Install Additional Components +Step 3 (Optional): Install Additional Components +++++++++++++++++++++++++++++++++++++++++++++++++ OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you install OpenVINO Runtime using archive files, OpenVINO Development Tools must be installed separately. -See the [Install OpenVINO Development Tools](installing-model-dev-tools.md) page for step-by-step installation instructions. +See the :doc:`Install OpenVINO Development Tools ` page for step-by-step installation instructions. + +OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the `instructions on GitHub `__. -OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the [instructions on GitHub](https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO). +What's Next? +#################### -## What's Next? Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials. -@sphinxdirective + .. tab:: Get started with Python Try the `Python Quick Start Example `_ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. - + .. image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif :width: 400 Visit the :ref:`Tutorials ` page for more Jupyter Notebooks to get you started with OpenVINO, such as: - + * `OpenVINO Python API Tutorial `_ * `Basic image classification program with Hello Image Classification `_ * `Convert a PyTorch model and use it for image background removal `_ @@ -121,44 +134,38 @@ Now that you've installed OpenVINO Runtime, you're ready to run your own machine .. tab:: Get started with C++ Try the `C++ Quick Start Example `_ for step-by-step instructions on building and running a basic image classification C++ application. - + .. image:: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :width: 400 Visit the :ref:`Samples ` page for other C++ example applications to get you started with OpenVINO, such as: - + * `Basic object detection with the Hello Reshape SSD C++ sample `_ * `Automatic speech recognition C++ sample `_ -@endsphinxdirective - -## Uninstalling Intel® Distribution of OpenVINO™ Toolkit +Uninstalling Intel® Distribution of OpenVINO™ Toolkit +##################################################### -To uninstall the toolkit, follow the steps on the [Uninstalling page](uninstalling-openvino.md). +To uninstall the toolkit, follow the steps on the :doc:`Uninstalling page `. -## Additional Resources - -@sphinxdirective +Additional Resources +#################### +* `OpenVINO Installation Selector Tool `__ * :ref:`Troubleshooting Guide for OpenVINO Installation & Configuration ` * Converting models for use with OpenVINO™: :ref:`Model Optimizer User Guide ` * Writing your own OpenVINO™ applications: :ref:`OpenVINO™ Runtime User Guide ` * Sample applications: :ref:`OpenVINO™ Toolkit Samples Overview ` * Pre-trained deep learning models: :ref:`Overview of OpenVINO™ Toolkit Pre-Trained Models ` -* IoT libraries and code samples in the GitHUB repository: `Intel® IoT Developer Kit`_ +* IoT libraries and code samples in the GitHUB repository: `Intel® IoT Developer Kit `__ -.. _Intel® IoT Developer Kit: https://github.com/intel-iot-devkit +---> @endsphinxdirective - -## Additional Resources - -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file diff --git a/docs/install_guides/installing-openvino-from-archive-windows.md b/docs/install_guides/installing-openvino-from-archive-windows.md index 2300a1dceb8423..a0bc2b7649c94e 100644 --- a/docs/install_guides/installing-openvino-from-archive-windows.md +++ b/docs/install_guides/installing-openvino-from-archive-windows.md @@ -1,112 +1,158 @@ # Install OpenVINO™ Runtime on Windows from an Archive File {#openvino_docs_install_guides_installing_openvino_from_archive_windows} -With the OpenVINO™ 2022.3 release, you can download and use archive files to install OpenVINO Runtime. The archive files contain pre-built binaries and library files needed for OpenVINO Runtime, as well as code samples. +@sphinxdirective -Installing OpenVINO Runtime from archive files is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. See the [Install OpenVINO from PyPI](installing-openvino-pip.md) page for instructions on how to install OpenVINO Runtime for Python using PyPI. +With the OpenVINO™ 2022.3 release, you can download and use archive files to install OpenVINO Runtime. The archive files contain pre-built binaries and library files needed for OpenVINO Runtime, as well as code samples. -> **NOTE**: Since the OpenVINO™ 2022.1 release, the following development tools: Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, Accuracy Checker, and Annotation Converter can be installed via [pypi.org](https://pypi.org/project/openvino-dev/) only. +Installing OpenVINO Runtime from archive files is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. See the :doc:`Install OpenVINO from PyPI ` page for instructions on how to install OpenVINO Runtime for Python using PyPI. -See the [Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes) for more information on updates in the latest release. +.. note:: -## System Requirements + Since the OpenVINO™ 2022.1 release, the following development tools: Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, Accuracy Checker, and Annotation Converter can be installed via `pypi.org `__ only. + + +See the `Release Notes `__ for more information on updates in the latest release. + +System Requirements +#################### -@sphinxdirective .. tab:: System Requirements | Full requirement listing is available in: | `System Requirements Page `_ - + .. tab:: Processor Notes - Processor graphics are not included in all processors. - See `Product Specifications`_ for information about your processor. - - .. _Product Specifications: https://ark.intel.com/ + Processor graphics are not included in all processors. + See `Product Specifications`_ for information about your processor. + + .. _Product Specifications: https://ark.intel.com/ .. tab:: Software - * `Microsoft Visual Studio 2019 with MSBuild `_ or `Microsoft Visual Studio 2022 `_ - * `CMake 3.14 or higher, 64-bit `_ (optional, only required for building sample applications) - * `Python 3.7 - 3.10, 64-bit `_ + * `Microsoft Visual Studio 2019 with MSBuild `_ or `Microsoft Visual Studio 2022 `_ + * `CMake 3.14 or higher, 64-bit `_ (optional, only required for building sample applications) + * `Python 3.7 - 3.10, 64-bit `_ - .. note:: - To install Microsoft Visual Studio 2019, follow the `Microsoft Visual Studio installation guide `_. You can choose to download the Community version. During installation in the **Workloads** tab, choose **Desktop development with C++**. + .. note:: - .. note:: - You can either use `cmake.msi` which is the installation wizard or `cmake.zip` where you have to go into the `bin` folder and then manually add the path to environmental variables. - - .. important:: - When installing Python, make sure you click the option **Add Python 3.x to PATH** to `add Python `_ to your `PATH` environment variable. + To install Microsoft Visual Studio 2019, follow the `Microsoft Visual Studio installation guide `_. You can choose to download the Community version. During installation in the **Workloads** tab, choose **Desktop development with C++**. -@endsphinxdirective + .. note:: -## Installing OpenVINO Runtime + You can either use `cmake.msi` which is the installation wizard or `cmake.zip` where you have to go into the `bin` folder and then manually add the path to environmental variables. -### Step 1: Download and Install OpenVINO Core Components + .. important:: + + When installing Python, make sure you click the option **Add Python 3.x to PATH** to `add Python `_ to your `PATH` environment variable. + + + +Installing OpenVINO Runtime +########################### + +.. _install-openvino-archive-windows: + +Step 1: Download and Install OpenVINO Core Components ++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +1. Create an ``Intel`` folder in the ``C:\Program Files (x86)\`` directory. Skip this step if the folder already exists. -1. Create an `Intel` folder in the `C:\Program Files (x86)\` directory. Skip this step if the folder already exists. - You can also do this via command-lines. Open a new command prompt window as administrator by right-clicking **Command Prompt** from the Start menu and select **Run as administrator**, and then run the following command: - ```sh - mkdir "C:\Program Files (x86)\Intel" - ``` - > **NOTE**: `C:\Program Files (x86)\Intel` is the recommended folder. You may also use a different path if desired or if you don't have administrator privileges on your computer. -2. Download the [OpenVINO Runtime archive file for Windows](https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/windows/) to your local `Downloads` folder. - + .. code-block:: sh + + mkdir "C:\Program Files (x86)\Intel" + + + .. note:: + + ``C:\Program Files (x86)\Intel`` is the recommended folder. You may also use a different path if desired or if you don't have administrator privileges on your computer. + + +2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. + If you prefer using command-lines, run the following commands in the command prompt window you opened: - ```sh - cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/windows/w_openvino_toolkit_windows_2022.3.0.9052.9752fafe8eb_x86_64.zip --output openvino_2022.3.0.zip - ``` - > **NOTE**: A `.sha256` file is provided together with the archive file to validate your download process. To do that, download the `.sha256` file from the same repository and run `CertUtil -hashfile openvino_2022.3.0.zip SHA256`. Compare the returned value in the output with what's in the `.sha256` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket [here](https://www.intel.com/content/www/us/en/support/contact-intel.html). + + .. code-block:: sh + + cd /Downloads + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/windows/w_openvino_toolkit_windows_2022.3.0.9052.9752fafe8eb_x86_64.zip --output openvino_2022. 3.0.zip -3. Use your favorite tool to extract the archive file, rename the extracted folder, and move it to the `C:\Program Files (x86)\Intel` directory. - + .. note:: + + A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2022.3.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. + + +3. Use your favorite tool to extract the archive file, rename the extracted folder, and move it to the ``C:\Program Files (x86)\Intel`` directory. + To do this step using command-lines, run the following commands in the command prompt window you opened: - ```sh - tar -xf openvino_2022.3.0.zip - ren w_openvino_toolkit_windows_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 - move openvino_2022.3.0 "C:\Program Files (x86)\Intel" - ``` + + .. code-block:: sh + + tar -xf openvino_2022.3.0.zip + ren w_openvino_toolkit_windows_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 + move openvino_2022.3.0 "C:\Program Files (x86)\Intel" + 4. For simplicity, it is useful to create a symbolic link. Open a command prompt window as administrator (see Step 1 for how to do this) and run the following commands: - ```sh - cd C:\Program Files (x86)\Intel - mklink /D openvino_2022 openvino_2022.3.0 - ``` - > **NOTE**: If you have already installed a previous release of OpenVINO 2022, a symbolic link to the `openvino_2022` folder may already exist. If you want to override it, nagivate to the `C:\Program Files (x86)\Intel` folder and delete the existing linked folder before running the `mklink` command. + .. code-block:: sh + + cd C:\Program Files (x86)\Intel + mklink /D openvino_2022 openvino_2022.3.0 -Congratulations, you finished the installation! The `C:\Program Files (x86)\Intel\openvino_2022` folder now contains the core components for OpenVINO. If you used a different path in Step 1, you will find the `openvino_2022` folder there. The path to the `openvino_2022` directory is also referred as `` throughout the OpenVINO documentation. -### Step 2: Configure the Environment + .. note:: -You must update several environment variables before you can compile and run OpenVINO™ applications. Open the Command Prompt, and run the `setupvars.bat` batch file to temporarily set your environment variables. If your is not `C:\Program Files (x86)\Intel\openvino_2022`, use the correct directory instead. + If you have already installed a previous release of OpenVINO 2022, a symbolic link to the ``openvino_2022`` folder may already exist. If you want to override it, nagivate to the ``C:\Program Files (x86)\Intel`` folder and delete the existing linked folder before running the ``mklink`` command. -```sh -"C:\Program Files (x86)\Intel\openvino_2022\setupvars.bat" -``` -> **Important**: The above command must be re-run every time a new Command Prompt window is opened. +Congratulations, you finished the installation! The ``C:\Program Files (x86)\Intel\openvino_2022`` folder now contains the core components for OpenVINO. If you used a different path in Step 1, you will find the ``openvino_2022`` folder there. The path to the ``openvino_2022`` directory is also referred as ```` throughout the OpenVINO documentation. + +.. _set-the-environment-variables-windows: + +Step 2: Configure the Environment ++++++++++++++++++++++++++++++++++ + +You must update several environment variables before you can compile and run OpenVINO™ applications. Open the Command Prompt, and run the ``setupvars.bat`` batch file to temporarily set your environment variables. If your ```` is not ``C:\Program Files (x86)\Intel\openvino_2022``, use the correct directory instead. + +.. code-block: sh + + "C:\Program Files (x86)\Intel\openvino_2022\setupvars.bat" + + +.. important:: + + The above command must be re-run every time a new Command Prompt window is opened. + + +.. note:: + + If you see an error indicating Python is not installed, Python may not be added to the PATH environment variable (as described `here `__). Check your system environment variables, and add Python if necessary. -> **NOTE**: If you see an error indicating Python is not installed, Python may not be added to the PATH environment variable (as described [here](https://docs.python.org/3/using/windows.html#finding-the-python-executable)). Check your system environment variables, and add Python if necessary. The environment variables are set. Continue to the next section if you want to download any additional components. -### Step 3 (Optional): Install Additional Components +.. _model-optimizer-windows: + +Step 3 (Optional): Install Additional Components +++++++++++++++++++++++++++++++++++++++++++++++++ OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you install OpenVINO Runtime using archive files, OpenVINO Development Tools must be installed separately. -See the [Install OpenVINO Development Tools](installing-model-dev-tools.md) page for step-by-step installation instructions. +See the :doc:`Install OpenVINO Development Tools ` page for step-by-step installation instructions. -OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the [instructions on GitHub](https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO). +OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the `instructions on GitHub `. + +.. _optional-steps-windows: + +Step 4 (Optional): Configure Inference on non-CPU Devices ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -### Step 4 (Optional): Configure Inference on non-CPU Devices OpenVINO Runtime has a plugin architecture that enables you to run inference on multiple devices without rewriting your code. Supported devices include integrated GPUs, discrete GPUs and GNAs. See the instructions below to set up OpenVINO on these devices. -@sphinxdirective .. tab:: GPU To enable the toolkit components to use processor graphics (GPU) on your system, follow the steps in :ref:`GPU Setup Guide `. @@ -114,69 +160,67 @@ OpenVINO Runtime has a plugin architecture that enables you to run inference on .. tab:: GNA To enable the toolkit components to use Intel® Gaussian & Neural Accelerator (GNA) on your system, follow the steps in :ref:`GNA Setup Guide `. - -@endsphinxdirective -## What's Next? + +.. _get-started-windows: + +What's Next? +#################### + Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials. -@sphinxdirective + .. tab:: Get started with Python - Try the `Python Quick Start Example `_ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. - + Try the `Python Quick Start Example `__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. + .. image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif :width: 400 Visit the :ref:`Tutorials ` page for more Jupyter Notebooks to get you started with OpenVINO, such as: - - * `OpenVINO Python API Tutorial `_ - * `Basic image classification program with Hello Image Classification `_ - * `Convert a PyTorch model and use it for image background removal `_ + + * `OpenVINO Python API Tutorial `__ + * `Basic image classification program with Hello Image Classification `__ + * `Convert a PyTorch model and use it for image background removal `__ .. tab:: Get started with C++ Try the `C++ Quick Start Example `_ for step-by-step instructions on building and running a basic image classification C++ application. - + .. image:: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :width: 400 Visit the :ref:`Samples ` page for other C++ example applications to get you started with OpenVINO, such as: - + * `Basic object detection with the Hello Reshape SSD C++ sample `_ * `Automatic speech recognition C++ sample `_ -@endsphinxdirective - -## Uninstalling OpenVINO Runtime -To uninstall OpenVINO, follow the steps on the [Uninstalling page](uninstalling-openvino.md). +.. _uninstall-from-windows: -## Additional Resources +Uninstalling OpenVINO Runtime +############################# -## Additional Resources +To uninstall OpenVINO, follow the steps on the :doc:`Uninstalling page ` -@sphinxdirective +Additional Resources +#################### +* `OpenVINO Installation Selector Tool `__ * :ref:`Troubleshooting Guide for OpenVINO Installation & Configuration ` * Converting models for use with OpenVINO™: :ref:`Model Optimizer Developer Guide ` * Writing your own OpenVINO™ applications: :ref:`OpenVINO™ Runtime User Guide ` * Sample applications: :ref:`OpenVINO™ Toolkit Samples Overview ` * Pre-trained deep learning models: :ref:`Overview of OpenVINO™ Toolkit Pre-Trained Models ` -* IoT libraries and code samples in the GitHUB repository: `Intel® IoT Developer Kit`_ - - -.. _Intel® IoT Developer Kit: https://github.com/intel-iot-devkit +---> @endsphinxdirective - -## Additional Resources - -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file diff --git a/docs/install_guides/installing-openvino-linux-header.md b/docs/install_guides/installing-openvino-linux-header.md index b415caf8e96e67..f45cbb2a5010a0 100644 --- a/docs/install_guides/installing-openvino-linux-header.md +++ b/docs/install_guides/installing-openvino-linux-header.md @@ -13,14 +13,19 @@ Using HomeBrew Using Docker -@endsphinxdirective -If you want to install OpenVINO™ Runtime on your Linux machine, there are a few ways to accomplish this. We prepared the following options for you: +If you want to install OpenVINO™ Runtime on your Linux machine, these are your options: + +* `Install OpenVINO Runtime from an Archive File ` +* `Install OpenVINO from PyPI ` +* `Install OpenVINO Runtime from APT ` +* `Install OpenVINO Runtime from YUM ` +* `Install OpenVINO Runtime using HomeBrew ` +* `Install OpenVINO with Docker Using Docker ` + +For a full selection of distribution channels, see the +`OpenVINO Installation Selector Tool `__ -* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-linux.md) -* [Install OpenVINO from PyPI](installing-openvino-pip.md) -* [Install OpenVINO Runtime from APT](installing-openvino-apt.md) -* [Install OpenVINO Runtime from YUM](installing-openvino-yum.md) -* [Install OpenVINO with Docker](installing-openvino-docker-linux.md) -For a full selection of distribution channels, see the [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) +@endsphinxdirective + diff --git a/docs/install_guides/installing-openvino-macos-header.md b/docs/install_guides/installing-openvino-macos-header.md index 7054d0c6e0dbc3..69b1e93df437c3 100644 --- a/docs/install_guides/installing-openvino-macos-header.md +++ b/docs/install_guides/installing-openvino-macos-header.md @@ -10,12 +10,13 @@ From PyPI Using HomeBrew -@endsphinxdirective +If you want to install OpenVINO™ Runtime on macOS, there are a few ways to accomplish this. We prepared following options for you: -If you want to install OpenVINO™ Runtime on macOS, there are a few ways to accomplish this. We prepared following options for you: +* :doc:`Install OpenVINO Runtime from an Archive File ` +* :doc:`Install OpenVINO Runtime via HomeBrew ` +* :doc:`Install OpenVINO from PyPI ` -* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-macos.md) -* [Install OpenVINO Runtime via HomeBrew](installing-openvino-brew.md) -* [Install OpenVINO from PyPI](installing-openvino-pip.md) +For a full selection of distribution channels, +see the `OpenVINO Installation Selector Tool `__ -For a full selection of distribution channels, see the [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) +@endsphinxdirective \ No newline at end of file diff --git a/docs/install_guides/installing-openvino-macos.md b/docs/install_guides/installing-openvino-macos.md index 9a98f9335e1ded..21c5053e082821 100644 --- a/docs/install_guides/installing-openvino-macos.md +++ b/docs/install_guides/installing-openvino-macos.md @@ -1,12 +1,16 @@ # Install OpenVINO™ Runtime for macOS from Installer +@sphinxdirective + Currently only the following ways are provided to install OpenVINO™: -* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-macos.md) -* [Install OpenVINO Runtime via HomeBrew](installing-openvino-brew.md) -* [Install OpenVINO from PyPI](installing-openvino-pip.md) -* [Build From Source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md) +* :doc:`Install OpenVINO Runtime from an Archive File ` +* :doc:`Install OpenVINO Runtime via HomeBrew ` +* :doc:`Install OpenVINO from PyPI ` +* `Build From Source `__ The other installation methods are temporarily unavailable. For a full selection of distribution channels, see the [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) + +@endsphinxdirective \ No newline at end of file diff --git a/docs/install_guides/installing-openvino-pip.md b/docs/install_guides/installing-openvino-pip.md index 17e1b944a02be7..21b7778a6319f0 100644 --- a/docs/install_guides/installing-openvino-pip.md +++ b/docs/install_guides/installing-openvino-pip.md @@ -2,11 +2,12 @@ @sphinxdirective -You can install both OpenVINO™ Runtime and OpenVINO Development Tools through the PyPI repository. This page provides the main steps for installing OpenVINO Runtime. +Using the PyPI repository, you can install either OpenVINO™ Runtime or OpenVINO Development Tools on Windows, Linux, and macOS systems. +This article focuses on OpenVINO™ Runtime. -.. note: +.. note - From the 2022.1 release, the OpenVINO™ Development Tools can only be installed via PyPI. See :doc:`Install OpenVINO Development Tools ` for detailed steps. + If you install OpenVINO Development Tools, OpenVINO Runtime will also be installed as a dependency, so you don't need to install it separately. Installing OpenVINO Runtime @@ -95,18 +96,8 @@ Run the command below: python -c "from openvino.runtime import Core" -If installation was successful, you will not see any error messages (no console output). +If installation was successful, you will not see any error messages (no console output). Congratulations! You have finished installing OpenVINO Runtime. -Congratulations! You finished installing OpenVINO Runtime. Now you can start exploring OpenVINO's functionality through Jupyter Notebooks and sample applications. See the :ref:`What's Next ` section to learn more! - -Installing OpenVINO Development Tools -##################################### - -OpenVINO Development Tools adds even more functionality to OpenVINO. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you install OpenVINO Development Tools, OpenVINO Runtime will also be installed as a dependency, so you don't need to install OpenVINO Runtime separately. - -See the :doc:`Install OpenVINO Development Tools ` page for step-by-step installation instructions. - -.. _whats-next: What's Next? #################### diff --git a/docs/install_guides/installing-openvino-raspbian.md b/docs/install_guides/installing-openvino-raspbian.md index 8e87595aca6022..d47cea800092e6 100644 --- a/docs/install_guides/installing-openvino-raspbian.md +++ b/docs/install_guides/installing-openvino-raspbian.md @@ -8,12 +8,8 @@ * These steps have been validated with Raspberry Pi 3. * There is also an open-source version of OpenVINO™ that can be compiled for arch64 (see `build instructions `_). -@endsphinxdirective - - -## Development and Target Systems - -@sphinxdirective +Development and Target Systems +############################### .. tab:: System Requirements @@ -28,12 +24,8 @@ .. _install-openvino: -@endsphinxdirective - - -## Step 1: Download and Install OpenVINO Runtime - -@sphinxdirective +Step 1: Download and Install OpenVINO Runtime +############################################# #. Open the Terminal or your preferred console application. #. Create an installation folder for OpenVINO. If the folder already exists, skip this step. @@ -99,12 +91,8 @@ Congratulations, you finished the installation! The ``/opt/intel/openvino_2022`` .. _install-external-dependencies: -@endsphinxdirective - - -## Step 2: Install External Software Dependencies - -@sphinxdirective +Step 2: Install External Software Dependencies +############################################## CMake version 3.10 or higher is required for building the OpenVINO™ toolkit sample application. To install, open a Terminal window and run the following command: @@ -117,12 +105,8 @@ CMake is installed. Continue to the next section to set the environment variable .. _set-the-environment-variables-raspbian: -@endsphinxdirective - - -## Step 3: Set the Environment Variables - -@sphinxdirective +Step 3: Set the Environment Variables +##################################### You must update several environment variables before you can compile and run OpenVINO applications. Open a terminal window and run the ``setupvars.sh`` script as shown below to temporarily set your environment variables. If your is not ``/opt/intel/openvino_2022``, use the correct one instead. @@ -141,12 +125,8 @@ The environment variables are set. Continue to the next section if you want to d .. _model-optimizer: -@endsphinxdirective - - -## Step 4 (Optional): Install Additional Components - -@sphinxdirective +Step 4 (Optional): Install Additional Components +################################################ If you want to use your model for inference, the model must be converted to the ``.bin`` and ``.xml`` Intermediate Representation (IR) files that are used as input by OpenVINO Runtime. To get the optimized models, you can use one of the following options: @@ -158,16 +138,11 @@ If you want to use your model for inference, the model must be converted to the * OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. See the :doc:`Install OpenVINO Development Tools ` page for step-by-step installation instructions. -@endsphinxdirective - - -## What's Next? - -@sphinxdirective +What's Next? +#################### Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials. - .. tab:: Get started with Python Try the `Python Quick Start Example `_ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. @@ -181,6 +156,7 @@ Now that you've installed OpenVINO Runtime, you're ready to run your own machine * `Basic image classification program with Hello Image Classification `_ * `Convert a PyTorch model and use it for image background removal `_ + .. tab:: Get started with C++ Try the `C++ Quick Start Example `_ for step-by-step instructions on building and running a basic image classification C++ application. @@ -193,24 +169,17 @@ Now that you've installed OpenVINO Runtime, you're ready to run your own machine * `Basic object detection with the Hello Reshape SSD C++ sample `_ * `Automatic speech recognition C++ sample `_ - To uninstall the toolkit, follow the steps on the :doc:`Uninstalling page `. -@endsphinxdirective - - -## Additional Resources - -@sphinxdirective +Additional Resources +#################### * :ref:`Troubleshooting Guide for OpenVINO Installation & Configuration ` * Converting models for use with OpenVINO™: :ref:`Model Optimizer User Guide ` * Writing your own OpenVINO™ applications: :ref:`OpenVINO™ Runtime User Guide ` * Sample applications: :ref:`OpenVINO™ Toolkit Samples Overview ` * Pre-trained deep learning models: :ref:`Overview of OpenVINO™ Toolkit Pre-Trained Models ` -* IoT libraries and code samples in the GitHUB repository: `Intel® IoT Developer Kit`_ +* IoT libraries and code samples in the GitHUB repository: `Intel® IoT Developer Kit `__ * :ref:`OpenVINO Installation Selector Tool ` -.. _Intel® IoT Developer Kit: https://github.com/intel-iot-devkit - -@endsphinxdirective \ No newline at end of file +@endsphinxdirective diff --git a/docs/install_guides/installing-openvino-runtime.md b/docs/install_guides/installing-openvino-runtime.md index a3e26d60cdbccd..02f2b4f17a0fbb 100644 --- a/docs/install_guides/installing-openvino-runtime.md +++ b/docs/install_guides/installing-openvino-runtime.md @@ -11,13 +11,16 @@ macOS Raspbian OS -@endsphinxdirective You can choose how to install OpenVINO™ Runtime according to your operating system: -* [Install OpenVINO Runtime on Linux](installing-openvino-linux-header.md) -* [Install OpenVINO Runtime on Windows](installing-openvino-windows-header.md) -* [Install OpenVINO Runtime on macOS](installing-openvino-macos-header.md) -* [Install OpenVINO Runtime on Raspbian OS](installing-openvino-raspbian.md) +* `Install OpenVINO Runtime on Linux ` +* `Install OpenVINO Runtime on Windows ` +* `Install OpenVINO Runtime on macOS ` +* `Install OpenVINO Runtime on Raspbian OS ` Enjoy your journey with OpenVINO. + + +@endsphinxdirective + diff --git a/docs/install_guides/installing-openvino-windows-header.md b/docs/install_guides/installing-openvino-windows-header.md index 2b0489fd384379..f5480a6a350f3a 100644 --- a/docs/install_guides/installing-openvino-windows-header.md +++ b/docs/install_guides/installing-openvino-windows-header.md @@ -10,12 +10,14 @@ From PyPI Using Docker -@endsphinxdirective -If you want to install OpenVINO™ Runtime on Windows, you have the following options: +If you want to install OpenVINO™ Runtime on Windows, you have the following options: + +* :doc:`Install OpenVINO Runtime from an Archive File ` +* :doc:`Install OpenVINO from PyPI ` +* :doc:`Install OpenVINO with Docker ` -* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-windows.md) -* [Install OpenVINO from PyPI](installing-openvino-pip.md) -* [Install OpenVINO with Docker](installing-openvino-docker-windows.md) +For a full selection of distribution channels, +see the `OpenVINO Installation Selector Tool `__ -For a full selection of distribution channels, see the [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) +@endsphinxdirective diff --git a/docs/install_guides/installing-openvino-yocto.md b/docs/install_guides/installing-openvino-yocto.md index 2aa69d56d3db52..479d03f2e87c49 100644 --- a/docs/install_guides/installing-openvino-yocto.md +++ b/docs/install_guides/installing-openvino-yocto.md @@ -116,7 +116,7 @@ If the image build is successful, it will return the list of packages as below: Additional Resources #################### -- :ref:`Troubleshooting Guide ` +- :ref:`Troubleshooting Guide ` - `Yocto Project `__ - official documentation webpage - `BitBake Tool `__ - `Poky `__ diff --git a/docs/install_guides/installing-openvino-yum.md b/docs/install_guides/installing-openvino-yum.md index 80455db5d04060..fc970033171129 100644 --- a/docs/install_guides/installing-openvino-yum.md +++ b/docs/install_guides/installing-openvino-yum.md @@ -1,26 +1,35 @@ # Install OpenVINO™ Runtime on Linux From YUM Repository {#openvino_docs_install_guides_installing_openvino_yum} + + @sphinxdirective -With the OpenVINO™ 2022.3 release, you can install OpenVINO Runtime on Linux using the YUM repository. OpenVINO™ Development Tools can be installed via PyPI only. See :ref:`Installing Additional Components ` for more information. +With the OpenVINO™ 2022.3 release, you can install OpenVINO Runtime on Linux using the YUM repository. +OpenVINO™ Development Tools can be installed via PyPI only. See +`Installing Additional Components `__ for more information. -See the `Release Notes `_ for more information on updates in the latest release. +See the `Release Notes `__ +for more information on updates in the latest release. -Installing OpenVINO Runtime from YUM is recommended for C++ developers. If you are working with Python, the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the :doc:`Install OpenVINO from PyPI ` page for instructions on how to install OpenVINO Runtime for Python using PyPI. +Installing OpenVINO Runtime from YUM is recommended for C++ developers. If you are working with Python, +the PyPI package has everything needed for Python development and deployment on CPU and GPUs. Visit the +:doc:`Install OpenVINO from PyPI ` +page for instructions on how to install OpenVINO Runtime for Python using PyPI. .. warning:: - By downloading and using this container and the included software, you agree to the terms and conditions of the `software license agreements `_. + By downloading and using this container and the included software, you agree to the terms and conditions of the + `software license agreements `__. -@endsphinxdirective -## Prerequisites +Prerequisites +############# + -@sphinxdirective .. tab:: System Requirements | Full requirement listing is available in: - | `System Requirements Page `_ + | `System Requirements Page `__ .. note:: @@ -29,9 +38,7 @@ Installing OpenVINO Runtime from YUM is recommended for C++ developers. If you a .. tab:: Processor Notes Processor graphics are not included in all processors. - See `Product Specifications`_ for information about your processor. - - .. _Product Specifications: https://ark.intel.com/ + See `Product Specifications `__ for information about your processor. .. tab:: Software @@ -39,15 +46,15 @@ Installing OpenVINO Runtime from YUM is recommended for C++ developers. If you a * GCC 8.2.0 * `Python 3.7 - 3.10, 64-bit `_ -@endsphinxdirective -## Install OpenVINO Runtime +Install OpenVINO Runtime +######################## -### Step 1: Set Up the Repository +Step 1: Set Up the Repository ++++++++++++++++++++++++++++++ -@sphinxdirective -1. Create a YUM repository file (`openvino-2022.repo`) in the `/tmp` directory as a normal user: +1. Create a YUM repository file (``openvino-2022.repo``) in the ``/tmp`` directory as a normal user: .. code-block:: sh @@ -61,7 +68,7 @@ Installing OpenVINO Runtime from YUM is recommended for C++ developers. If you a gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB EOF -2. Move the new `openvino-2022.repo` file to the YUM configuration directory, i.e. `/etc/yum.repos.d`: +2. Move the new ``openvino-2022.repo`` file to the YUM configuration directory, i.e. ``/etc/yum.repos.d``: .. code-block:: sh @@ -82,13 +89,14 @@ To list available OpenVINO packages, use the following command: yum list 'openvino*' -@endsphinxdirective -### Step 2: Install OpenVINO Runtime Using the YUM Package Manager -#### Install OpenVINO Runtime +Step 2: Install OpenVINO Runtime Using the YUM Package Manager +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Install OpenVINO Runtime +------------------------- -@sphinxdirective .. tab:: The Latest Version @@ -112,11 +120,11 @@ To list available OpenVINO packages, use the following command: sudo yum install openvino-2022.3.0 -@endsphinxdirective -#### Check for Installed Packages and Version -@sphinxdirective +Check for Installed Packages and Version +----------------------------------------- + Run the following command: @@ -124,30 +132,28 @@ Run the following command: yum list installed 'openvino*' -.. _intall additional components yum: - -@endsphinxdirective +Step 3 (Optional): Install Additional Components ++++++++++++++++++++++++++++++++++++++++++++++++++ -### Step 3 (Optional): Install Additional Components - -@sphinxdirective - -OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you installed OpenVINO Runtime using YUM, OpenVINO Development Tools must be installed separately. +OpenVINO Development Tools is a set of utilities for working with OpenVINO and OpenVINO models. +It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and +Open Model Zoo Downloader. If you installed OpenVINO Runtime using YUM, OpenVINO Development +Tools must be installed separately. See **For C++ Developers** section on the :doc:`Install OpenVINO Development Tools ` page for instructions. -@endsphinxdirective - -### Step 4 (Optional): Configure Inference on Non-CPU Devices -To enable the toolkit components to use processor graphics (GPU) on your system, follow the steps in [GPU Setup Guide](@ref openvino_docs_install_guides_configurations_for_intel_gpu). +Step 4 (Optional): Configure Inference on Non-CPU Devices +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -### Step 5: Build Samples +To enable the toolkit components to use processor graphics (GPU) on your system, follow the steps in +:doc:`GPU Setup Guide `. -@sphinxdirective +Step 5: Build Samples +++++++++++++++++++++++ -To build the C++ or C sample applications for Linux, run the `build_samples.sh` script: +To build the C++ or C sample applications for Linux, run the ``build_samples.sh`` script: .. tab:: C++ @@ -161,13 +167,12 @@ To build the C++ or C sample applications for Linux, run the `build_samples.sh` /usr/share/openvino/samples/c/build_samples.sh -@endsphinxdirective -For more information, refer to Build the Sample Applications on Linux. +For more information, refer to :doc:`Build the Sample Applications on Linux `. -### Uninstalling OpenVINO Runtime -@sphinxdirective +Uninstalling OpenVINO Runtime +############################## To uninstall OpenVINO Runtime via YUM, run the following command based on your needs: @@ -190,15 +195,16 @@ To uninstall OpenVINO Runtime via YUM, run the following command based on your n sudo yum autoremove openvino-2022.3.0 -@endsphinxdirective -## What's Next? -@sphinxdirective +What's Next? +############# -Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials: +Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! +Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials: -* Try the `C++ Quick Start Example `_ for step-by-step instructions on building and running a basic image classification C++ application. +* Try the `C++ Quick Start Example `_ + for step-by-step instructions on building and running a basic image classification C++ application. .. image:: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :width: 400 @@ -217,8 +223,14 @@ You can also try the following things: * See sample applications in :doc:`OpenVINO toolkit Samples Overview `. * Take a glance at the OpenVINO product home page: https://software.intel.com/en-us/openvino-toolkit. + + +Additional Resources +##################### + +- `OpenVINO Installation Selector Tool `__ + + @endsphinxdirective -## Additional Resources -- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) \ No newline at end of file diff --git a/docs/install_guides/troubleshooting-issues.md b/docs/install_guides/troubleshooting-issues.md index fd539d6ea64845..a381f0f46c1517 100644 --- a/docs/install_guides/troubleshooting-issues.md +++ b/docs/install_guides/troubleshooting-issues.md @@ -1,52 +1,71 @@ # Issues & Solutions for OpenVINO™ Installation & Configuration {#openvino_docs_get_started_guide_troubleshooting_issues} +@sphinxdirective + This page lists issues that you may encounter during the installation and configuration of OpenVINO™, as well as their possible solutions. -## Errors with Installing via PIP for Users in China +.. _install_for_prc: + +Errors with Installing via PIP for Users in China +################################################# Users in China might encounter errors while downloading sources via PIP during OpenVINO™ installation. To resolve the issues, try one of the following options: * Add the download source using the ``-i`` parameter with the Python ``pip`` command. For example: - ``` sh - pip install openvino-dev -i https://mirrors.aliyun.com/pypi/simple/ - ``` - Use the ``--trusted-host`` parameter if the URL above is ``http`` instead of ``https``. - You can also run the following command to install specific framework. For example: + .. code-block:: sh + + pip install openvino-dev -i https://mirrors.aliyun.com/pypi/simple/ + + Use the ``--trusted-host`` parameter if the URL above is ``http`` instead of ``https``. + You can also run the following command to install specific framework. For example: + + .. code-block:: sh + + pip install openvino-dev[tensorflow2] -i https://mirrors.aliyun.com/pypi/simple/ - ``` - pip install openvino-dev[tensorflow2] -i https://mirrors.aliyun.com/pypi/simple/ - ``` * For C++ developers, if you have installed OpenVINO Runtime via APT, YUM, or the archive file, and then installed OpenVINO Development Tools via PyPI, you may run into issues. To resolve that, install the components in ``requirements.txt`` by using the following command: - ``` sh - pip install -r /tools/requirements.txt - ``` - For APT and YUM users, replace the `INSTALL_DIR` with `/usr/share/openvino`. + + .. code-block:: sh + + pip install -r /tools/requirements.txt + + For APT and YUM users, replace the ``INSTALL_DIR`` with ``/usr/share/openvino``. -## Issues with Installing OpenVINO on Linux from Docker +Issues with Installing OpenVINO on Linux from Docker +#################################################### + +.. _proxy-issues: -### Proxy Issues +Proxy Issues +++++++++++++ -If you met proxy issues during the installation with Docker, you need set up proxy settings for Docker. See the [Docker guide](https://docs.docker.com/network/proxy/) for more details. +If you met proxy issues during the installation with Docker, you need set up proxy settings for Docker. See the `Docker guide `__ for more details. +.. _yocto_install_issues: -@anchor yocto-install-issues -## Issues with Creating a Yocto Image for OpenVINO +Issues with Creating a Yocto Image for OpenVINO +############################################### -### Error while adding "meta-intel" layer +Error while adding "meta-intel" layer ++++++++++++++++++++++++++++++++++++++ -When using the `bitbake-layers add-layer meta-intel` command, the following error might occur: -```sh -NOTE: Starting bitbake server... -ERROR: The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed: chrpath diffstat pzstd zstd -``` +When using the ``bitbake-layers add-layer meta-intel`` command, the following error might occur: -To resolve the issue, install the `chrpath diffstat zstd` tools: +.. code-block:: sh + + NOTE: Starting bitbake server... + ERROR: The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed: chrpath diffstat pzstd zstd + + +To resolve the issue, install the ``chrpath diffstat zstd`` tools: + +.. code-block:: sh + + sudo apt-get install chrpath diffstat zstd -```sh -sudo apt-get install chrpath diffstat zstd -``` +@endsphinxdirective diff --git a/docs/install_guides/troubleshooting-steps.md b/docs/install_guides/troubleshooting-steps.md index ec4049f445325b..fd5f03c73b21d2 100644 --- a/docs/install_guides/troubleshooting-steps.md +++ b/docs/install_guides/troubleshooting-steps.md @@ -1,56 +1,69 @@ # Troubleshooting Steps for OpenVINO™ Installation and Configurations {#openvino_docs_get_started_guide_troubleshooting_steps} +@sphinxdirective + If you run into issues while installing or configuring OpenVINO™, you can try the following methods to do some quick checks first. -## Check the versions of OpenVINO Runtime and Developement Tools +Check the versions of OpenVINO Runtime and Developement Tools +############################################################# * To check the version of OpenVINO Development Tools, use the following command: - ```sh - mo --version - ``` + + .. code-block:: sh + + mo --version + * To check the version of OpenVINO Runtime, use the following code: - ```sh - from openvino.runtime import get_version get_version() - ``` + + .. code-block:: sh + + from openvino.runtime import get_version get_version() + -## Check the versions of Python and PIP +Check the versions of Python and PIP +#################################### -To check your Python version, run `python -VV` or `python --version`. The supported Python versions should be 64-bit and between 3.7 and 3.10. If you are using Python 3.6, you are recommended to upgrade the version to 3.7 or higher. +To check your Python version, run ``python -VV`` or ``python --version``. The supported Python versions should be 64-bit and between 3.7 and 3.10. If you are using Python 3.6, you are recommended to upgrade the version to 3.7 or higher. If your Python version does not meet the requirements, update Python: -* For Windows, **do not install Python from a Windows Store** as it can cause issues. You are highly recommended to install Python from . -* For Linux and macOS systems, download and install a proper Python version from . See the [Python Beginners' Guide](https://wiki.python.org/moin/BeginnersGuide/Download) for more information on selecting a version. Note that macOS 10.x comes with python 2.7 installed, which is not supported, so you must install Python from the official website. +* For Windows, **do not install Python from a Windows Store** as it can cause issues. You are highly recommended to install Python from `official website `__ . +* For Linux and macOS systems, download and install a proper Python version from `official website `__ . See the `Python Beginners' Guide `__ for more information on selecting a version. Note that macOS 10.x comes with python 2.7 installed, which is not supported, so you must install Python from the official website. For PIP, make sure that you have installed the latest version. To check and upgrade your PIP version, run the following command: -```sh -python -m pip install --upgrade pip -``` + +.. code-block:: sh + + python -m pip install --upgrade pip - -## Check if required external dependencies are installed (for pre-2022.2 releases) +Check if required external dependencies are installed (for pre-2022.2 releases) +############################################################################### For OpenVINO releases prior to 2022.2: -- If you are using Ubuntu or RHEL 8 systems, and installed OpenVINO Runtime via the archive file, APT, or YUM repository, and then decided to [install OpenVINO Development Tools](installing-model-dev-tools.md), make sure that you **Install External Software Dependencies** first by following the steps in the corresponding installation pages. -- For C++ developers with Windows systems, make sure that Microsoft Visual Studio 2019 with MSBuild and CMake 3.14 or higher (64-bit) are installed. While installing Microsoft Visual Studio 2019, make sure that you have selected **Desktop development with C++** in the **Workloads** tab. If not, launch the installer again to select that option. For more information on modifying the installation options for Microsoft Visual Studio, see its [official support page](https://docs.microsoft.com/en-us/visualstudio/install/modify-visual-studio?view=vs-2019). -## Check if environment variables are set correctly +- If you are using Ubuntu or RHEL 8 systems, and installed OpenVINO Runtime via the archive file, APT, or YUM repository, and then decided to :doc:`install OpenVINO Development Tools `, make sure that you **Install External Software Dependencies** first by following the steps in the corresponding installation pages. +- For C++ developers with Windows systems, make sure that Microsoft Visual Studio 2019 with MSBuild and CMake 3.14 or higher (64-bit) are installed. While installing Microsoft Visual Studio 2019, make sure that you have selected **Desktop development with C++** in the **Workloads** tab. If not, launch the installer again to select that option. For more information on modifying the installation options for Microsoft Visual Studio, see its `official support page `__ . -- For Python developers, if you previously installed OpenVINO using the archive file, and are now installing OpenVINO using PIP, remove all the PATH settings and the lines with `setupvars` from `.bashrc`. Note that if you installed OpenVINO with PIP in a virtual environment, you don't need to set any environment variables. -- If you have installed OpenVINO before, you probably have added `setupvars` to your `PATH /.bashrc` or Windows environment variables. After restarting your environment, you should see similar information as below: -```sh -[setupvars.sh] OpenVINO™ environment initialized -``` - - If you don't see the information above, your PATH variables may be configured incorrectly. Check if you have typed the correct or you are trying to activate in the correct directory. - - If you added it to a `.bashrc` file, make sure that the command is correctly written and the file is found in the `~/.bashrc` folder. +Check if environment variables are set correctly +################################################ -## Verify that OpenVINO is correctly installed +- For Python developers, if you previously installed OpenVINO using the archive file, and are now installing OpenVINO using PIP, remove all the PATH settings and the lines with ``setupvars`` from ``.bashrc``. Note that if you installed OpenVINO with PIP in a virtual environment, you don't need to set any environment variables. +- If you have installed OpenVINO before, you probably have added ``setupvars`` to your ``PATH /.bashrc`` or Windows environment variables. After restarting your environment, you should see similar information as below: -@sphinxdirective + .. code-block:: sh + + [setupvars.sh] OpenVINO™ environment initialized + + + - If you don't see the information above, your PATH variables may be configured incorrectly. Check if you have typed the correct or you are trying to activate in the correct directory. + - If you added it to a ``.bashrc`` file, make sure that the command is correctly written and the file is found in the ``~/.bashrc`` folder. + +Verify that OpenVINO is correctly installed +########################################### * For Python developers, to verify if OpenVINO is correctly installed, use the following command: @@ -66,24 +79,24 @@ For OpenVINO releases prior to 2022.2: * If you installed OpenVINO Runtime from YUM, use the ``yum list installed 'openvino*'`` command to list the installed OpenVINO packages. -@endsphinxdirective - -## Check if GPU drvier is installed +Check if GPU drvier is installed +################################ -[Additional configurations](configurations-header.md) may be required in order to use OpenVINO with different hardware such as Intel® GPUs. +:doc:`Additional configurations ` may be required in order to use OpenVINO with different hardware such as Intel® GPUs. -To run inference on an Intel® GPU, make sure that you have installed the correct GPU driver. To check that, see [additional configurations for GPU](configurations-for-intel-gpu.md). +To run inference on an Intel® GPU, make sure that you have installed the correct GPU driver. To check that, see :doc:`additional configurations for GPU `. -## Check firewall and network settings +Check firewall and network settings +################################### Make sure that your firewall and network settings are configured correctly. For example, consider configuring system-wide proxy settings and specifying options for using PIP behind the proxy: -@sphinxdirective +.. code-block:: sh + + pip install --proxy http://address:port --trusted-host pypi.org openvino - .. code-block:: sh - pip install --proxy http://address:port --trusted-host pypi.org openvino +For specific issues, see :ref:`Errors with Installing via PIP for Users in China ` and :ref:`proxy issues with installing OpenVINO on Linux from Docker `. @endsphinxdirective -For specific issues, see Errors with Installing via PIP for Users in China and proxy issues with installing OpenVINO on Linux from Docker. \ No newline at end of file diff --git a/docs/install_guides/troubleshooting.md b/docs/install_guides/troubleshooting.md index 9963a579978025..99e3fd7ca8ea97 100644 --- a/docs/install_guides/troubleshooting.md +++ b/docs/install_guides/troubleshooting.md @@ -9,16 +9,14 @@ Issues & Solutions Troubleshooting Steps -@endsphinxdirective - -@sphinxdirective .. _troubleshooting guide for install: -@endsphinxdirective - This guide provides general troubleshooting steps and solutions to possible issues that can be encountered while installing and configuring OpenVINO™. -The [Issues & Solutions](./troubleshooting-issues.md) page lists common installation and configuration errors, and their possible solutions. If you encountered a specific error while installing or configuring OpenVINO, check this page to see if there is a solution. +The :doc:`Issues & Solutions ` page lists common installation and configuration errors, and their possible solutions. If you encountered a specific error while installing or configuring OpenVINO, check this page to see if there is a solution. + +The :doc:`Troubleshooting Steps ` page provides a set of instructions for diagnosing and resolving installation and configuration issues. If you had problems during installation and configuration, walk through these steps to try and resolve your issue. + +@endsphinxdirective -The [Troubleshooting Steps](./troubleshooting-steps.md) page provides a set of instructions for diagnosing and resolving installation and configuration issues. If you had problems during installation and configuration, walk through these steps to try and resolve your issue. diff --git a/docs/nbdoc/consts.py b/docs/nbdoc/consts.py index d633d63d1e578e..946cfeb6842f44 100644 --- a/docs/nbdoc/consts.py +++ b/docs/nbdoc/consts.py @@ -8,7 +8,7 @@ repo_name = "openvino_notebooks" -artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230317115622/dist/rst_files/" +artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230323220806/dist/rst_files/" blacklisted_extensions = ['.xml', '.bin'] diff --git a/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/css/openvino_sphinx_theme.css b/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/css/openvino_sphinx_theme.css index 2a6063da77b8b3..a9678d9a73249a 100644 --- a/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/css/openvino_sphinx_theme.css +++ b/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/css/openvino_sphinx_theme.css @@ -60,6 +60,7 @@ body { .scrollbox { overflow-y:scroll; height:300px; + margin-bottom: 20px; } /* Syntax Highlighting */ diff --git a/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/js/sortable_tables.js b/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/js/sortable_tables.js new file mode 100644 index 00000000000000..523d7b4d4a7031 --- /dev/null +++ b/docs/openvino_sphinx_theme/openvino_sphinx_theme/static/js/sortable_tables.js @@ -0,0 +1,76 @@ +document.addEventListener('DOMContentLoaded', function () { + sortableTables(); +}); + +function sortableTables() { + + var tablestosort = $('div.sort-table').find('table.table').addClass('table-sortable'); + var tablestosortnext = $('div.sort-table').next('table.table').addClass('table-sortable'); + var sortables = $('table.table-sortable').append(tablestosortnext); + var tables = sortables.append(tablestosort); + tables.each(function () { + var table = $(this); + var headings = table.find('th'); + headings.each(function () { + var th = $(this); + var index = th.index(); + var sortBtn = $(''); + th.addClass('sort-header'); + th.click(function () { + var counter = 0; + sortBtn.addClass('sort-active'); + sortBy = sortBtn.data('sortby'); + var trs = table.find('tbody tr'); + sortBtn.toggleClass('ascending'); + trs.sort(function (item1, item2) { + + if (sortBtn.hasClass('ascending')) { + var text1 = $(item1).find('td').eq(index).text(); + var text2 = $(item2).find('td').eq(index).text(); + } + else { + var text1 = $(item2).find('td').eq(index).text(); + var text2 = $(item1).find('td').eq(index).text(); + } + // try converting to num + var _text1 = parseFloat(text1); + var _text2 = parseFloat(text2); + + if (!isNaN(_text1) && !isNaN(_text2)) { + text1 = _text1; + text2 = _text2; + } + if (text1 > text2) { + return 1; + } + else if (text1 < text2) { + return -1; + } + else { + return 0; + } + }).map(function () { + var row = $(this); + if (counter % 2 === 0) { + row.removeClass('row-odd'); + row.addClass('row-even'); + } + else { + row.removeClass('row-even'); + row.addClass('row-odd'); + } + counter++; + table.find('tbody').append(row); + }); + + headings.each(function () { + if ($(this).index() !== index) { + $(this).find('.sort-btn').removeClass('ascending'); + $(this).find('.sort-btn').removeClass('sort-active'); + } + }); + }); + th.find('p').append(sortBtn); + }); + }); +} \ No newline at end of file diff --git a/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html b/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html index 5a0155f3f8de5a..678ad2b37a7ede 100644 --- a/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html +++ b/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html @@ -19,6 +19,7 @@ + {% endblock %} {%- block content %} diff --git a/docs/ops/image/Interpolate_11.md b/docs/ops/image/Interpolate_11.md index 14b37383a2ad92..05014c8a153bf5 100644 --- a/docs/ops/image/Interpolate_11.md +++ b/docs/ops/image/Interpolate_11.md @@ -91,7 +91,7 @@ * **2**: `scales_or_sizes` - 1D tensor containing the data used to calculate the spatial output shape. The number of elements must match the number of values in the `axes` input tensor, the order needs to match as well. The type of this input tensor is either *T_SCALES* or *T_SIZES* depending on the value of the `shape_calculation_mode` attribute. **Required.** -* **4**: `axes` - 1D tensor of type *T_AXES* specifying dimension indices where interpolation is applied, and `axes` is any unordered list of indices of different dimensions of input tensor, e.g. `[0, 4]`, `[4, 0]`, `[4, 2, 1]`, `[1, 2, 3]`. These indices should be non-negative integers from `0` to `rank(image) - 1` inclusively. Other dimensions do not change. The order of elements in `axes` attribute matters and is mapped directly to the elements in the 2nd input `scales_or_sizes`. **Optional** with default value `[X,...,rank(image) - 1]` pointing to the spatial dimensions of the input image. The number of the default axes matches the number of elements in the `scales_or_sizes` input tensor. +* **3**: `axes` - 1D tensor of type *T_AXES* specifying dimension indices where interpolation is applied, and `axes` is any unordered list of indices of different dimensions of input tensor, e.g. `[0, 4]`, `[4, 0]`, `[4, 2, 1]`, `[1, 2, 3]`. These indices should be non-negative integers from `0` to `rank(image) - 1` inclusively. Other dimensions do not change. The order of elements in `axes` attribute matters and is mapped directly to the elements in the 2nd input `scales_or_sizes`. **Optional** with default value `[X,...,rank(image) - 1]` pointing to the spatial dimensions of the input image. The number of the default axes matches the number of elements in the `scales_or_sizes` input tensor. **Outputs** diff --git a/docs/optimization_guide/dldt_deployment_optimization_guide.md b/docs/optimization_guide/dldt_deployment_optimization_guide.md index 9ae41edc37a0dc..8dfad7f088d87b 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_guide.md +++ b/docs/optimization_guide/dldt_deployment_optimization_guide.md @@ -16,7 +16,7 @@ Runtime optimization, or deployment optimization, focuses on tuning inference parameters and execution means (e.g., the optimum number of requests executed simultaneously). Unlike model-level optimizations, they are highly specific to the hardware and case they are used for, and often come at a cost. -`ov::inference_precision `__ is a "typical runtime configuration" which trades accuracy for performance, allowing ``fp16/bf16`` execution for the layers that remain in ``fp32`` after quantization of the original ``fp32`` model. +`ov::hint::inference_precision `__ is a "typical runtime configuration" which trades accuracy for performance, allowing ``fp16/bf16`` execution for the layers that remain in ``fp32`` after quantization of the original ``fp32`` model. Therefore, optimization should start with defining the use case. For example, if it is about processing millions of samples by overnight jobs in data centers, throughput could be prioritized over latency. On the other hand, real-time usages would likely trade off throughput to deliver the results at minimal latency. A combined scenario is also possible, targeting the highest possible throughput, while maintaining a specific latency threshold. diff --git a/docs/optimization_guide/dldt_deployment_optimization_internals.md b/docs/optimization_guide/dldt_deployment_optimization_internals.md index b03742d351f180..ab596e49c98e0b 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_internals.md +++ b/docs/optimization_guide/dldt_deployment_optimization_internals.md @@ -1,168 +1,70 @@ # Further Low-Level Implementation Details {#openvino_docs_deployment_optimization_guide_internals} -## Throughput on the CPU: Internals -As explained in the [throughput-related section](./dldt_deployment_optimization_tput.md), the OpenVINO streams are means of running multiple requests in parallel. -In order to best serve multiple inference requests executed simultaneously, the inference threads are grouped/pinned to the particular CPU cores, constituting the "CPU" streams. -This provides much better performance for the networks than batching, especially for the multiple-core systems: - -@sphinxdirective - -.. container:: row-two-col-content - - .. container:: column-two-col-content - - **Conventional Approach** - - | Every CNN op is internally parallelized over a full number of CPU cores and it is detrimental for non-scalable ops. - | A lot of synchronization between many threads results in overhead. - | An only option to improve efficiency is batching. - - .. container:: column-two-col-content - - **Streams** - - | CPU cores are evenly distributed between execution streams (each 1-4 threads). - | Less threads per stream means less synchronization, better locality, and finer granularity. - -@endsphinxdirective - -@sphinxdirective - -.. raw:: html - -
-
- -@endsphinxdirective - - -![](../img/cpu_execution_conventional_approach.svg) @sphinxdirective -.. raw:: html +Throughput on the CPU: Internals +################################ -
-
- -@endsphinxdirective - - -![](../img/cpu_execution_streams.svg) - -@sphinxdirective - -.. raw:: html - -
-
- -@endsphinxdirective - - -@sphinxdirective - -.. container:: row-two-col-content - - .. container:: column-two-col-content - - .. raw:: html +As explained in the :doc:`throughput-related section `, the OpenVINO streams are means of running multiple requests in parallel. +In order to best serve multiple inference requests executed simultaneously, the inference threads are grouped/pinned to the particular CPU cores, constituting the "CPU" streams. +This provides much better performance for the networks than batching, especially for the multiple-core systems: -
+.. list-table:: + :header-rows: 1 - .. container:: column-two-col-content + * - Conventional Approach + - Streams + * - | Every CNN op is internally parallelized over a full number of CPU cores and it is detrimental for non-scalable ops. + | A lot of synchronization between many threads results in overhead. + | An only option to improve efficiency is batching. + - | CPU cores are evenly distributed between execution streams (each 1-4 threads). + | Less threads per stream means less synchronization, better locality, and finer granularity. + * - |conventional-approach| + - | |execution-streams| + | Requests are executed in parallel with a small number of threads. + | Layer-wise, the streams imply much less synchronization. - | Requests are executed in parallel with a small number of threads. - | **Layer-wise, the streams imply much less synchronization.** - -@endsphinxdirective +.. |conventional-approach| image:: _static/images/cpu_execution_conventional_approach.svg +.. |execution-streams| image:: _static/images/cpu_execution_streams.svg Compared to the batching, the parallelism is somewhat transposed (performed over inputs with much less synchronization within CNN ops): -@sphinxdirective - -.. container:: row-two-col-content - - .. container:: column-two-col-content - - **Large Batch Approach** - - | All threads process all inputs at once. - | Assumes all layers are parallelized well. - | "Fat" requests are executed one by one. - - .. container:: column-two-col-content - - **Streams** - - | CPU cores are evenly distributed between execution streams. - | "Parallelize the outermost loop" rule of thumb. - | Individual requests are executed in parallel. - -@endsphinxdirective - - -@sphinxdirective - -.. raw:: html - -
-
- -@endsphinxdirective - - -![](../img/large_batch_approach.svg) - -@sphinxdirective - -.. raw:: html +.. list-table:: + :header-rows: 1 -
-
+ * - Large Batch Approach + - Streams + * - | All threads process all inputs at once. + | Assumes all layers are parallelized well. + | “Fat” requests are executed one by one. + - | CPU cores are evenly distributed between execution streams. + | “Parallelize the outermost loop” rule of thumb. + | Individual requests are executed in parallel. + * - |large-batch-approach| + - | |execution-streams-2| + | Inputs-wise the streams are the “transposed” batch. -@endsphinxdirective - - -![](../img/cpu_execution_streams_2.svg) - -@sphinxdirective - -.. raw:: html +.. |large-batch-approach| image:: _static/images/large_batch_approach.svg -
-
- -@endsphinxdirective - - -@sphinxdirective +.. |execution-streams-2| image:: _static/images/cpu_execution_streams_2.svg -.. container:: row-two-col-content - .. container:: column-two-col-content +Keep in mind that :doc:`high-level performance hints ` allow the implementation to select the optimal number of streams depending on model's compute demands and CPU capabilities, including :doc:`int8 inference ` hardware acceleration, number of cores, etc. - .. raw:: html +Automatic Batching Internals +############################ -
- - .. container:: column-two-col-content - - **Inputs-wise the streams are the “transposed” batch.** - -@endsphinxdirective - - -Keep in mind that [high-level performance hints](../OV_Runtime_UG/performance_hints.md) allow the implementation to select the optimal number of streams depending on model's compute demands and CPU capabilities, including [int8 inference](@ref openvino_docs_model_optimization_guide) hardware acceleration, number of cores, etc. - -## Automatic Batching Internals -[Automatic batching](../OV_Runtime_UG/automatic_batching.md) performs on-the-fly grouping of inference requests to improve device utilization. +:doc:`Automatic batching ` performs on-the-fly grouping of inference requests to improve device utilization. It relaxes the requirement for an application to saturate devices such as GPU by using a large batch "explicitly". It performs transparent input gathering from individual inference requests followed by the actual batched execution, with no programming effort from the user: -![](../img/batch_device.svg) + +.. image:: _static/images/batch_device.svg Essentially, Automatic Batching shifts asynchronicity from individual requests to groups of requests that constitute the batches. Furthermore, for the execution to be efficient, it is very important that the requests arrive timely, without causing a batching timeout. Normally, the timeout should never be hit. It is rather a graceful way to handle the application exit (when the inputs are not arriving anymore, so the full batch is not possible to collect). If a workload experiences timeouts, which lead to a drop in performance due to increased latency of every request, consider balancing its value against the batch size. For example, a smaller batch size and timeout value may yield better results than a large batch size coupled with a timeout value that cannot guarantee accommodating all the required requests. -Finally, following the `get_tensor` idiom section from the [general optimizations](./dldt_deployment_optimization_common.md) helps Automatic Batching to save on inputs/outputs copies. According to that, you should always prefer the "get" versions of the tensors' data access APIs in your applications. +Finally, following the ``get_tensor`` idiom section from the :doc:`general optimizations ` helps Automatic Batching to save on inputs/outputs copies. According to that, you should always prefer the "get" versions of the tensors' data access APIs in your applications. + +@endsphinxdirective diff --git a/docs/optimization_guide/dldt_deployment_optimization_latency.md b/docs/optimization_guide/dldt_deployment_optimization_latency.md index 66a5818f7d0ea9..a8702a539e237a 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_latency.md +++ b/docs/optimization_guide/dldt_deployment_optimization_latency.md @@ -38,6 +38,8 @@ For example, when the model is used exactly once, or when it is unloaded and rel Such a "first-inference latency" scenario may pose an additional limitation on the model load\compilation time, as inference accelerators (other than the CPU) usually require a certain level of model compilation upon loading. The :doc:`model caching ` option is a way to lessen the impact over multiple application runs. If model caching is not possible, for example, it may require write permissions for the application, the CPU offers the fastest model load time almost every time. +To improve common "first-inference latency" scenario, model reading was replaced with model mapping (using `mmap`) into a memory. But in some use cases (first of all, if model is located on removable or network drive) mapping may lead to latency increase. To switch mapping to reading, specify ``ov::enable_mmap(false)`` property for the ``ov::Core``. + Another way of dealing with first-inference latency is using the :doc:`AUTO device selection inference mode `. It starts inference on the CPU, while waiting for the actual accelerator to load the model. At that point, it shifts to the new device seamlessly. Finally, note that any :doc:`throughput-oriented options ` may significantly increase the model uptime. diff --git a/docs/optimization_guide/model_optimization_guide.md b/docs/optimization_guide/model_optimization_guide.md index ca2d463227cacf..5936cf9c9b7563 100644 --- a/docs/optimization_guide/model_optimization_guide.md +++ b/docs/optimization_guide/model_optimization_guide.md @@ -1,21 +1,17 @@ - # Model Optimization Guide {#openvino_docs_model_optimization_guide} +# Model Optimization Guide {#openvino_docs_model_optimization_guide} @sphinxdirective .. toctree:: :maxdepth: 1 :hidden: - + ptq_introduction tmo_introduction (Experimental) Protecting Model -@endsphinxdirective - - Model optimization is an optional offline step of improving final model performance by applying special optimization methods, such as quantization, pruning, preprocessing optimization, etc. OpenVINO provides several tools to optimize models at different steps of model development: - -@sphinxdirective +Model optimization is an optional offline step of improving final model performance by applying special optimization methods, such as quantization, pruning, preprocessing optimization, etc. OpenVINO provides several tools to optimize models at different steps of model development: - :doc:`Model Optimizer ` implements most of the optimization parameters to a model by default. Yet, you are free to configure mean/scale values, batch size, RGB vs BGR input channels, and other parameters to speed up preprocess of a model (:doc:`Embedding Preprocessing Computation `). @@ -23,25 +19,29 @@ - :doc:`Training-time Optimization `, a suite of advanced methods for training-time model optimization within the DL framework, such as PyTorch and TensorFlow 2.x. It supports methods, like Quantization-aware Training and Filter Pruning. NNCF-optimized models can be inferred with OpenVINO using all the available workflows. -@endsphinxdirective +Detailed workflow: +################## -## Detailed workflow: To understand which development optimization tool you need, refer to the diagram: -![](../img/DEVELOPMENT_FLOW_V3_crunch.svg) +.. image:: _static/images/DEVELOPMENT_FLOW_V3_crunch.svg Post-training methods are limited in terms of achievable accuracy-performance trade-off for optimizing models. In this case, training-time optimization with NNCF is an option. Once the model is optimized using the aforementioned tools it can be used for inference using the regular OpenVINO inference workflow. No changes to the inference code are required. -![](../img/WHAT_TO_USE.svg) +.. image:: _static/images/WHAT_TO_USE.svg Post-training methods are limited in terms of achievable accuracy, which may degrade for certain scenarios. In such cases, training-time optimization with NNCF may give better results. Once the model has been optimized using the aforementioned tools, it can be used for inference using the regular OpenVINO inference workflow. No changes to the code are required. -If you are not familiar with model optimization methods, refer to [post-training methods](@ref pot_introduction). +If you are not familiar with model optimization methods, refer to :doc:`post-training methods `. + +Additional Resources +#################### -## Additional Resources -- [Deployment optimization](./dldt_deployment_optimization_guide.md) \ No newline at end of file +- :doc:`Deployment optimization ` + +@endsphinxdirective diff --git a/docs/optimization_guide/nncf/introduction.md b/docs/optimization_guide/nncf/introduction.md index ba2a2662ba3a17..a4fcbbead198b4 100644 --- a/docs/optimization_guide/nncf/introduction.md +++ b/docs/optimization_guide/nncf/introduction.md @@ -9,10 +9,11 @@ qat_introduction filter_pruning -@endsphinxdirective -## Introduction -Training-time model compression improves model performance by applying optimizations (such as quantization) during the training. The training process minimizes the loss associated with the lower-precision optimizations, so it is able to maintain the model’s accuracy while reducing its latency and memory footprint. Generally, training-time model optimization results in better model performance and accuracy than [post-training optimization](@ref pot_introduction), but it can require more effort to set up. +Introduction +#################### + +Training-time model compression improves model performance by applying optimizations (such as quantization) during the training. The training process minimizes the loss associated with the lower-precision optimizations, so it is able to maintain the model’s accuracy while reducing its latency and memory footprint. Generally, training-time model optimization results in better model performance and accuracy than :doc:`post-training optimization `, but it can require more effort to set up. OpenVINO provides the Neural Network Compression Framework (NNCF) tool for implementing compression algorithms on models to improve their performance. NNCF is a Python library that integrates into PyTorch and TensorFlow training pipelines to add training-time compression methods to the pipeline. To apply training-time compression methods with NNCF, you need: @@ -22,65 +23,93 @@ OpenVINO provides the Neural Network Compression Framework (NNCF) tool for imple Adding compression to a training pipeline only requires a few lines of code. The compression techniques are defined through a single configuration file that specifies which algorithms to use during fine-tuning. -### NNCF Quick Start Examples +NNCF Quick Start Examples ++++++++++++++++++++++++++ + See the following Jupyter Notebooks for step-by-step examples showing how to add model compression to a PyTorch or Tensorflow training pipeline with NNCF: -- [Quantization Aware Training with NNCF and PyTorch](https://docs.openvino.ai/latest/notebooks/302-pytorch-quantization-aware-training-with-output.html). -- [Quantization Aware Training with NNCF and TensorFlow](https://docs.openvino.ai/latest/notebooks/305-tensorflow-quantization-aware-training-with-output.html). +- `Quantization Aware Training with NNCF and PyTorch `__. +- `Quantization Aware Training with NNCF and TensorFlow `__. + +Installation +#################### + +NNCF is open-sourced on `GitHub `__ and distributed as a separate package from OpenVINO. It is also available on PyPI. Install it to the same Python environment where PyTorch or TensorFlow is installed. -## Installation -NNCF is open-sourced on [GitHub](https://github.com/openvinotoolkit/nncf) and distributed as a separate package from OpenVINO. It is also available on PyPI. Install it to the same Python environment where PyTorch or TensorFlow is installed. +Install from PyPI +++++++++++++++++++++ -### Install from PyPI To install the latest released version via pip manager run the following command: -``` -pip install nncf -``` -> **NOTE**: To install with specific frameworks, use the `pip install nncf[extras]` command, where extras is a list of possible extras, for example, `torch`, `tf`, `onnx`. +.. code-block:: sh + + pip install nncf + + +.. note:: + + To install with specific frameworks, use the `pip install nncf[extras]` command, where extras is a list of possible extras, for example, `torch`, `tf`, `onnx`. + -To install the latest NNCF version from source follow the instruction on [GitHub](https://github.com/openvinotoolkit/nncf#installation). +To install the latest NNCF version from source follow the instruction on `GitHub `__. -> **NOTE**: NNCF does not have OpenVINO as an installation requirement. To deploy optimized models you should install OpenVINO separately. +.. note:: + + NNCF does not have OpenVINO as an installation requirement. To deploy optimized models you should install OpenVINO separately. + +Working with NNCF +#################### -## Working with NNCF The figure below shows a common workflow of applying training-time compressions with NNCF. The NNCF optimizations are added to the TensorFlow or PyTorch training script, and then the model undergoes fine-tuning. The optimized model can then be exported to OpenVINO IR format for accelerated performance with OpenVINO Runtime. -![](../../img/nncf_workflow.svg) +.. image:: _static/images/nncf_workflow.svg + +Training-Time Compression Methods ++++++++++++++++++++++++++++++++++ -### Training-Time Compression Methods -NNCF provides several methods for improving model performance with training-time compression. +NNCF provides several methods for improving model performance with training-time compression. -#### Quantization -Quantization is the process of converting the weights and activation values in a neural network from a high-precision format (such as 32-bit floating point) to a lower-precision format (such as 8-bit integer). It helps to reduce the model’s memory footprint and latency. NNCF uses quantization-aware training to quantize models. +Quantization +-------------------- +Quantization is the process of converting the weights and activation values in a neural network from a high-precision format (such as 32-bit floating point) to a lower-precision format (such as 8-bit integer). It helps to reduce the model’s memory footprint and latency. NNCF uses quantization-aware training to quantize models. Quantization-aware training inserts nodes into the neural network during training that simulate the effect of lower precision. This allows the training algorithm to consider quantization errors as part of the overall training loss that gets minimized during training. The network is then able to achieve enhanced accuracy when quantized. -The officially supported method of quantization in NNCF is uniform 8-bit quantization. This means all the weights and activation functions in the neural network are converted to 8-bit values. See the [Quantization-ware Training guide](@ref qat_introduction) to learn more. +The officially supported method of quantization in NNCF is uniform 8-bit quantization. This means all the weights and activation functions in the neural network are converted to 8-bit values. See the :doc:`Quantization-ware Training guide ` to learn more. + +Filter pruning +-------------------- + +Filter pruning algorithms compress models by zeroing out the output filters of convolutional layers based on a certain filter importance criterion. During fine-tuning, an importance criteria is used to search for redundant filters that don’t significantly contribute to the network’s output and zero them out. After fine-tuning, the zeroed-out filters are removed from the network. For more information, see the :doc:`Filter Pruning ` page. -#### Filter pruning -Filter pruning algorithms compress models by zeroing out the output filters of convolutional layers based on a certain filter importance criterion. During fine-tuning, an importance criteria is used to search for redundant filters that don’t significantly contribute to the network’s output and zero them out. After fine-tuning, the zeroed-out filters are removed from the network. For more information, see the [Filter Pruning](@ref filter_pruning) page. +Experimental methods +-------------------- -#### Experimental methods NNCF also provides state-of-the-art compression techniques that are still in experimental stages of development and are only recommended for expert developers. These include: - Mixed-precision quantization - Sparsity - Binarization -To learn more about these methods, visit the [NNCF repository on GitHub](https://github.com/openvinotoolkit/nncf). +To learn more about these methods, visit the `NNCF repository on GitHub `__. + +Recommended Workflow +++++++++++++++++++++ -### Recommended Workflow Using compression-aware training requires a training pipeline, an annotated dataset, and compute resources (such as CPUs or GPUs). If you don't already have these set up and available, it can be easier to start post-training quantization to quickly see quantized results. Then you can use compression-aware training if the model isn't accurate enough. We recommend the following workflow for compressing models with NNCF: -1. [Perform post-training quantization](@ref pot_introduction) on your model and then compare performance to the original model. -2. If the accuracy is too degraded, use [Quantization-aware Training](@ref qat_introduction) to increase accuracy while still achieving faster inference time. -3. If the quantized model is still too slow, use [Filter Pruning](@ref filter_pruning) to further improve the model’s inference speed. +1. :doc:`Perform post-training quantization ` on your model and then compare performance to the original model. +2. If the accuracy is too degraded, use :doc:`Quantization-aware Training ` to increase accuracy while still achieving faster inference time. +3. If the quantized model is still too slow, use :doc:`Filter Pruning ` to further improve the model’s inference speed. + +Additional Resources +#################### -## Additional Resources -- [Quantizing Models Post-training](@ref pot_introduction) -- [NNCF GitHub repository](https://github.com/openvinotoolkit/nncf) -- [NNCF FAQ](https://github.com/openvinotoolkit/nncf/blob/develop/docs/FAQ.md) -- [Quantization Aware Training with NNCF and PyTorch](https://docs.openvino.ai/latest/notebooks/302-pytorch-quantization-aware-training-with-output.html) -- [Quantization Aware Training with NNCF and TensorFlow](https://docs.openvino.ai/latest/notebooks/305-tensorflow-quantization-aware-training-with-output.html) \ No newline at end of file +- :doc:`Quantizing Models Post-training ` +- `NNCF GitHub repository `__ +- `NNCF FAQ `__ +- `Quantization Aware Training with NNCF and PyTorch `__ +- `Quantization Aware Training with NNCF and TensorFlow `__ + +@endsphinxdirective diff --git a/docs/optimization_guide/nncf/ptq/basic_quantization_flow.md b/docs/optimization_guide/nncf/ptq/basic_quantization_flow.md index 2f315c04705fbd..fb5ab52aa0211d 100644 --- a/docs/optimization_guide/nncf/ptq/basic_quantization_flow.md +++ b/docs/optimization_guide/nncf/ptq/basic_quantization_flow.md @@ -1,4 +1,4 @@ -# Basic Quantization Flow {#basic_qauntization_flow} +# Basic Quantization Flow {#basic_quantization_flow} @sphinxdirective diff --git a/docs/optimization_guide/nncf/ptq/ptq_introduction.md b/docs/optimization_guide/nncf/ptq/ptq_introduction.md index a87e5f9d29339b..2cd880b50602f8 100644 --- a/docs/optimization_guide/nncf/ptq/ptq_introduction.md +++ b/docs/optimization_guide/nncf/ptq/ptq_introduction.md @@ -6,17 +6,21 @@ :maxdepth: 1 :hidden: - basic_qauntization_flow + basic_quantization_flow quantization_w_accuracy_control -@endsphinxdirective -Neural Network Compression Framework (NNCF) provides a new post-training quantization API available in Python that is aimed at reusing the code for model training or validation that is usually available with the model in the source framework, for example, PyTorch* or TensroFlow*. The API is cross-framework and currently supports models representing in the following frameworks: PyTorch, TensorFlow 2.x, ONNX, and OpenVINO. +Neural Network Compression Framework (NNCF) provides a new post-training quantization API available in Python that is aimed at reusing the code for model training or validation that is usually available with the model in the source framework, for example, PyTorch or TensroFlow. The API is cross-framework and currently supports models representing in the following frameworks: PyTorch, TensorFlow 2.x, ONNX, and OpenVINO. + This API has two main capabilities to apply 8-bit post-training quantization: -* [Basic quantization](@ref basic_qauntization_flow) - the simplest quantization flow that allows to apply 8-bit integer quantization to the model. -* [Quantization with accuracy control](@ref quantization_w_accuracy_control) - the most advanced quantization flow that allows to apply 8-bit quantization to the model with accuracy control. -## See also +* :doc:`Basic quantization ` - the simplest quantization flow that allows to apply 8-bit integer quantization to the model. +* :doc:`Quantization with accuracy control ` - the most advanced quantization flow that allows to apply 8-bit quantization to the model with accuracy control. + +Additional Resources +#################### -* [NNCF GitHub](https://github.com/openvinotoolkit/nncf) -* [Optimizing Models at Training Time](@ref tmo_introduction) \ No newline at end of file +* `NNCF GitHub `__ +* :doc:`Optimizing Models at Training Time ` + +@endsphinxdirective diff --git a/docs/optimization_guide/nncf/ptq/quantization_w_accuracy_control.md b/docs/optimization_guide/nncf/ptq/quantization_w_accuracy_control.md index 65d5ede50e4d8e..fec080c0b0aafc 100644 --- a/docs/optimization_guide/nncf/ptq/quantization_w_accuracy_control.md +++ b/docs/optimization_guide/nncf/ptq/quantization_w_accuracy_control.md @@ -5,12 +5,12 @@ Introduction #################### -This is the advanced quantization flow that allows to apply 8-bit quantization to the model with control of accuracy metric. This is achieved by keeping the most impactful operations within the model in the original precision. The flow is based on the :doc:`Basic 8-bit quantization ` and has the following differences: +This is the advanced quantization flow that allows to apply 8-bit quantization to the model with control of accuracy metric. This is achieved by keeping the most impactful operations within the model in the original precision. The flow is based on the :doc:`Basic 8-bit quantization ` and has the following differences: * Beside the calibration dataset, a **validation dataset** is required to compute accuracy metric. They can refer to the same data in the simplest case. * **Validation function**, used to compute accuracy metric is required. It can be a function that is already available in the source framework or a custom function. -* Since accuracy validation is run several times during the quantization process, quantization with accuracy control can take more time than the [Basic 8-bit quantization](@ref basic_qauntization_flow) flow. -* The resulted model can provide smaller performance improvement than the :doc:`Basic 8-bit quantization ` flow because some of the operations are kept in the original precision. +* Since accuracy validation is run several times during the quantization process, quantization with accuracy control can take more time than the [Basic 8-bit quantization](@ref basic_quantization_flow) flow. +* The resulted model can provide smaller performance improvement than the :doc:`Basic 8-bit quantization ` flow because some of the operations are kept in the original precision. .. note:: Currently, this flow is available only for models in OpenVINO representation. @@ -19,7 +19,7 @@ The steps for the quantization with accuracy control are described below. Prepare datasets #################### -This step is similar to the :doc:`Basic 8-bit quantization ` flow. The only difference is that two datasets, calibration and validation, are required. +This step is similar to the :doc:`Basic 8-bit quantization ` flow. The only difference is that two datasets, calibration and validation, are required. .. tab:: OpenVINO diff --git a/docs/optimization_guide/ptq_introduction.md b/docs/optimization_guide/ptq_introduction.md index acf2fa3ec23b56..1f218cb780c004 100644 --- a/docs/optimization_guide/ptq_introduction.md +++ b/docs/optimization_guide/ptq_introduction.md @@ -9,23 +9,27 @@ pot_introduction nncf_ptq_introduction -@endsphinxdirective Post-training model optimization is the process of applying special methods that transform the model into a more hardware-friendly representation without retraining or fine-tuning. The most popular and widely-spread method here is 8-bit post-training quantization because it is: + * It is easy-to-use. * It does not hurt accuracy a lot. * It provides significant performance improvement. * It suites many hardware available in stock since most of them support 8-bit computation natively. -8-bit integer quantization lowers the precision of weights and activations to 8 bits, which leads to almost 4x reduction in the model footprint and significant improvements in inference speed, mostly due to lower throughput required for the inference. This lowering step is done offline, before the actual inference, so that the model gets transformed into the quantized representation. The process does not require a training dataset or a training pipeline in the source DL framework. +8-bit integer quantization lowers the precision of weights and activations to 8 bits, which leads to almost 4x reduction in the model footprint and significant improvements in inference speed, mostly due to lower throughput required for the inference. This lowering step is done offline, before the actual inference, so that the model gets transformed into the quantized representation. The process does not require a training dataset or a training pipeline in the source DL framework. -![](../img/quantization_picture.svg) +.. image:: _static/images/quantization_picture.svg To apply post-training methods in OpenVINO, you need: + * A floating-point precision model, FP32 or FP16, converted into the OpenVINO Intermediate Representation (IR) format that can be run on CPU. * A representative calibration dataset, representing a use case scenario, for example, of 300 samples. * In case of accuracy constraints, a validation dataset and accuracy metrics should be available. Currently, OpenVINO provides two workflows with post-training quantization capabilities: -* [Post-training Quantization with POT](@ref pot_introduction) - works with models in OpenVINO Intermediate Representation (IR) only. -* [Post-training Quantization with NNCF](@ref nncf_ptq_introduction) - cross-framework solution for model optimization that provides a new simple API for post-training quantization. \ No newline at end of file + +* :doc:`Post-training Quantization with POT ` - works with models in OpenVINO Intermediate Representation (IR) only. +* :doc:`Post-training Quantization with NNCF ` - cross-framework solution for model optimization that provides a new simple API for post-training quantization. + +@endsphinxdirective diff --git a/docs/requirements.txt b/docs/requirements.txt index 19829c8e09af24..1de32203db2e10 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -31,7 +31,7 @@ six==1.15.0 snowballstemmer==2.1.0 soupsieve==2.2.1 sphinx==4.5.0 -sphinx-copybutton==0.3.3 +sphinx-copybutton==0.5.1 sphinx-design==0.3.0 sphinx-inline-tabs==2021.8.17b10 sphinx-panels==0.6.0 diff --git a/docs/resources/prerelease_information.md b/docs/resources/prerelease_information.md index 2fc0974f3a40bd..1e8f0f3fb76a6b 100644 --- a/docs/resources/prerelease_information.md +++ b/docs/resources/prerelease_information.md @@ -15,11 +15,40 @@ a general changelog and the schedule for all versions for the current year. * Introduced to allow early testing and get early feedback from the community. -.. dropdown:: OpenVINO Toolkit 2023.0.0.dev20230217 + +.. dropdown:: OpenVINO Toolkit 2023.0.0.dev20230229 :open: :animate: fade-in-slide-down :color: primary + * Added remote tensor support (accepting tensor located in graph memory) in C API 2.0 + * Introduced model caching on GPU. Model Caching which reduces First Inference Latency (FIL) is + extended to work as a single method on both CPU and GPU plug-ins. + * Added the post-training Accuracy-Aware Quantization mechanism for OpenVINO IR. By using this mechanism + the user can define the accuracy drop criteria and NNCF will consider it during the quantization. + * CPU plugin migrated to OneDNN 3.1. + * AUTO supports fall-back to CPU device in case of run-time failure of networks on accelerator devices. + * AUTO supports the option to disable CPU as the initial acceleration device to speed up first-inference latency. + * Implementing a parameter enabling network inference independently of IR precision. The default mode will be FP16 + with FP32 as an option to increase accuracy. General hints (performance vs accuracy) to be provided. + * Adding support for new model use cases or optimizing the existing support (better accuracy or performance). + * Performance on dGPU optimized with Intel oneDNN v3.1, especially for transformer models. + * Dynamic shapes supported on iGPU and dGPU for Transformer(NLP) models. Not all dynamic models are supported but model coverage will be expanded in future releases. + * Performance improvement for Transformer models for NLP pipelines on CPU. + * Models: + + * Enabled MLPerf RNN-T model. + * Enabled Detectron2 MaskRCNN. + * Enabled OpenSeeFace models. + * Enabled Clip model. + * Optimized WeNet model. + + + +.. dropdown:: OpenVINO Toolkit 2023.0.0.dev20230217 + :animate: fade-in-slide-down + :color: secondary + OpenVINO™ repository tag: `2023.0.0.dev20230217 `__ * Enabled PaddlePaddle Framework 2.4 diff --git a/docs/snippets/CMakeLists.txt b/docs/snippets/CMakeLists.txt index 425e7475038c8a..4d824e7c2bc32f 100644 --- a/docs/snippets/CMakeLists.txt +++ b/docs/snippets/CMakeLists.txt @@ -60,7 +60,10 @@ target_include_directories(${TARGET_NAME} PRIVATE "${OpenVINO_SOURCE_DIR}/src/in "${OpenVINO_SOURCE_DIR}/src/common/util/include" "${OpenVINO_SOURCE_DIR}/src/common/low_precision_transformations/include" "${OpenVINO_SOURCE_DIR}/src/frontends/common/include" - "${OpenVINO_SOURCE_DIR}/src/core/template_extension/new/") + "${OpenVINO_SOURCE_DIR}/src/core/template_extension/new" + "${OpenVINO_SOURCE_DIR}/src/frontends/onnx/frontend/include" + "${OpenVINO_SOURCE_DIR}/src/frontends/tensorflow/include") + ov_mark_target_as_cc(${TARGET_NAME}) if(TARGET OpenCL::OpenCL) diff --git a/docs/snippets/cpu/Bfloat16Inference1.cpp b/docs/snippets/cpu/Bfloat16Inference1.cpp index 58f42ebfcaf779..51850c6018db69 100644 --- a/docs/snippets/cpu/Bfloat16Inference1.cpp +++ b/docs/snippets/cpu/Bfloat16Inference1.cpp @@ -6,7 +6,7 @@ using namespace InferenceEngine; ov::Core core; auto network = core.read_model("sample.xml"); auto exec_network = core.compile_model(network, "CPU"); -auto inference_precision = exec_network.get_property(ov::inference_precision); +auto inference_precision = exec_network.get_property(ov::hint::inference_precision); //! [part1] return 0; diff --git a/docs/snippets/cpu/Bfloat16Inference2.cpp b/docs/snippets/cpu/Bfloat16Inference2.cpp index 762329269fc7f5..c06a6491b89432 100644 --- a/docs/snippets/cpu/Bfloat16Inference2.cpp +++ b/docs/snippets/cpu/Bfloat16Inference2.cpp @@ -4,7 +4,7 @@ int main() { using namespace InferenceEngine; //! [part2] ov::Core core; -core.set_property("CPU", ov::inference_precision(ov::element::f32)); +core.set_property("CPU", ov::hint::inference_precision(ov::element::f32)); //! [part2] return 0; diff --git a/docs/snippets/gpu/context_sharing_va_c.cpp b/docs/snippets/gpu/context_sharing_va_c.cpp new file mode 100644 index 00000000000000..4501087a08c608 --- /dev/null +++ b/docs/snippets/gpu/context_sharing_va_c.cpp @@ -0,0 +1,155 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#ifdef ENABLE_LIBVA +#include +#include +#include + +VADisplay get_va_display(); +VASurfaceID decode_va_surface(); + +int main() { + ov_core_t* core = NULL; + ov_model_t* model = NULL; + ov_compiled_model_t* compiled_model = NULL; + ov_infer_request_t* infer_request = NULL; + ov_remote_context_t* shared_va_context = NULL; + ov_tensor_t* remote_tensor = NULL; + ov_preprocess_prepostprocessor_t* preprocess = NULL; + ov_preprocess_input_info_t* preprocess_input_info = NULL; + ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info = NULL; + ov_preprocess_preprocess_steps_t* preprocess_input_steps = NULL; + ov_preprocess_input_model_info_t* preprocess_input_model_info = NULL; + ov_layout_t* layout = NULL; + ov_model_t* new_model = NULL; + + ov_output_const_port_t* input_port = NULL; + char* in_tensor_name = NULL; + char* out_tensor_name = NULL; + ov_shape_t* input_shape = NULL; + ov_element_type_e input_type; + + const int height = 480; + const int width = 640; + + // initialize the objects + ov_core_create(&core); + ov_core_read_model(core, "model.xml", "model.bin", &model); + + // ... + + //! [context_sharing_va] + + // ... + + ov_preprocess_prepostprocessor_create(model, &preprocess); + ov_preprocess_prepostprocessor_get_input_info(preprocess, &preprocess_input_info); + ov_preprocess_input_info_get_tensor_info(preprocess_input_info, &preprocess_input_tensor_info); + ov_preprocess_input_tensor_info_set_element_type(preprocess_input_tensor_info, U8); + ov_preprocess_input_tensor_info_set_color_format_with_subname(preprocess_input_tensor_info, + NV12_TWO_PLANES, + 2, + "y", + "uv"); + ov_preprocess_input_tensor_info_set_memory_type(preprocess_input_tensor_info, "GPU_SURFACE"); + ov_preprocess_input_tensor_info_set_spatial_static_shape(preprocess_input_tensor_info, height, width); + ov_preprocess_input_info_get_preprocess_steps(preprocess_input_info, &preprocess_input_steps); + ov_preprocess_preprocess_steps_convert_color(preprocess_input_steps, BGR); + ov_preprocess_preprocess_steps_resize(preprocess_input_steps, RESIZE_LINEAR); + ov_preprocess_input_info_get_model_info(preprocess_input_info, &preprocess_input_model_info); + ov_layout_create("NCHW", &layout); + ov_preprocess_input_model_info_set_layout(preprocess_input_model_info, layout); + ov_preprocess_prepostprocessor_build(preprocess, &new_model); + + VADisplay display = get_va_display(); + // create the shared context object + ov_core_create_context(core, + "GPU", + 4, + &shared_va_context, + ov_property_key_intel_gpu_context_type, + "VA_SHARED", + ov_property_key_intel_gpu_va_device, + display); + + // compile model within a shared context + ov_core_compile_model_with_context(core, new_model, shared_va_context, 0, &compiled_model); + + ov_output_const_port_t* port_0 = NULL; + char* input_name_0 = NULL; + ov_model_const_input_by_index(new_model, 0, &port_0); + ov_port_get_any_name(port_0, &input_name_0); + + ov_output_const_port_t* port_1 = NULL; + char* input_name_1 = NULL; + ov_model_const_input_by_index(new_model, 1, &port_1); + ov_port_get_any_name(port_1, &input_name_1); + + ov_shape_t shape_y = {0, NULL}; + ov_shape_t shape_uv = {0, NULL}; + ov_const_port_get_shape(port_0, &shape_y); + ov_const_port_get_shape(port_1, &shape_uv); + + // execute decoding and obtain decoded surface handle + VASurfaceID va_surface = decode_va_surface(); + // ... + //wrap decoder output into RemoteBlobs and set it as inference input + + ov_tensor_t* remote_tensor_y = NULL; + ov_tensor_t* remote_tensor_uv = NULL; + ov_remote_context_create_tensor(shared_va_context, + U8, + shape_y, + 6, + &remote_tensor_y, + ov_property_key_intel_gpu_shared_mem_type, + "VA_SURFACE", + ov_property_key_intel_gpu_dev_object_handle, + va_surface, + ov_property_key_intel_gpu_va_plane, + 0); + ov_remote_context_create_tensor(shared_va_context, + U8, + shape_uv, + 6, + &remote_tensor_uv, + ov_property_key_intel_gpu_shared_mem_type, + "VA_SURFACE", + ov_property_key_intel_gpu_dev_object_handle, + va_surface, + ov_property_key_intel_gpu_va_plane, + 1); + + ov_compiled_model_create_infer_request(compiled_model, &infer_request); + ov_infer_request_set_tensor(infer_request, input_name_0, remote_tensor_y); + ov_infer_request_set_tensor(infer_request, input_name_1, remote_tensor_uv); + ov_infer_request_infer(infer_request); + //! [context_sharing_va] + + // deinitialization + ov_free(input_name_0); + ov_free(input_name_1); + ov_output_const_port_free(port_0); + ov_output_const_port_free(port_1); + ov_layout_free(layout); + ov_preprocess_input_model_info_free(preprocess_input_model_info); + ov_preprocess_preprocess_steps_free(preprocess_input_steps); + ov_preprocess_input_tensor_info_free(preprocess_input_tensor_info); + ov_preprocess_input_info_free(preprocess_input_info); + ov_model_free(new_model); + ov_preprocess_prepostprocessor_free(preprocess); + ov_tensor_free(remote_tensor_y); + ov_tensor_free(remote_tensor_uv); + ov_shape_free(&shape_y); + ov_shape_free(&shape_uv); + ov_infer_request_free(infer_request); + ov_compiled_model_free(compiled_model); + ov_model_free(model); + ov_remote_context_free(shared_va_context); + ov_core_free(core); + + return 0; +} +#endif // ENABLE_LIBVA diff --git a/docs/snippets/gpu/preprocessing_nv12_two_planes_c.cpp b/docs/snippets/gpu/preprocessing_nv12_two_planes_c.cpp new file mode 100644 index 00000000000000..826af8ddffe32b --- /dev/null +++ b/docs/snippets/gpu/preprocessing_nv12_two_planes_c.cpp @@ -0,0 +1,126 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include + +cl::Image2D get_y_image(); +cl::Image2D get_uv_image(); + +int main() { + ov_core_t* core = NULL; + ov_model_t* model = NULL; + ov_compiled_model_t* compiled_model = NULL; + ov_infer_request_t* infer_request = NULL; + ov_preprocess_prepostprocessor_t* preprocess = NULL; + ov_preprocess_input_info_t* preprocess_input_info = NULL; + ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info = NULL; + ov_preprocess_preprocess_steps_t* preprocess_input_steps = NULL; + ov_preprocess_input_model_info_t* preprocess_input_model_info = NULL; + ov_layout_t* layout = NULL; + ov_model_t* model_with_preproc = NULL; + ov_remote_context_t* gpu_context = NULL; + char* input_name0 = NULL; + char* input_name1 = NULL; + ov_output_const_port_t* input_port0 = NULL; + ov_output_const_port_t* input_port1 = NULL; + size_t height = 480; + size_t width = 640; + + ov_core_create(&core); + ov_core_read_model(core, "model.xml", "model.bin", &model); + + //! [init_preproc] + ov_preprocess_prepostprocessor_create(model, &preprocess); + ov_preprocess_prepostprocessor_get_input_info(preprocess, &preprocess_input_info); + ov_preprocess_input_info_get_tensor_info(preprocess_input_info, &preprocess_input_tensor_info); + ov_preprocess_input_tensor_info_set_element_type(preprocess_input_tensor_info, ov_element_type_e::U8); + ov_preprocess_input_tensor_info_set_color_format_with_subname(preprocess_input_tensor_info, + ov_color_format_e::NV12_TWO_PLANES, + 2, + "y", + "uv"); + ov_preprocess_input_tensor_info_set_memory_type(preprocess_input_tensor_info, "GPU_SURFACE"); + ov_preprocess_input_tensor_info_set_spatial_static_shape(preprocess_input_tensor_info, height, width); + ov_preprocess_input_info_get_preprocess_steps(preprocess_input_info, &preprocess_input_steps); + ov_preprocess_preprocess_steps_convert_color(preprocess_input_steps, ov_color_format_e::BGR); + ov_preprocess_preprocess_steps_resize(preprocess_input_steps, RESIZE_LINEAR); + ov_preprocess_input_info_get_model_info(preprocess_input_info, &preprocess_input_model_info); + ov_layout_create("NCHW", &layout); + ov_preprocess_input_model_info_set_layout(preprocess_input_model_info, layout); + ov_preprocess_prepostprocessor_build(preprocess, &model_with_preproc); + //! [init_preproc] + + ov_core_compile_model(core, model_with_preproc, "GPU", 0, &compiled_model); + ov_compiled_model_get_context(compiled_model, &gpu_context); + ov_compiled_model_create_infer_request(compiled_model, &infer_request); + + { + //! [single_batch] + ov_model_const_input_by_index(model, 0, &input_port0); + ov_model_const_input_by_index(model, 1, &input_port1); + ov_port_get_any_name(input_port0, &input_name0); + ov_port_get_any_name(input_port1, &input_name1); + + ov_shape_t shape_y, shape_uv; + ov_tensor_t* remote_tensor_y = NULL; + ov_tensor_t* remote_tensor_uv = NULL; + ov_const_port_get_shape(input_port0, &shape_y); + ov_const_port_get_shape(input_port1, &shape_uv); + + cl::Image2D image_y = get_y_image(); + cl::Image2D image_uv = get_uv_image(); + ov_remote_context_create_tensor(gpu_context, + ov_element_type_e::U8, + shape_y, + 4, + &remote_tensor_y, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_IMAGE2D", + ov_property_key_intel_gpu_mem_handle, + image_y.get()); + + ov_remote_context_create_tensor(gpu_context, + ov_element_type_e::U8, + shape_uv, + 4, + &remote_tensor_y, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_IMAGE2D", + ov_property_key_intel_gpu_mem_handle, + image_uv.get()); + + ov_infer_request_set_tensor(infer_request, input_name0, remote_tensor_y); + ov_infer_request_set_tensor(infer_request, input_name1, remote_tensor_uv); + ov_infer_request_infer(infer_request); + //! [single_batch] + + ov_free(input_name0); + ov_free(input_name1); + ov_output_const_port_free(input_port0); + ov_output_const_port_free(input_port1); + + ov_layout_free(layout); + ov_preprocess_input_model_info_free(preprocess_input_model_info); + ov_preprocess_preprocess_steps_free(preprocess_input_steps); + ov_preprocess_input_tensor_info_free(preprocess_input_tensor_info); + ov_preprocess_input_info_free(preprocess_input_info); + ov_preprocess_prepostprocessor_free(preprocess); + + ov_tensor_free(remote_tensor_y); + ov_tensor_free(remote_tensor_uv); + ov_shape_free(&shape_y); + ov_shape_free(&shape_uv); + + ov_infer_request_free(infer_request); + ov_compiled_model_free(compiled_model); + ov_model_free(model); + ov_model_free(model_with_preproc); + ov_remote_context_free(gpu_context); + ov_core_free(core); + } + + return 0; +} diff --git a/docs/snippets/gpu/remote_objects_creation_c.cpp b/docs/snippets/gpu/remote_objects_creation_c.cpp new file mode 100644 index 00000000000000..c870da6bb06a26 --- /dev/null +++ b/docs/snippets/gpu/remote_objects_creation_c.cpp @@ -0,0 +1,283 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include + +#ifdef WIN32 +typedef void* ID3D11Device; +#elif defined(ENABLE_LIBVA) +#include +#endif + +void* allocate_usm_buffer(size_t size); +cl_mem allocate_cl_mem(size_t size); +cl_context get_cl_context(); +cl_command_queue get_cl_queue(); +cl::Buffer allocate_buffer(size_t size); +cl::Image2D allocate_image(size_t size); + +#ifdef WIN32 +ID3D11Device* get_d3d_device(); +#elif defined(ENABLE_LIBVA) +VADisplay get_va_display(); +#endif + +int main() { + ov_core_t* core = NULL; + ov_model_t* model = NULL; + ov_compiled_model_t* compiled_model = NULL; + ov_remote_context_t* gpu_context = NULL; + ov_tensor_t* remote_tensor = NULL; + + ov_output_const_port* input_port = NULL; + char* in_tensor_name = NULL; + char* out_tensor_name = NULL; + ov_shape_t input_shape; + ov_element_type_e input_type; + + ov_core_create(&core); + ov_core_read_model(core, "model.xml", "model.bin", &model); + + ov_model_const_input(model, &input_port); + ov_port_get_any_name(input_port, &in_tensor_name); + ov_const_port_get_shape(input_port, &input_shape); + ov_port_get_element_type(input_port, &input_type); + size_t input_size = 1; + for (auto i = 0; i < input_shape.rank; i++) + input_size *= input_shape.dims[i]; + + ov_core_compile_model(core, model, "GPU", 0, &compiled_model); + ov_compiled_model_get_context(compiled_model, &gpu_context); + +{ + //! [wrap_usm_pointer] + void* shared_buffer = allocate_usm_buffer(input_size); + ov_remote_context_create_tensor(gpu_context, + input_type, + input_shape, + 4, + &remote_tensor, + ov_property_key_intel_gpu_shared_mem_type, + "USM_USER_BUFFER", + ov_property_key_intel_gpu_mem_handle, + shared_buffer); + //! [wrap_usm_pointer] +} + +{ + //! [wrap_cl_mem] + cl_mem shared_buffer = allocate_cl_mem(input_size); + ov_remote_context_create_tensor(gpu_context, + input_type, + input_shape, + 4, + &remote_tensor, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_BUFFER", + ov_property_key_intel_gpu_mem_handle, + shared_buffer); + //! [wrap_cl_mem] +} + +{ + //! [wrap_cl_buffer] + cl::Buffer shared_buffer = allocate_buffer(input_size); + ov_remote_context_create_tensor(gpu_context, + input_type, + input_shape, + 4, + &remote_tensor, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_BUFFER", + ov_property_key_intel_gpu_mem_handle, + shared_buffer.get()); + //! [wrap_cl_buffer] +} + +{ + //! [wrap_cl_image] + cl::Image2D shared_buffer = allocate_image(input_size); + ov_remote_context_create_tensor(gpu_context, + input_type, + input_shape, + 4, + &remote_tensor, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_IMAGE2D", + ov_property_key_intel_gpu_mem_handle, + shared_buffer.get()); + //! [wrap_cl_image] +} + +{ + //! [allocate_usm_device] + ov_remote_context_create_tensor(gpu_context, + input_type, + input_shape, + 2, + &remote_tensor, + ov_property_key_intel_gpu_shared_mem_type, + "USM_USER_BUFFER"); + // Extract raw usm pointer from remote tensor + void* usm_ptr = NULL; + ov_tensor_data(remote_tensor, &usm_ptr); + //! [allocate_usm_device] +} + +{ + //! [allocate_usm_host] + ov_remote_context_create_tensor(gpu_context, + input_type, + input_shape, + 2, + &remote_tensor, + ov_property_key_intel_gpu_shared_mem_type, + "USM_HOST_BUFFER"); + // Extract raw usm pointer from remote tensor + void* usm_ptr = NULL; + ov_tensor_data(remote_tensor, &usm_ptr); + //! [allocate_usm_host] +} + +{ + int64_t width = 1024; + int64_t height = 768; + + int64_t y_plane_size = width * height; + int64_t uv_plane_size = width * height / 2; + + ov_shape_t shape_y = {0, NULL}; + int64_t dims_y[4] = {1, 1, height, width}; + ov_shape_t shape_uv = {0, NULL}; + int64_t dims_uv[4] = {1, 2, height / 2, width / 2}; + ov_tensor_t* remote_tensor_y = NULL; + ov_tensor_t* remote_tensor_uv = NULL; + + ov_shape_create(4, dims_y, &shape_y); + ov_shape_create(4, dims_uv, &shape_uv); + + //! [create_nv12_surface] + cl::Image2D y_plane_surface = allocate_image(y_plane_size); + cl::Image2D uv_plane_surface = allocate_image(uv_plane_size); + + ov_remote_context_create_tensor(gpu_context, + input_type, + shape_y, + 4, + &remote_tensor_y, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_IMAGE2D", + ov_property_key_intel_gpu_mem_handle, + y_plane_surface.get()); + + ov_remote_context_create_tensor(gpu_context, + input_type, + shape_uv, + 4, + &remote_tensor_uv, + ov_property_key_intel_gpu_shared_mem_type, + "OCL_IMAGE2D", + ov_property_key_intel_gpu_mem_handle, + uv_plane_surface.get()); + + ov_tensor_free(remote_tensor_y); + ov_tensor_free(remote_tensor_uv); + ov_shape_free(&shape_y); + ov_shape_free(&shape_uv); + //! [create_nv12_surface] +} + +{ + //! [context_from_cl_context] + cl_context cl_context = get_cl_context(); + ov_core_create_context(core, + "GPU", + 4, + &gpu_context, + ov_property_key_intel_gpu_context_type, + "OCL", + ov_property_key_intel_gpu_ocl_context, + cl_context); + //! [context_from_cl_context] +} + +{ + //! [context_from_cl_queue] + cl_command_queue cl_queue = get_cl_queue(); + cl_context cl_context = get_cl_context(); + ov_core_create_context(core, + "GPU", + 6, + &gpu_context, + ov_property_key_intel_gpu_context_type, + "OCL", + ov_property_key_intel_gpu_ocl_context, + cl_context, + ov_property_key_intel_gpu_ocl_queue, + cl_queue); + //! [context_from_cl_queue] +} + +#ifdef WIN32 +{ + //! [context_from_d3d_device] + ID3D11Device* device = get_d3d_device(); + ov_core_create_context(core, + "GPU", + 4, + &gpu_context, + ov_property_key_intel_gpu_context_type, + "VA_SHARED", + ov_property_key_intel_gpu_va_device, + device); + //! [context_from_d3d_device] +} +#elif defined(ENABLE_LIBVA) +{ + //! [context_from_va_display] + VADisplay display = get_va_display(); + ov_core_create_context(core, + "GPU", + 4, + &gpu_context, + ov_property_key_intel_gpu_context_type, + "VA_SHARED", + ov_property_key_intel_gpu_va_device, + display); + //! [context_from_va_display] +} +#endif +{ + //! [default_context_from_core] + ov_core_get_default_context(core, "GPU", &gpu_context); + // Extract ocl context handle from RemoteContext + size_t size = 0; + char* params = nullptr; + // params is format like: "CONTEXT_TYPE OCL OCL_CONTEXT 0x5583b2ec7b40 OCL_QUEUE 0x5583b2e98ff0" + // You need parse it. + ov_remote_context_get_params(gpu_context, &size, ¶ms); + //! [default_context_from_core] +} + +{ + //! [default_context_from_model] + ov_compiled_model_get_context(compiled_model, &gpu_context); + // Extract ocl context handle from RemoteContext + size_t size = 0; + char* params = nullptr; + // params is format like: "CONTEXT_TYPE OCL OCL_CONTEXT 0x5583b2ec7b40 OCL_QUEUE 0x5583b2e98ff0" + // You need parse it. + ov_remote_context_get_params(gpu_context, &size, ¶ms); + //! [default_context_from_model] +} + +ov_compiled_model_free(compiled_model); +ov_model_free(model); +ov_remote_context_free(gpu_context); +ov_core_free(core); + +return 0; +} diff --git a/docs/snippets/ov_extensions.cpp b/docs/snippets/ov_extensions.cpp index 895ee6e59d9e55..a9c11e421f5e96 100644 --- a/docs/snippets/ov_extensions.cpp +++ b/docs/snippets/ov_extensions.cpp @@ -17,6 +17,12 @@ #include //! [frontend_extension_ThresholdedReLU_header] +//! [frontend_extension_framework_map_macro_headers] +#include +#include +#include +//! [frontend_extension_framework_map_macro_headers] + #include //! [frontend_extension_CustomOperation] @@ -40,6 +46,27 @@ class CustomOperation : public ov::op::Op { std::shared_ptr clone_with_new_inputs(const ov::OutputVector&) const override { return nullptr; } }; +//! [frontend_extension_framework_map_macro_CustomOp] +class CustomOp : public ov::op::Op { + std::string m_mode; + int m_axis; + +public: + OPENVINO_OP("CustomOp"); + OPENVINO_FRAMEWORK_MAP(onnx, "CustomOp", { {"m_mode", "mode"} }, { {"m_axis", -1} }); + OPENVINO_FRAMEWORK_MAP(tensorflow, "CustomOpV3", { {"m_axis", "axis"} }, { {"m_mode", "linear"} }); + + bool visit_attributes(ov::AttributeVisitor& visitor) override { + visitor.on_attribute("m_mode", m_mode); + visitor.on_attribute("m_axis", m_axis); + return true; + } + + // ... implement other required methods +//! [frontend_extension_framework_map_macro_CustomOp] + std::shared_ptr clone_with_new_inputs(const ov::OutputVector&) const override { return nullptr; } +}; + int main() { { //! [add_extension] @@ -94,14 +121,14 @@ core.add_extension(ov::frontend::OpExtension()); //! [frontend_extension_CustomOperation_rename] core.add_extension(ov::frontend::OpExtension( - { {"attr1", "fw_attr1"}, {"attr2", "fw_attr2"} }, + std::map{ {"attr1", "fw_attr1"}, {"attr2", "fw_attr2"} }, {} )); //! [frontend_extension_CustomOperation_rename] //! [frontend_extension_CustomOperation_rename_set] core.add_extension(ov::frontend::OpExtension( - { {"attr1", "fw_attr1"} }, + std::map{ {"attr1", "fw_attr1"} }, { {"attr2", 5} } )); //! [frontend_extension_CustomOperation_rename_set] @@ -125,5 +152,12 @@ ov::Core core; core.add_extension("openvino_template_extension.so"); //! [add_extension_lib] } + +{ +//! [frontend_extension_framework_map_macro_add_extension] +ov::Core core; +core.add_extension(ov::frontend::OpExtension()); +//! [frontend_extension_framework_map_macro_add_extension] +} return 0; } diff --git a/docs/snippets/ov_hetero.cpp b/docs/snippets/ov_hetero.cpp index 2f5cf3f5c9eec3..791340afff56ef 100644 --- a/docs/snippets/ov_hetero.cpp +++ b/docs/snippets/ov_hetero.cpp @@ -49,7 +49,7 @@ auto compiled_model = core.compile_model(model, "HETERO", // profiling is enabled only for GPU ov::device::properties("GPU", ov::enable_profiling(true)), // FP32 inference precision only for CPU - ov::device::properties("CPU", ov::inference_precision(ov::element::f32)) + ov::device::properties("CPU", ov::hint::inference_precision(ov::element::f32)) ); //! [configure_fallback_devices] } diff --git a/docs/snippets/ov_properties_api.cpp b/docs/snippets/ov_properties_api.cpp index 7815291ee7b90e..e5f1ff7648fcf5 100644 --- a/docs/snippets/ov_properties_api.cpp +++ b/docs/snippets/ov_properties_api.cpp @@ -19,7 +19,7 @@ auto model = core.read_model("sample.xml"); //! [compile_model_with_property] auto compiled_model = core.compile_model(model, "CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::inference_precision(ov::element::f32)); + ov::hint::inference_precision(ov::element::f32)); //! [compile_model_with_property] } diff --git a/docs/snippets/ov_properties_migration.cpp b/docs/snippets/ov_properties_migration.cpp index 6ee3279395cc52..7be66b4a1d1266 100644 --- a/docs/snippets/ov_properties_migration.cpp +++ b/docs/snippets/ov_properties_migration.cpp @@ -25,7 +25,7 @@ auto model = core.read_model("sample.xml"); auto compiled_model = core.compile_model(model, "MULTI", ov::device::priorities("GPU", "CPU"), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::inference_precision(ov::element::f32)); + ov::hint::inference_precision(ov::element::f32)); //! [core_compile_model] //! [compiled_model_set_property] diff --git a/docs/tutorials.md b/docs/tutorials.md index b7f69cb75e48d7..aebf5c71aac3f1 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -105,7 +105,7 @@ Tutorials that explain how to optimize and quantize models with OpenVINO tools. +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `102-pytorch-onnx-to-openvino `__ | Convert PyTorch models to OpenVINO IR. | |n102-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `103-paddle-onnx-to-openvino `__ |br| |n103| | Convert PaddlePaddle models to OpenVINO IR. | |n103-img1| | +| `103-paddle-onnx-to-openvino `__ |br| |n103| | Convert PaddlePaddle models to OpenVINO IR. | |n103-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `104-model-tools `__ |br| |n104| | Download, convert and benchmark models from Open Model Zoo. | |n104-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -117,24 +117,26 @@ Tutorials that explain how to optimize and quantize models with OpenVINO tools. +==============================================================================================================================+==================================================================================================================================+ | `105-language-quantize-bert `__ | Optimize and quantize a pre-trained BERT model | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ - | `106-auto-device `__ | Demonstrates how to use AUTO Device | + | `106-auto-device `__ |br| |n106| | Demonstrates how to use AUTO Device | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ | `107-speech-recognition-quantization `__ | Optimize and quantize a pre-trained Wav2Vec2 speech model | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ - | `110-ct-segmentation-quantize `__ | Quantize a kidney segmentation model and show live inference | + | `110-ct-segmentation-quantize `__ |br| |n110| | Quantize a kidney segmentation model and show live inference | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ | `111-detection-quantization `__ |br| |n111| | Quantize an object detection model | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ | `112-pytorch-post-training-quantization-nncf `__ | Use Neural Network Compression Framework (NNCF) to quantize PyTorch model in post-training mode (without model fine-tuning) | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ - | `113-image-classification-quantization `__ | Quantize mobilenet image classification | + | `113-image-classification-quantization `__ |br| |n113| | Quantize mobilenet image classification | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ - | `114-quantization-simplified-mode `__ | Quantize Image Classification Models with POT in Simplified Mode | + | `114-quantization-simplified-mode `__ |br| |n114| | Quantize Image Classification Models with POT in Simplified Mode | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ - | `115-async-api `__ | Use Asynchronous Execution to Improve Data Pipelining | + | `115-async-api `__ |br| |n115| | Use Asynchronous Execution to Improve Data Pipelining | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ | `116-sparsity-optimization `__ | Improve performance of sparse Transformer models | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ + | `117-model-server `__ | Improve performance of sparse Transformer models | + +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ | `118-optimize-preprocessing `__ | Improve performance of image preprocessing step | +------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ @@ -151,19 +153,17 @@ Demos that demonstrate inference on a particular model. +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | Notebook | Description | Preview | +===============================================================================================================================+============================================================================================================================================+===========================================+ -| `210-ct-scan-live-inference `__ |br| |n210| | Show live inference on segmentation of CT-scan data. | |n210-img1| | +| `205-vision-background-removal `__ |br| |n205| | Remove and replace the background in an image using salient object detection. | |n205-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `211-speech-to-text `__ |br| |n211| | Run inference on speech-to-text recognition model. | |n211-img1| | +| `209-handwritten-ocr `__ |br| |n209| | OCR for handwritten simplified Chinese and Japanese. | |n209-img1| |br| |chinese-text| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `208-optical-character-recognition `__ | Annotate text on images using text recognition resnet. | |n208-img1| | +| `211-speech-to-text `__ |br| |n211| | Run inference on speech-to-text recognition model. | |n211-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `209-handwritten-ocr `__ |br| |n209| | OCR for handwritten simplified Chinese and Japanese. | |n209-img1| |br| |chinese-text| | +| `215-image-inpainting `__ |br| |n215| | Fill missing pixels with image in-painting. | |n215-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `218-vehicle-detection-and-recognition `__ | Use pre-trained models to detect and recognize vehicles and their attributes with OpenVINO. | |n218-img1| | +| `218-vehicle-detection-and-recognition `__ |br| |n218| | Use pre-trained models to detect and recognize vehicles and their attributes with OpenVINO. | |n218-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - - .. dropdown:: Explore more notebooks below. +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -177,31 +177,29 @@ Demos that demonstrate inference on a particular model. +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `203-meter-reader `__ |br| |n203| | PaddlePaddle pre-trained models to read industrial meter's value | |n203-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `204-named-entity-recognition `__ |br| |n204| | Perform named entity recognition on simple text. | |n204-img1| | + | `206-vision-paddlegan-anime `__ | Turn an image into anime using a GAN. | |n206-img1| → |n206-img2| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `205-vision-background-removal `__ |br| |n205| | Remove and replace the background in an image using salient object detection. | |n205-img1| | + | `207-vision-paddlegan-superresolution `__ | Upscale small images with superresolution using a PaddleGAN model. | |n207-img1| → |n207-img2| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `206-vision-paddlegan-anime `__ |br| |n206| | Turn an image into anime using a GAN. | |n206-img1| → |n206-img2| | + | `208-optical-character-recognition `__ | Annotate text on images using text recognition resnet. | |n208-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `207-vision-paddlegan-superresolution `__ |br| |n207| | Upscale small images with superresolution using a PaddleGAN model. | |n207-img1| → |n207-img2| | + | `212-pyannote-speaker-diarization `__ | Run inference on speaker diarization pipeline | |n212-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `212-onnx-style-transfer `__ |br| |n212| | Transform images to five different styles with neural style transfer. | |n212-img1| → |n212-img2| | + | `213-question-answering `__ |br| |n213| | Answer your questions basing on a context. | |n213-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `214-vision-paddle-classification `__ |br| |n214| | PaddlePaddle Image Classification with OpenVINO. | | + | `214-grammar-correction `__ | Grammatical Error Correction with OpenVINO | | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `215-image-inpainting `__ | Fill missing pixels with image in-painting. | |n215-img1| | - +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `216-license-plate-recognition `__ | Recognize Chinese license plates in traffic. | |n216-img1| | + | `216-license-plate-recognition `__ |br| |n216| | Recognize Chinese license plates in traffic. | |n216-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `217-vision-deblur `__ |br| |n217| | Deblur Images with DeblurGAN-v2. | |n217-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `219-knowledge-graphs-conve `__ | Optimize the knowledge graph embeddings model (ConvE) with OpenVINO | | + | `219-knowledge-graphs-conve `__ |br| |n219| | Optimize the knowledge graph embeddings model (ConvE) with OpenVINO | | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `220-yolov5-accuracy-check-and-quantization `__ | Quantize the Ultralytics YOLOv5 model and check accuracy using the OpenVINO POT API | |n220-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `221-machine-translation `__ | Real-time translation from English to German | | + | `221-machine-translation `__ |br| |n221| | Real-time translation from English to German | | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `222-vision-image-colorization `__ | Use pre-trained models to colorize black & white images using OpenVINO | |n222-img1| | + | `222-vision-image-colorization `__ |br| |n222| | Use pre-trained models to colorize black & white images using OpenVINO | |n222-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `223-gpt2-text-prediction `__ | Use GPT-2 to perform text prediction on an input sequence | |n223-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -215,7 +213,7 @@ Demos that demonstrate inference on a particular model. +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `228-clip-zero-shot-image-classification `__ | Perform Zero-shot Image Classification with CLIP and OpenVINO | |n228-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ - | `229-distilbert-sequence-classification `__ | Sequence Classification with OpenVINO | |n229-img1| | + | `229-distilbert-sequence-classification `__ |br| |n229| | Sequence Classification with OpenVINO | |n229-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `230-yolov8-optimization `__ | Optimize YOLOv8 using NNCF PTQ API | |n230-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -296,11 +294,11 @@ The following tutorials are guaranteed to provide a great experience with infere +===============================================================================================================================+============================================================================================================================================+===========================================+ | `Vision-monodepth `__ |br| |n201| | Monocular depth estimation with images and video. | |n201-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `CT-scan-live-inference `__ |br| |n210| | Show live inference on segmentation of CT-scan data. | |n210-img1| | +| `Vision-background-removal `__ |br| |n205| | Remove and replace the background in an image using salient object detection. | |n205-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `Object-detection-webcam `__ |br| |n401| | Object detection with a webcam or video file. | |n401-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ -| `Pose-estimation-webcam `__ |br| |n402| | Human pose estimation with a webcam or video file. | |n402-img1| | +| `Pose-estimation-webcam `__ |br| |n402| | Human pose estimation with a webcam or video file. | |n402-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ | `Action-recognition-webcam `__ |br| |n403| | Human action recognition with a webcam or video file. | |n403-img1| | +-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+ @@ -362,16 +360,6 @@ Made with `contributors-img `__. :target: https://user-images.githubusercontent.com/15709723/127779326-dc14653f-a960-4877-b529-86908a6f2a61.png .. |n104-img1| image:: https://user-images.githubusercontent.com/10940214/157541917-c5455105-b0d9-4adf-91a7-fbc142918015.png :target: https://user-images.githubusercontent.com/10940214/157541917-c5455105-b0d9-4adf-91a7-fbc142918015.png -.. |n210-img1| image:: https://user-images.githubusercontent.com/15709723/134784204-cf8f7800-b84c-47f5-a1d8-25a9afab88f8.gif - :target: https://user-images.githubusercontent.com/15709723/134784204-cf8f7800-b84c-47f5-a1d8-25a9afab88f8.gif -.. |n211-img1| image:: https://user-images.githubusercontent.com/36741649/140987347-279de058-55d7-4772-b013-0f2b12deaa61.png - :target: https://user-images.githubusercontent.com/36741649/140987347-279de058-55d7-4772-b013-0f2b12deaa61.png -.. |n213-img1| image:: https://user-images.githubusercontent.com/4547501/152571639-ace628b2-e3d2-433e-8c28-9a5546d76a86.gif - :target: https://user-images.githubusercontent.com/4547501/152571639-ace628b2-e3d2-433e-8c28-9a5546d76a86.gif -.. |n208-img1| image:: https://user-images.githubusercontent.com/36741649/129315292-a37266dc-dfb2-4749-bca5-2ac9c1e93d64.jpg - :target: https://user-images.githubusercontent.com/36741649/129315292-a37266dc-dfb2-4749-bca5-2ac9c1e93d64.jpg -.. |n209-img1| image:: https://user-images.githubusercontent.com/36741649/132660640-da2211ec-c389-450e-8980-32a75ed14abb.png - :target: https://user-images.githubusercontent.com/36741649/132660640-da2211ec-c389-450e-8980-32a75ed14abb.png .. |n201-img1| image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif :target: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif .. |n202i-img1| image:: https://user-images.githubusercontent.com/36741649/170005347-e4409f9e-ec34-416b-afdf-a9d8185929ca.jpg @@ -388,8 +376,6 @@ Made with `contributors-img `__. :target: https://user-images.githubusercontent.com/15709723/127269258-a8e2c03e-731e-4317-b5b2-ed2ee767ff5e.gif .. |n203-img1| image:: https://user-images.githubusercontent.com/91237924/166135627-194405b0-6c25-4fd8-9ad1-83fb3a00a081.jpg :target: https://user-images.githubusercontent.com/91237924/166135627-194405b0-6c25-4fd8-9ad1-83fb3a00a081.jpg -.. |n204-img1| image:: https://user-images.githubusercontent.com/33627846/169470030-0370963e-6ad8-49e3-be7a-f02a2c677733.gif - :target: https://user-images.githubusercontent.com/33627846/169470030-0370963e-6ad8-49e3-be7a-f02a2c677733.gif .. |n205-img1| image:: https://user-images.githubusercontent.com/15709723/125184237-f4b6cd00-e1d0-11eb-8e3b-d92c9a728372.png :target: https://user-images.githubusercontent.com/15709723/125184237-f4b6cd00-e1d0-11eb-8e3b-d92c9a728372.png .. |n206-img1| image:: https://user-images.githubusercontent.com/15709723/127788059-1f069ae1-8705-4972-b50e-6314a6f36632.jpeg @@ -402,10 +388,16 @@ Made with `contributors-img `__. .. |n207-img2| image:: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :target: https://user-images.githubusercontent.com/36741649/127170593-86976dc3-e5e4-40be-b0a6-206379cd7df5.jpg :width: 130 -.. |n212-img1| image:: https://user-images.githubusercontent.com/77325899/147358090-ff5b21f5-0efb-4aff-8444-9d07add49b92.png - :target: https://user-images.githubusercontent.com/77325899/147358090-ff5b21f5-0efb-4aff-8444-9d07add49b92.png -.. |n212-img2| image:: https://user-images.githubusercontent.com/77325899/147358009-0cf10d51-3150-40cb-a776-074558b98da5.png - :target: https://user-images.githubusercontent.com/77325899/147358009-0cf10d51-3150-40cb-a776-074558b98da5.png +.. |n208-img1| image:: https://user-images.githubusercontent.com/36741649/129315292-a37266dc-dfb2-4749-bca5-2ac9c1e93d64.jpg + :target: https://user-images.githubusercontent.com/36741649/129315292-a37266dc-dfb2-4749-bca5-2ac9c1e93d64.jpg +.. |n209-img1| image:: https://user-images.githubusercontent.com/36741649/132660640-da2211ec-c389-450e-8980-32a75ed14abb.png + :target: https://user-images.githubusercontent.com/36741649/132660640-da2211ec-c389-450e-8980-32a75ed14abb.png +.. |n211-img1| image:: https://user-images.githubusercontent.com/36741649/140987347-279de058-55d7-4772-b013-0f2b12deaa61.png + :target: https://user-images.githubusercontent.com/36741649/140987347-279de058-55d7-4772-b013-0f2b12deaa61.png +.. |n213-img1| image:: https://user-images.githubusercontent.com/4547501/152571639-ace628b2-e3d2-433e-8c28-9a5546d76a86.gif + :target: https://user-images.githubusercontent.com/4547501/152571639-ace628b2-e3d2-433e-8c28-9a5546d76a86.gif +.. |n212-img1| image:: https://user-images.githubusercontent.com/29454499/218432101-0bd0c424-e1d8-46af-ba1d-ee29ed6d1229.png + :target: https://user-images.githubusercontent.com/29454499/218432101-0bd0c424-e1d8-46af-ba1d-ee29ed6d1229.png .. |n215-img1| image:: https://user-images.githubusercontent.com/4547501/167121084-ec58fbdb-b269-4de2-9d4c-253c5b95de1e.png :target: https://user-images.githubusercontent.com/4547501/167121084-ec58fbdb-b269-4de2-9d4c-253c5b95de1e.png .. |n216-img1| image:: https://user-images.githubusercontent.com/70456146/162759539-4a0a996f-dabe-40ea-98d6-85b4dce8511d.png @@ -465,6 +457,7 @@ Made with `contributors-img `__. .. |Apache License Version 2.0| image:: https://img.shields.io/badge/license-Apache_2.0-green.svg :target: https://github.com/openvinotoolkit/openvino_notebooks/blob/main/LICENSE + .. |nbval| image:: https://github.com/openvinotoolkit/openvino_notebooks/actions/workflows/nbval.yml/badge.svg :target: https://github.com/openvinotoolkit/openvino_notebooks/actions/workflows/nbval.yml?query=branch%3Amain .. |nbval-docker| image:: https://github.com/openvinotoolkit/openvino_notebooks/actions/workflows/docker.yml/badge.svg @@ -478,20 +471,26 @@ Made with `contributors-img `__. :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F003-hello-segmentation%2F003-hello-segmentation.ipynb .. |n004| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F004-hello-detection%2F004-hello-detection.ipynb + .. |n101| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F101-tensorflow-to-openvino%2F101-tensorflow-to-openvino.ipynb .. |n103| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F103-paddle-onnx-to-openvino-classification%2F103-paddle-onnx-to-openvino-classification.ipynb .. |n104| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F104-model-tools%2F104-model-tools.ipynb +.. |n106| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F106-auto-device%2F106-auto-device.ipynb +.. |n110| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F110-ct-segmentation-quantize%2F110-ct-scan-live-inference.ipynb .. |n111| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F111-detection-quantization%2F111-detection-quantization.ipynb -.. |n210| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F210-ct-scan-live-inference%2F210-ct-scan-live-inference.ipynb -.. |n211| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F211-speech-to-text%2F211-speech-to-text.ipynb -.. |n213| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F213-question-answering%2F213-question-answering.ipynb +.. |n113| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F113-image-classification-quantization%2F113-image-classification-quantization.ipynb +.. |n114| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F114-quantization-simplified-mode%2F114-quantization-simplified-mode.ipynb +.. |n115| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F115-async-api%2F115-async-api.ipynb + .. |n209| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F209-handwritten-ocr%2F209-handwritten-ocr.ipynb .. |n201| image:: https://mybinder.org/badge_logo.svg @@ -502,20 +501,28 @@ Made with `contributors-img `__. :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F202-vision-superresolution%2F202-vision-superresolution-video.ipynb .. |n203| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F203-meter-reader%2F203-meter-reader.ipynb -.. |n204| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F204-named-entity-recognition%2F204-named-entity-recognition.ipynb .. |n205| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F205-vision-background-removal%2F205-vision-background-removal.ipynb -.. |n206| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F206-vision-paddlegan-anime%2F206-vision-paddlegan-anime.ipynb -.. |n207| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F207-vision-paddlegan-superresolution%2F207-vision-paddlegan-superresolution.ipynb -.. |n212| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F212-onnx-style-transfer%2F212-onnx-style-transfer.ipynb -.. |n214| image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F214-vision-paddle-classification%2F214-vision-paddle-classification.ipynb +.. |n211| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F211-speech-to-text%2F211-speech-to-text.ipynb +.. |n213| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F213-question-answering%2F213-question-answering.ipynb +.. |n215| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F215-image-inpainting%2F215-image-inpainting.ipynb +.. |n216| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F216-license-plate-recognition%2F216-license-plate-recognition.ipynb .. |n217| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/ThanosM97/openvino_notebooks/217-vision-deblur?labpath=notebooks%2F217-vision-deblur%2F217-vision-deblur.ipynb +.. |n218| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F218-vehicle-detection-and-recognition%2F218-vehicle-detection-and-recognition.ipynb +.. |n219| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F219-knowledge-graphs-conve%2F219-knowledge-graphs-conve.ipynb +.. |n221| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F221-machine-translation%2F221-machine-translation.ipynb +.. |n222| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F222-vision-image-colorization%2F222-vision-image-colorization.ipynb +.. |n229| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?labpath=notebooks%2F229-distilbert-sequence-classification%2F229-distilbert-sequence-classification.ipynb .. |n401| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F401-object-detection-webcam%2F401-object-detection.ipynb .. |n402| image:: https://mybinder.org/badge_logo.svg diff --git a/install_build_dependencies.sh b/install_build_dependencies.sh index 542a4d465b10aa..d307327e93b8a7 100755 --- a/install_build_dependencies.sh +++ b/install_build_dependencies.sh @@ -55,6 +55,8 @@ if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ] ; then opencl-headers \ `# GPU plugin extensions` \ libva-dev \ + `# For TF FE saved models` \ + libsnappy-dev \ `# python API` \ python3-pip \ python3-venv \ @@ -116,6 +118,8 @@ elif [ -f /etc/redhat-release ] || grep -q "rhel" /etc/os-release ; then pugixml-devel \ `# GPU plugin dependency` \ libva-devel \ + `# For TF FE saved models` \ + snappy-devel \ `# OpenCL for GPU` \ ocl-icd-devel \ opencl-headers \ @@ -148,11 +152,13 @@ elif [ -f /etc/os-release ] && grep -q "SUSE" /etc/os-release ; then rpmlint \ `# check bash scripts for correctness` \ ShellCheck \ - `# main openvino dependencies` \ + `# main openvino dependencies` \ tbb-devel \ pugixml-devel \ `# GPU plugin dependency` \ - libva-devel \ + libva-devel \ + `# For TF FE saved models` \ + snappy-devel \ `# OpenCL for GPU` \ ocl-icd-devel \ opencl-cpp-headers \ diff --git a/licensing/third-party-programs.txt b/licensing/third-party-programs.txt index dc5c2ae4d7009e..b1fdd9a5ca0bbb 100644 --- a/licensing/third-party-programs.txt +++ b/licensing/third-party-programs.txt @@ -1547,3 +1547,96 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. See the License for the specific language governing permissions and limitations under the License. +------------------------------------------------------------- + +28. Snappy (https://github.com/google/snappy/) + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +------------------------------------------------------------- + +29. Pillow (https://github.com/python-pillow/Pillow) + +The Python Imaging Library (PIL) is + + Copyright © 1997-2011 by Secret Labs AB + Copyright © 1995-2011 by Fredrik Lundh + +Pillow is the friendly PIL fork. It is + + Copyright © 2010-2023 by Jeffrey A. Clark (Alex) and contributors. + +Like PIL, Pillow is licensed under the open source HPND License: + +By obtaining, using, and/or copying this software and/or its associated +documentation, you agree that you have read, understood, and will comply +with the following terms and conditions: + +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appears in all copies, and that +both that copyright notice and this permission notice appear in supporting +documentation, and that the name of Secret Labs AB or the author not be +used in advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, +INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/samples/c/hello_classification/README.md b/samples/c/hello_classification/README.md index 2cbc2b8065df62..4d47bca2c8a948 100644 --- a/samples/c/hello_classification/README.md +++ b/samples/c/hello_classification/README.md @@ -1,98 +1,139 @@ # Hello Classification C Sample {#openvino_inference_engine_ie_bridges_c_samples_hello_classification_README} +@sphinxdirective + This sample demonstrates how to execute an inference of image classification networks like AlexNet and GoogLeNet using Synchronous Inference Request API and input auto-resize feature. Hello Classification C sample application demonstrates how to use the C API from OpenVINO in applications. -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | `ov_get_openvino_version` | Get Openvino API version | -| Basic Infer Flow | `ov_core_create`, `ov_core_read_model`, `ov_core_compile_model`, `ov_compiled_model_create_infer_request`, `ov_infer_request_set_input_tensor_by_index`, `ov_infer_request_get_output_tensor_by_index` | Common API to do inference: read and compile a model, create an infer request, configure input and output tensors | -| Synchronous Infer | `ov_infer_request_infer` | Do synchronous inference | -| Model Operations | `ov_model_const_input`, `ov_model_const_output` | Get inputs and outputs of a model | -| Tensor Operations | `ov_tensor_create_from_host_ptr` | Create a tensor shape | -| Preprocessing | `ov_preprocess_prepostprocessor_create`, `ov_preprocess_prepostprocessor_get_input_info_by_index`, `ov_preprocess_input_info_get_tensor_info`, `ov_preprocess_input_tensor_info_set_from`, `ov_preprocess_input_tensor_info_set_layout`, `ov_preprocess_input_info_get_preprocess_steps`, `ov_preprocess_preprocess_steps_resize`, `ov_preprocess_input_model_info_set_layout`, `ov_preprocess_output_set_element_type`, `ov_preprocess_prepostprocessor_build` | Set image of the original size as input for a model with other input size. Resize and layout conversions are performed automatically by the corresponding plugin just before inference. | - -| Options | Values | -|:--- |:--- -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) -| Model Format | Inference Engine Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) -| Validated images | The sample uses OpenCV\* to [read input image](https://docs.opencv.org/master/d4/da8/group__imgcodecs.html#ga288b8b3da0892bd651fce07b3bbd3a56) (\*.bmp, \*.png) -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/hello_classification/README.md), [Python](../../python/hello_classification/README.md) | - -## How It Works ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Feature | API | Description | ++=====================================+=============================================================+=========================================================================================================================================================================================+ +| OpenVINO Runtime Version | ``ov_get_openvino_version`` | Get Openvino API version | ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Basic Infer Flow | ``ov_core_create``, | Common API to do inference: read and compile a model, create an infer request, configure input and output tensors | +| | ``ov_core_read_model``, | | +| | ``ov_core_compile_model``, | | +| | ``ov_compiled_model_create_infer_request``, | | +| | ``ov_infer_request_set_input_tensor_by_index``, | | +| | ``ov_infer_request_get_output_tensor_by_index`` | | ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Synchronous Infer | ``ov_infer_request_infer`` | Do synchronous inference | ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Operations | ``ov_model_const_input``, | Get inputs and outputs of a model | +| | ``ov_model_const_output`` | + ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Tensor Operations | ``ov_tensor_create_from_host_ptr`` | Create a tensor shape | ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Preprocessing | ``ov_preprocess_prepostprocessor_create``, | Set image of the original size as input for a model with other input size. Resize and layout conversions are performed automatically by the corresponding plugin just before inference. | +| | ``ov_preprocess_prepostprocessor_get_input_info_by_index``, | | +| | ``ov_preprocess_input_info_get_tensor_info``, | | +| | ``ov_preprocess_input_tensor_info_set_from``, | | +| | ``ov_preprocess_input_tensor_info_set_layout``, | | +| | ``ov_preprocess_input_info_get_preprocess_steps``, | | +| | ``ov_preprocess_preprocess_steps_resize``, | | +| | ``ov_preprocess_input_model_info_set_layout``, | | +| | ``ov_preprocess_output_set_element_type``, | | +| | ``ov_preprocess_prepostprocessor_build`` | | ++-------------------------------------+-------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Options | Values | ++============================+============================================================================================================================================================================+ +| Validated Models | :doc:`alexnet `, :doc:`googlenet-v1 ` | ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Format | Inference Engine Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Validated images | The sample uses OpenCV\* to `read input image `__ (\*.bmp, \*.png) | ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ `, :doc:`Python ` | ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +How It Works +############ Upon the start-up, the sample application reads command line parameters, loads specified network and an image to the Inference Engine plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application outputs data to the standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in Inference Engine Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in Inference Engine Samples guide. -## Running +Running +####### To run the sample, you need specify a model and image: -- you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- you can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `__. + +.. note:: + + - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (\*.onnx) that do not require preprocessing. -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (\*.onnx) that do not require preprocessing. +Example ++++++++ -### Example 1. Download a pre-trained model using [Model Downloader](@ref omz_tools_downloader): - ``` - python /downloader.py --name alexnet - ``` + + .. code-block:: console + + python /downloader.py --name alexnet 2. If a model is not in the Inference Engine IR or ONNX format, it must be converted. You can do this using the model converter script: - ``` - python /converter.py --name alexnet - ``` + + .. code-block:: console + + python /converter.py --name alexnet -3. Perform inference of `car.bmp` using `alexnet` model on a `GPU`, for example: - ``` - /hello_classification_c /alexnet.xml /car.bmp GPU - ``` +3. Perform inference of ``car.bmp`` using ``alexnet`` model on a ``GPU``, for example: + + .. code-block:: console + + /hello_classification_c /alexnet.xml /car.bmp GPU -## Sample Output +Sample Output +############# The application outputs top-10 inference results. -``` -Top 10 results: - -Image /opt/intel/openvino/samples/scripts/car.png - -classid probability -------- ----------- -656 0.666479 -654 0.112940 -581 0.068487 -874 0.033385 -436 0.026132 -817 0.016731 -675 0.010980 -511 0.010592 -569 0.008178 -717 0.006336 - -This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate OpenVINO™ into Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) -- [C API Reference](https://docs.openvino.ai/latest/api/api_reference.html) \ No newline at end of file +.. code-block:: console + + Top 10 results: + + Image /opt/intel/openvino/samples/scripts/car.png + + classid probability + ------- ----------- + 656 0.666479 + 654 0.112940 + 581 0.068487 + 874 0.033385 + 436 0.026132 + 817 0.016731 + 675 0.010980 + 511 0.010592 + 569 0.008178 + 717 0.006336 + + This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + +See Also +######## + +- :doc:`Integrate OpenVINO™ into Your Application ` +- :doc:`Using OpenVINO™ Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` +- :doc:`C API Reference ` + +@endsphinxdirective + diff --git a/samples/c/hello_classification/main.c b/samples/c/hello_classification/main.c index b3748356ad2fd0..4fbf0c5f7a99b6 100644 --- a/samples/c/hello_classification/main.c +++ b/samples/c/hello_classification/main.c @@ -72,6 +72,7 @@ struct infer_result* tensor_to_infer_result(ov_tensor_t* tensor, size_t* result_ results[i].probability = float_data[i]; } + ov_shape_free(&output_shape); return results; } diff --git a/samples/c/hello_nv12_input_classification/README.md b/samples/c/hello_nv12_input_classification/README.md index 8d6778d491f076..efb267398405f9 100644 --- a/samples/c/hello_nv12_input_classification/README.md +++ b/samples/c/hello_nv12_input_classification/README.md @@ -1,114 +1,129 @@ # Hello NV12 Input Classification C Sample {#openvino_inference_engine_ie_bridges_c_samples_hello_nv12_input_classification_README} +@sphinxdirective + This sample demonstrates how to execute an inference of image classification networks like AlexNet with images in NV12 color format using Synchronous Inference Request API. Hello NV12 Input Classification C Sample demonstrates how to use the NV12 automatic input pre-processing API in your applications: -| Feature | API | Description | -| :--- | :--- | :--- | -| Node Operations | `ov_port_get_any_name` | Get a layer name | -| Infer Request Operations | `ov_infer_request_set_tensor`, `ov_infer_request_get_output_tensor_by_index` | Operate with tensors | -| Preprocessing | `ov_preprocess_input_tensor_info_set_color_format`, `ov_preprocess_preprocess_steps_convert_element_type`, `ov_preprocess_preprocess_steps_convert_color` | Change the color format of the input data | ++-----------------------------------------+-----------------------------------------------------------+--------------------------------------------------------+ +| Feature | API | Description | ++=========================================+===========================================================+========================================================+ +| Node Operations | ``ov_port_get_any_name`` | Get a layer name | ++-----------------------------------------+-----------------------------------------------------------+--------------------------------------------------------+ +| Infer Request Operations | ``ov_infer_request_set_tensor``, | Operate with tensors | +| | ``ov_infer_request_get_output_tensor_by_index`` | | ++-----------------------------------------+-----------------------------------------------------------+--------------------------------------------------------+ +| Preprocessing | ``ov_preprocess_input_tensor_info_set_color_format``, | Change the color format of the input data | +| | ``ov_preprocess_preprocess_steps_convert_element_type``, | | +| | ``ov_preprocess_preprocess_steps_convert_color`` | | ++-----------------------------------------+-----------------------------------------------------------+--------------------------------------------------------+ + -Basic Inference Engine API is covered by [Hello Classification C sample](../hello_classification/README.md). +Basic Inference Engine API is covered by :doc:`Hello Classification C sample `. -| Options | Values | -|:--- |:--- -| Validated Models | [alexnet](@ref omz_models_model_alexnet) -| Model Format | Inference Engine Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) -| Validated images | An uncompressed image in the NV12 color format - \*.yuv -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/hello_nv12_input_classification/README.md) | ++-----------------------------------------+---------------------------------------------------------------------------------------+ +| Options | Values | ++=========================================+=======================================================================================+ +| Validated Models | :doc:`alexnet ` | ++-----------------------------------------+---------------------------------------------------------------------------------------+ +| Model Format | Inference Engine Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | ++-----------------------------------------+---------------------------------------------------------------------------------------+ +| Validated images | An uncompressed image in the NV12 color format - \*.yuv | ++-----------------------------------------+---------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++-----------------------------------------+---------------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++-----------------------------------------+---------------------------------------------------------------------------------------+ -## How It Works +How It Works +############ -Upon the start-up, the sample application reads command-line parameters, loads specified network and an -image in the NV12 color format to an Inference Engine plugin. Then, the sample creates an synchronous inference request object. When inference is done, the -application outputs data to the standard output stream. +Upon the start-up, the sample application reads command-line parameters, loads specified network and an image in the NV12 color format to an Inference Engine plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application outputs data to the standard output stream. -You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in Inference Engine Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in Inference Engine Samples guide. -## Running +Running +####### To run the sample, you need specify a model and image: -- you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- you can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. - -The sample accepts an uncompressed image in the NV12 color format. To run the sample, you need to -convert your BGR/RGB image to NV12. To do this, you can use one of the widely available tools such -as FFmpeg\* or GStreamer\*. The following command shows how to convert an ordinary image into an -uncompressed NV12 image using FFmpeg: - -```sh -ffmpeg -i cat.jpg -pix_fmt nv12 cat.yuv -``` - -> **NOTES**: -> -> - Because the sample reads raw image files, you should provide a correct image size along with the -> image path. The sample expects the logical size of the image, not the buffer size. For example, -> for 640x480 BGR/RGB image the corresponding NV12 logical image size is also 640x480, whereas the -> buffer size is 640x720. -> - By default, this sample expects that network input has BGR channels order. If you trained your -> model to work with RGB order, you need to reconvert your model using the Model Optimizer tool -> with `--reverse_input_channels` argument specified. For more information about the argument, -> refer to **When to Reverse Input Channels** section of -> [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> - Before running the sample with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. - -### Example -1. Download a pre-trained model using [Model Downloader](@ref omz_tools_downloader): - ``` - python /downloader.py --name alexnet - ``` +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `__. + +The sample accepts an uncompressed image in the NV12 color format. To run the sample, you need to convert your BGR/RGB image to NV12. To do this, you can use one of the widely available tools such as FFmpeg\* or GStreamer\*. The following command shows how to convert an ordinary image into an uncompressed NV12 image using FFmpeg: + +.. code-block:: bash + + ffmpeg -i cat.jpg -pix_fmt nv12 cat.yuv + +.. note:: + + - Because the sample reads raw image files, you should provide a correct image size along with the image path. The sample expects the logical size of the image, not the buffer size. For example, for 640x480 BGR/RGB image the corresponding NV12 logical image size is also 640x480, whereas the buffer size is 640x720. + - By default, this sample expects that network input has BGR channels order. If you trained your model to work with RGB order, you need to reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + +Example ++++++++ + +1. Download a pre-trained model using :doc:`Model Downloader `: + + .. code-block:: console + + python /downloader.py --name alexnet 2. If a model is not in the Inference Engine IR or ONNX format, it must be converted. You can do this using the model converter script: - ``` - python /converter.py --name alexnet - ``` + + .. code-block:: console + + python /converter.py --name alexnet 3. Perform inference of NV12 image using `alexnet` model on a `CPU`, for example: - ``` - /hello_nv12_input_classification_c /alexnet.xml /cat.yuv 300x300 CPU - ``` + + .. code-block:: console + + /hello_nv12_input_classification_c /alexnet.xml /cat.yuv 300x300 CPU -## Sample Output +Sample Output +############# The application outputs top-10 inference results. -``` -Top 10 results: - -Image ./cat.yuv - -classid probability -------- ----------- -435 0.091733 -876 0.081725 -999 0.069305 -587 0.043726 -666 0.038957 -419 0.032892 -285 0.030309 -700 0.029941 -696 0.021628 -855 0.020339 - -This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ into Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) -- [C API Reference](https://docs.openvino.ai/latest/api/api_reference.html) \ No newline at end of file +.. code-block:: console + + Top 10 results: + + Image ./cat.yuv + + classid probability + ------- ----------- + 435 0.091733 + 876 0.081725 + 999 0.069305 + 587 0.043726 + 666 0.038957 + 419 0.032892 + 285 0.030309 + 700 0.029941 + 696 0.021628 + 855 0.020339 + + This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + +See Also +######## + +- :doc:`Integrate the OpenVINO™ into Your Application ` +- :doc:`Using OpenVINO™ Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` +- `C API Reference `__ + +@endsphinxdirective + diff --git a/samples/cpp/benchmark/sync_benchmark/README.md b/samples/cpp/benchmark/sync_benchmark/README.md index f3132d8675cc83..afe299e975f3d9 100644 --- a/samples/cpp/benchmark/sync_benchmark/README.md +++ b/samples/cpp/benchmark/sync_benchmark/README.md @@ -1,97 +1,132 @@ # Sync Benchmark C++ Sample {#openvino_inference_engine_samples_sync_benchmark_README} -This sample demonstrates how to estimate performace of a model using Synchronous Inference Request API. It makes sence to use synchronous inference only in latency oriented scenarios. Models with static input shapes are supported. Unlike [demos](@ref omz_demos) this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. +@sphinxdirective -The following C++ API is used in the application: - -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | `ov::get_openvino_version` | Get Openvino API version | -| Basic Infer Flow | `ov::Core`, `ov::Core::compile_model`, `ov::CompiledModel::create_infer_request`, `ov::InferRequest::get_tensor` | Common API to do inference: compile a model, create an infer request, configure input tensors | -| Synchronous Infer | `ov::InferRequest::infer` | Do synchronous inference | -| Model Operations | `ov::CompiledModel::inputs` | Get inputs of a model | -| Tensor Operations | `ov::Tensor::get_shape` | Get a tensor shape | -| Tensor Operations | `ov::Tensor::get_shape`, `ov::Tensor::data` | Get a tensor shape and its data. | +This sample demonstrates how to estimate performance of a model using Synchronous Inference Request API. It makes sense to use synchronous inference only in latency oriented scenarios. Models with static input shapes are supported. Unlike :doc:`demos ` this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. -| Options | Values | -| :--- | :--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) [yolo-v3-tf](@ref omz_models_model_yolo_v3_tf), [face-detection-0200](@ref omz_models_model_face_detection_0200) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../python/benchmark/sync_benchmark/README.md) | +The following C++ API is used in the application: -## How It Works ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Feature | API | Description | ++==========================+==============================================+==============================================+ +| OpenVINO Runtime Version | ``ov::get_openvino_version`` | Get Openvino API version. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Basic Infer Flow | ``ov::Core``, ``ov::Core::compile_model``, | Common API to do inference: compile a model, | +| | ``ov::CompiledModel::create_infer_request``, | create an infer request, | +| | ``ov::InferRequest::get_tensor`` | configure input tensors. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Synchronous Infer | ``ov::InferRequest::infer``, | Do synchronous inference. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Model Operations | ``ov::CompiledModel::inputs`` | Get inputs of a model. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Tensor Operations | ``ov::Tensor::get_shape``, | Get a tensor shape and its data. | +| | ``ov::Tensor::data`` | | ++--------------------------+----------------------------------------------+----------------------------------------------+ + ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Options | Values | ++================================+================================================================================================+ +| Validated Models | :doc:`alexnet `, | +| | :doc:`googlenet-v1 `, | +| | :doc:`yolo-v3-tf `, | +| | :doc:`face-detection-0200 ` | ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation | +| | (\*.xml + \*.bin), ONNX (\*.onnx) | ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`Python ` | ++--------------------------------+------------------------------------------------------------------------------------------------+ + + +How It Works +#################### The sample compiles a model for a given device, randomly generates input data, performs synchronous inference multiple times for a given number of seconds. Then processes and reports performance results. You can see the explicit description of -each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +#################### -To build the sample, please use instructions available at [Build the Sample Applications](../../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +#################### + +.. code-block:: sh + + sync_benchmark -``` -sync_benchmark -``` To run the sample, you need to specify a model: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -> **NOTES**: -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. + +.. note:: -### Example + Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: + The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + +Example +++++++++++++++++++++ + +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: + + .. code-block:: sh + + python -m pip install openvino-dev[caffe] -``` -python -m pip install openvino-dev[caffe] -``` 2. Download a pre-trained model using: -``` -omz_downloader --name googlenet-v1 -``` + .. code-block:: sh + + omz_downloader --name googlenet-v1 + 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: -``` -omz_converter --name googlenet-v1 -``` + .. code-block:: sh + + omz_converter --name googlenet-v1 + + +4. Perform benchmarking using the ``googlenet-v1`` model on a ``CPU``: + + .. code-block:: sh -4. Perform benchmarking using the `googlenet-v1` model on a `CPU`: + sync_benchmark googlenet-v1.xml -``` -sync_benchmark googlenet-v1.xml -``` -## Sample Output +Sample Output +#################### The application outputs performance results. -``` -[ INFO ] OpenVINO: -[ INFO ] Build ................................. -[ INFO ] Count: 992 iterations -[ INFO ] Duration: 15009.8 ms -[ INFO ] Latency: -[ INFO ] Median: 14.00 ms -[ INFO ] Average: 15.13 ms -[ INFO ] Min: 9.33 ms -[ INFO ] Max: 53.60 ms -[ INFO ] Throughput: 66.09 FPS -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: sh + + [ INFO ] OpenVINO: + [ INFO ] Build ................................. + [ INFO ] Count: 992 iterations + [ INFO ] Duration: 15009.8 ms + [ INFO ] Latency: + [ INFO ] Median: 14.00 ms + [ INFO ] Average: 15.13 ms + [ INFO ] Min: 9.33 ms + [ INFO ] Max: 53.60 ms + [ INFO ] Throughput: 66.09 FPS + + +See Also +#################### + +* :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +* :doc:`Using OpenVINO Samples ` +* :doc:`Model Downloader ` +* :doc:`Model Optimizer ` + +@endsphinxdirective diff --git a/samples/cpp/benchmark/throughput_benchmark/README.md b/samples/cpp/benchmark/throughput_benchmark/README.md index b8caccea4c1357..a6d0776c52f214 100644 --- a/samples/cpp/benchmark/throughput_benchmark/README.md +++ b/samples/cpp/benchmark/throughput_benchmark/README.md @@ -1,98 +1,135 @@ # Throughput Benchmark C++ Sample {#openvino_inference_engine_samples_throughput_benchmark_README} -This sample demonstrates how to estimate performace of a model using Asynchronous Inference Request API in throughput mode. Unlike [demos](@ref omz_demos) this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. +@sphinxdirective -The reported results may deviate from what [benchmark_app](../../benchmark_app/README.md) reports. One example is model input precision for computer vision tasks. benchmark_app sets uint8, while the sample uses default model precision which is usually float32. +This sample demonstrates how to estimate performance of a model using Asynchronous Inference Request API in throughput mode. Unlike :doc:`demos ` this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. -The following C++ API is used in the application: - -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | `ov::get_openvino_version` | Get Openvino API version | -| Basic Infer Flow | `ov::Core`, `ov::Core::compile_model`, `ov::CompiledModel::create_infer_request`, `ov::InferRequest::get_tensor` | Common API to do inference: compile a model, create an infer request, configure input tensors | -| Asynchronous Infer | `ov::InferRequest::start_async`, `ov::InferRequest::set_callback` | Do asynchronous inference with callback. | -| Model Operations | `ov::CompiledModel::inputs` | Get inputs of a model | -| Tensor Operations | `ov::Tensor::get_shape`, `ov::Tensor::data` | Get a tensor shape and its data. | +The reported results may deviate from what :doc:`benchmark_app ` reports. One example is model input precision for computer vision tasks. benchmark_app sets ``uint8``, while the sample uses default model precision which is usually ``float32``. -| Options | Values | -| :--- | :--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) [yolo-v3-tf](@ref omz_models_model_yolo_v3_tf), [face-detection-0200](@ref omz_models_model_face_detection_0200) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../python/benchmark/throughput_benchmark/README.md) | +The following C++ API is used in the application: -## How It Works ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Feature | API | Description | ++==========================+==============================================+==============================================+ +| OpenVINO Runtime Version | ``ov::get_openvino_version`` | Get Openvino API version. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Basic Infer Flow | ``ov::Core``, ``ov::Core::compile_model``, | Common API to do inference: compile a model, | +| | ``ov::CompiledModel::create_infer_request``, | create an infer request, | +| | ``ov::InferRequest::get_tensor`` | configure input tensors. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Asynchronous Infer | ``ov::InferRequest::start_async``, | Do asynchronous inference with callback. | +| | ``ov::InferRequest::set_callback`` | | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Model Operations | ``ov::CompiledModel::inputs`` | Get inputs of a model. | ++--------------------------+----------------------------------------------+----------------------------------------------+ +| Tensor Operations | ``ov::Tensor::get_shape``, | Get a tensor shape and its data. | +| | ``ov::Tensor::data`` | | ++--------------------------+----------------------------------------------+----------------------------------------------+ + ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Options | Values | ++================================+================================================================================================+ +| Validated Models | :doc:`alexnet `, | +| | :doc:`googlenet-v1 `, | +| | :doc:`yolo-v3-tf `, | +| | :doc:`face-detection-0200 ` | ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation | +| | (\*.xml + \*.bin), ONNX (\*.onnx) | ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++--------------------------------+------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`Python ` | ++--------------------------------+------------------------------------------------------------------------------------------------+ + + +How It Works +#################### The sample compiles a model for a given device, randomly generates input data, performs asynchronous inference multiple times for a given number of seconds. Then processes and reports performance results. You can see the explicit description of -each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +#################### -To build the sample, please use instructions available at [Build the Sample Applications](../../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +#################### + +.. code-block:: sh + + throughput_benchmark -``` -throughput_benchmark -``` To run the sample, you need to specify a model: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -> **NOTES**: -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. + +.. note:: -### Example + Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: + The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + +Example +++++++++++++++++++++ + +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: + + .. code-block:: sh + + python -m pip install openvino-dev[caffe] -``` -python -m pip install openvino-dev[caffe] -``` 2. Download a pre-trained model using: -``` -omz_downloader --name googlenet-v1 -``` + .. code-block:: sh + + omz_downloader --name googlenet-v1 + 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: -``` -omz_converter --name googlenet-v1 -``` + .. code-block:: sh + + omz_converter --name googlenet-v1 + + +4. Perform benchmarking using the ``googlenet-v1`` model on a ``CPU``: + + .. code-block:: sh -4. Perform benchmarking using the `googlenet-v1` model on a `CPU`: + throughput_benchmark googlenet-v1.xml -``` -throughput_benchmark googlenet-v1.xml -``` -## Sample Output +Sample Output +#################### The application outputs performance results. -``` -[ INFO ] OpenVINO: -[ INFO ] Build ................................. -[ INFO ] Count: 1577 iterations -[ INFO ] Duration: 15024.2 ms -[ INFO ] Latency: -[ INFO ] Median: 38.02 ms -[ INFO ] Average: 38.08 ms -[ INFO ] Min: 25.23 ms -[ INFO ] Max: 49.16 ms -[ INFO ] Throughput: 104.96 FPS -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: sh + + [ INFO ] OpenVINO: + [ INFO ] Build ................................. + [ INFO ] Count: 1577 iterations + [ INFO ] Duration: 15024.2 ms + [ INFO ] Latency: + [ INFO ] Median: 38.02 ms + [ INFO ] Average: 38.08 ms + [ INFO ] Min: 25.23 ms + [ INFO ] Max: 49.16 ms + [ INFO ] Throughput: 104.96 FPS + + +See Also +#################### + +* :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +* :doc:`Using OpenVINO Samples ` +* :doc:`Model Downloader ` +* :doc:`Model Optimizer ` + +@endsphinxdirective diff --git a/samples/cpp/benchmark_app/README.md b/samples/cpp/benchmark_app/README.md index f9cdda078e0af2..a0aaaf17d39c22 100644 --- a/samples/cpp/benchmark_app/README.md +++ b/samples/cpp/benchmark_app/README.md @@ -1,246 +1,306 @@ # Benchmark C++ Tool {#openvino_inference_engine_samples_benchmark_app_README} +@sphinxdirective + This page demonstrates how to use the Benchmark C++ Tool to estimate deep learning inference performance on supported devices. -> **NOTE**: This page describes usage of the C++ implementation of the Benchmark Tool. For the Python implementation, refer to the [Benchmark Python Tool](../../../tools/benchmark_tool/README.md) page. The Python version is recommended for benchmarking models that will be used in Python applications, and the C++ version is recommended for benchmarking models that will be used in C++ applications. Both tools have a similar command interface and backend. +.. note:: + + This page describes usage of the C++ implementation of the Benchmark Tool. For the Python implementation, refer to the :doc:`Benchmark Python Tool ` page. The Python version is recommended for benchmarking models that will be used in Python applications, and the C++ version is recommended for benchmarking models that will be used in C++ applications. Both tools have a similar command interface and backend. + + +Basic Usage +#################### +To use the C++ benchmark_app, you must first build it following the :doc:`Build the Sample Applications ` instructions and then set up paths and environment variables by following the :doc:`Get Ready for Running the Sample Applications ` instructions. Navigate to the directory where the benchmark_app C++ sample binary was built. -## Basic Usage -To use the C++ benchmark_app, you must first build it following the [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) instructions and then set up paths and environment variables by following the [Get Ready for Running the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) instructions. Navigate to the directory where the benchmark_app C++ sample binary was built. +.. note:: -> **NOTE**: If you installed OpenVINO Runtime using PyPI or Anaconda Cloud, only the [Benchmark Python Tool](../../../tools/benchmark_tool/README.md) is available, and you should follow the usage instructions on that page instead. + If you installed OpenVINO Runtime using PyPI or Anaconda Cloud, only the :doc:`Benchmark Python Tool ` is available, and you should follow the usage instructions on that page instead. -The benchmarking application works with models in the OpenVINO IR (`model.xml` and `model.bin`) and ONNX (`model.onnx`) formats. Make sure to [convert your models](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) if necessary. +The benchmarking application works with models in the OpenVINO IR (``model.xml`` and ``model.bin``) and ONNX (``model.onnx``) formats. Make sure to :doc:`convert your models ` if necessary. To run benchmarking with default options on a model, use the following command: -``` -./benchmark_app -m model.xml -``` +.. code-block:: sh + + ./benchmark_app -m model.xml + By default, the application will load the specified model onto the CPU and perform inferencing on batches of randomly-generated data inputs for 60 seconds. As it loads, it prints information about benchmark parameters. When benchmarking is completed, it reports the minimum, average, and maximum inferencing latency and average the throughput. You may be able to improve benchmark results beyond the default configuration by configuring some of the execution parameters for your model. For example, you can use "throughput" or "latency" performance hints to optimize the runtime for higher FPS or reduced inferencing time. Read on to learn more about the configuration options available with benchmark_app. -## Configuration Options -The benchmark app provides various options for configuring execution parameters. This section covers key configuration options for easily tuning benchmarking to achieve better performance on your device. A list of all configuration options is given in the [Advanced Usage](#advanced-usage-cpp-benchmark) section. +Configuration Options +##################### + +The benchmark app provides various options for configuring execution parameters. This section covers key configuration options for easily tuning benchmarking to achieve better performance on your device. A list of all configuration options is given in the :ref:`Advanced Usage ` section. + +Performance hints: latency and throughput ++++++++++++++++++++++++++++++++++++++++++ -### Performance hints: latency and throughput The benchmark app allows users to provide high-level "performance hints" for setting latency-focused or throughput-focused inference modes. This hint causes the runtime to automatically adjust runtime parameters, such as the number of processing streams and inference batch size, to prioritize for reduced latency or high throughput. The performance hints do not require any device-specific settings and they are completely portable between devices. Parameters are automatically configured based on whichever device is being used. This allows users to easily port applications between hardware targets without having to re-determine the best runtime parameters for the new device. -If not specified, throughput is used as the default. To set the hint explicitly, use `-hint latency` or `-hint throughput` when running benchmark_app: +If not specified, throughput is used as the default. To set the hint explicitly, use ``-hint latency`` or ``-hint throughput`` when running benchmark_app: + +.. code-block:: sh + + ./benchmark_app -m model.xml -hint latency + ./benchmark_app -m model.xml -hint throughput + -``` -./benchmark_app -m model.xml -hint latency -./benchmark_app -m model.xml -hint throughput -``` +.. note:: -> **NOTE** -It is up to the user to ensure the environment on which the benchmark is running is optimized for maximum performance. -Otherwise, different results may occur when using the application in different environment settings (such as power optimization settings, processor overclocking, thermal throttling). + It is up to the user to ensure the environment on which the benchmark is running is optimized for maximum performance. Otherwise, different results may occur when using the application in different environment settings (such as power optimization settings, processor overclocking, thermal throttling). + +Latency +-------------------- -#### Latency Latency is the amount of time it takes to process a single inference request. In applications where data needs to be inferenced and acted on as quickly as possible (such as autonomous driving), low latency is desirable. For conventional devices, lower latency is achieved by reducing the amount of parallel processing streams so the system can utilize as many resources as possible to quickly calculate each inference request. However, advanced devices like multi-socket CPUs and modern GPUs are capable of running multiple inference requests while delivering the same latency. -When benchmark_app is run with `-hint latency`, it determines the optimal number of parallel inference requests for minimizing latency while still maximizing the parallelization capabilities of the hardware. It automatically sets the number of processing streams and inference batch size to achieve the best latency. +When benchmark_app is run with ``-hint latency``, it determines the optimal number of parallel inference requests for minimizing latency while still maximizing the parallelization capabilities of the hardware. It automatically sets the number of processing streams and inference batch size to achieve the best latency. + +Throughput +-------------------- -#### Throughput Throughput is the amount of data an inferencing pipeline can process at once, and it is usually measured in frames per second (FPS) or inferences per second. In applications where large amounts of data needs to be inferenced simultaneously (such as multi-camera video streams), high throughput is needed. To achieve high throughput, the runtime focuses on fully saturating the device with enough data to process. It utilizes as much memory and as many parallel streams as possible to maximize the amount of data that can be processed simultaneously. -When benchmark_app is run with `-hint throughput`, it maximizes the number of parallel inference requests to utilize all the threads available on the device. On GPU, it automatically sets the inference batch size to fill up the GPU memory available. +When benchmark_app is run with ``-hint throughput``, it maximizes the number of parallel inference requests to utilize all the threads available on the device. On GPU, it automatically sets the inference batch size to fill up the GPU memory available. + +For more information on performance hints, see the :doc:`High-level Performance Hints ` page. For more details on optimal runtime configurations and how they are automatically determined using performance hints, see :doc:`Runtime Inference Optimizations `. -For more information on performance hints, see the [High-level Performance Hints](../../../docs/OV_Runtime_UG/performance_hints.md) page. For more details on optimal runtime configurations and how they are automatically determined using performance hints, see [Runtime Inference Optimizations](../../../docs/optimization_guide/dldt_deployment_optimization_guide.md). +Device +++++++++++++++++++++ -### Device -To set which device benchmarking runs on, use the `-d ` argument. This will tell benchmark_app to run benchmarking on that specific device. The benchmark app supports "CPU", "GPU", and "GNA" devices. In order to use the GPU or GNA, the system must have the appropriate drivers installed. If no device is specified, benchmark_app will default to using CPU. +To set which device benchmarking runs on, use the ``-d `` argument. This will tell benchmark_app to run benchmarking on that specific device. The benchmark app supports "CPU", "GPU", and "GNA" devices. In order to use the GPU or GNA, the system must have the appropriate drivers installed. If no device is specified, benchmark_app will default to using CPU. For example, to run benchmarking on GPU, use: -``` -./benchmark_app -m model.xml -d GPU -``` +.. code-block:: sh -You may also specify "AUTO" as the device, in which case the benchmark_app will automatically select the best device for benchmarking and support it with the CPU at the model loading stage. This may result in increased performance, thus, should be used purposefully. For more information, see the [Automatic device selection](../../../docs/OV_Runtime_UG/auto_device_selection.md) page. + ./benchmark_app -m model.xml -d GPU + + +You may also specify "AUTO" as the device, in which case the benchmark_app will automatically select the best device for benchmarking and support it with the CPU at the model loading stage. This may result in increased performance, thus, should be used purposefully. For more information, see the :doc:`Automatic device selection ` page. (Note: If the latency or throughput hint is set, it will automatically configure streams and batch sizes for optimal performance based on the specified device.) -### Number of iterations +Number of iterations +++++++++++++++++++++ + By default, the benchmarking app will run for a predefined duration, repeatedly performing inferencing with the model and measuring the resulting inference speed. There are several options for setting the number of inference iterations: -* Explicitly specify the number of iterations the model runs using the `-niter ` option. -* Set how much time the app runs for using the `-t ` option. +* Explicitly specify the number of iterations the model runs using the ``-niter `` option. +* Set how much time the app runs for using the ``-t `` option. * Set both of them (execution will continue until both conditions are met). * If neither -niter nor -t are specified, the app will run for a predefined duration that depends on the device. -The more iterations a model runs, the better the statistics will be for determing average latency and throughput. +The more iterations a model runs, the better the statistics will be for determining average latency and throughput. + +Inputs +++++++++++++++++++++ + +The benchmark tool runs benchmarking on user-provided input images in ``.jpg``, ``.bmp``, or ``.png`` format. Use ``-i `` to specify the path to an image, or folder of images. For example, to run benchmarking on an image named ``test1.jpg``, use: + +.. code-block:: sh + + ./benchmark_app -m model.xml -i test1.jpg -### Inputs -The benchmark tool runs benchmarking on user-provided input images in `.jpg`, `.bmp`, or `.png` format. Use `-i ` to specify the path to an image, or folder of images. For example, to run benchmarking on an image named `test1.jpg`, use: -``` -./benchmark_app -m model.xml -i test1.jpg -``` +The tool will repeatedly loop through the provided inputs and run inferencing on them for the specified amount of time or number of iterations. If the ``-i`` flag is not used, the tool will automatically generate random data to fit the input shape of the model. -The tool will repeatedly loop through the provided inputs and run inferencing on them for the specified amount of time or number of iterations. If the `-i` flag is not used, the tool will automatically generate random data to fit the input shape of the model. +Examples +++++++++++++++++++++ -### Examples -For more usage examples (and step-by-step instructions on how to set up a model for benchmarking), see the [Examples of Running the Tool](#examples-of-running-the-tool-cpp) section. +For more usage examples (and step-by-step instructions on how to set up a model for benchmarking), see the :ref:`Examples of Running the Tool ` section. -## Advanced Usage +.. _advanced-usage-cpp-benchmark: -> **NOTE**: By default, OpenVINO samples, tools and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channel order in the sample or demo application or reconvert your model using the Model Optimizer tool with --reverse_input_channels argument specified. For more information about the argument, refer to When to Reverse Input Channels section of Converting a Model to Intermediate Representation (IR). +Advanced Usage +#################### -### Per-layer performance and logging -The application also collects per-layer Performance Measurement (PM) counters for each executed infer request if you enable statistics dumping by setting the `-report_type` parameter to one of the possible values: +.. note:: -* `no_counters` report includes configuration options specified, resulting FPS and latency. -* `average_counters` report extends the `no_counters` report and additionally includes average PM counters values for each layer from the network. -* `detailed_counters` report extends the `average_counters` report and additionally includes per-layer PM counters and latency for each executed infer request. + By default, OpenVINO samples, tools and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channel order in the sample or demo application or reconvert your model using the Model Optimizer tool with --reverse_input_channels argument specified. For more information about the argument, refer to When to Reverse Input Channels section of Converting a Model to Intermediate Representation (IR). + +Per-layer performance and logging ++++++++++++++++++++++++++++++++++ + +The application also collects per-layer Performance Measurement (PM) counters for each executed infer request if you enable statistics dumping by setting the ``-report_type`` parameter to one of the possible values: + +* ``no_counters`` report includes configuration options specified, resulting FPS and latency. +* ``average_counters`` report extends the ``no_counters`` report and additionally includes average PM counters values for each layer from the network. +* ``detailed_counters`` report extends the ``average_counters`` report and additionally includes per-layer PM counters and latency for each executed infer request. Depending on the type, the report is stored to benchmark_no_counters_report.csv, benchmark_average_counters_report.csv, or benchmark_detailed_counters_report.csv file located in the path specified in -report_folder. The application also saves executable graph information serialized to an XML file if you specify a path to it with the -exec_graph_path parameter. -### All configuration options - -Running the application with the `-h` or `--help` option yields the following usage message: - -``` -[Step 1/11] Parsing and validating input arguments -[ INFO ] Parsing input parameters -usage: benchmark_app [OPTION] - -Options: - -h, --help Print the usage message - -m Required. Path to an .xml/.onnx file with a trained model or to a .blob files with a trained compiled model. - -i Optional. Path to a folder with images and/or binaries or to specific image or binary file. - In case of dynamic shapes models with several inputs provide the same number of files for each input (except cases with single file for any input):"input1:1.jpg input2:1.bin", "input1:1.bin,2.bin input2:3.bin input3:4.bin,5.bin ". Also you can pass specific keys for inputs: "random" - for fillling input with random data, "image_info" - for filling input with image size. - You should specify either one files set to be used for all inputs (without providing input names) or separate files sets for every input of model (providing inputs names). - Currently supported data types: bmp, bin, npy. - If OPENCV is enabled, this functionality is extended with the following data types: - dib, jpeg, jpg, jpe, jp2, png, pbm, pgm, ppm, sr, ras, tiff, tif. - -d Optional. Specify a target device to infer on (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify HETERO plugin. Use "-d MULTI:" format to specify MULTI plugin. The application looks for a suitable plugin for the specified device. - -hint (latency or throughput or cumulative_throughput or none) Optional. Performance hint allows the OpenVINO device to select the right model-specific settings. - 'throughput' or 'tput': device performance mode will be set to THROUGHPUT. - 'cumulative_throughput' or 'ctput': device performance mode will be set to CUMULATIVE_THROUGHPUT. - 'latency': device performance mode will be set to LATENCY. - 'none': device performance mode will be set to UNDEFINED. - Using explicit 'nstreams' or other device-specific options, please set hint to 'none' - -niter Optional. Number of iterations. If not specified, the number of iterations is calculated depending on a device. - -t Optional. Time in seconds to execute topology. - -Input shapes - -b Optional. Batch size value. If not specified, the batch size value is determined from Intermediate Representation. - -shape Optional. Set shape for model input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size. This parameter affect model input shape and can be dynamic. For dynamic dimensions use symbol `?` or '-1'. Ex. [?,3,?,?]. For bounded dimensions specify range 'min..max'. Ex. [1..10,3,?,?]. - -data_shape Required for models with dynamic shapes. Set shape for input blobs. In case of one input size: "[1,3,224,224]" or "input1[1,3,224,224],input2[1,4]". In case of several input sizes provide the same number for each input (except cases with single shape for any input): "[1,3,128,128][3,3,128,128][1,3,320,320]", "input1[1,1,128,128][1,1,256,256],input2[80,1]" or "input1[1,192][1,384],input2[1,192][1,384],input3[1,192][1,384],input4[1,192][1,384]". If model shapes are all static specifying the option will cause an exception. - -layout Optional. Prompts how model layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. - -Advanced options - -extensions Required for custom layers (extensions). Absolute path to a shared library with the kernels implementations. - -c Required for GPU custom kernels. Absolute path to an .xml file with the kernels description. - -cache_dir Optional. Enables caching of loaded models to specified directory. List of devices which support caching is shown at the end of this message. - -load_from_file Optional. Loads model from file directly without read_model. All CNNNetwork options (like re-shape) will be ignored - -api Optional (deprecated). Enable Sync/Async API. Default value is "async". - -nireq Optional. Number of infer requests. Default value is determined automatically for device. - -nstreams Optional. Number of streams to use for inference on the CPU or GPU devices (for HETERO and MULTI device cases use format :,: or just ). Default value is determined automatically for a device.Please note that although the automatic selection usually provides a reasonable performance, it still may be non - optimal for some cases, especially for very small models. See sample's README for more details. Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency estimations the number of streams should be set to 1. - -inference_only Optional. Measure only inference stage. Default option for static models. Dynamic models are measured in full mode which includes inputs setup stage, inference only mode available for them with single input data shape only. To enable full mode for static models pass "false" value to this argument: ex. "-inference_only=false". - -infer_precision Optional. Specifies the inference precision. Example #1: '-infer_precision bf16'. Example #2: '-infer_precision CPU:bf16,GPU:f32' - -Preprocessing options: - -ip Optional. Specifies precision for all input layers of the model. - -op Optional. Specifies precision for all output layers of the model. - -iop Optional. Specifies precision for input and output layers by name. - Example: -iop "input:FP16, output:FP16". - Notice that quotes are required. - Overwrites precision from ip and op options for specified layers. - -mean_values [R,G,B] Optional. Mean values to be used for the input image per channel. Values to be provided in the [R,G,B] format. Can be defined for desired input of the model, for example: "--mean_values data[255,255,255],info[255,255,255]". The exact meaning and order of channels depend on how the original model was trained. Applying the values affects performance and may cause type conversion - -scale_values [R,G,B] Optional. Scale values to be used for the input image per channel. Values are provided in the [R,G,B] format. Can be defined for desired input of the model, for example: "--scale_values data[255,255,255],info[255,255,255]". The exact meaning and order of channels depend on how the original model was trained. If both --mean_values and --scale_values are specified, the mean is subtracted first and then scale is applied regardless of the order of options in command line. Applying the values affects performance and may cause type conversion - -Device-specific performance options: - -nthreads Optional. Number of threads to use for inference on the CPU (including HETERO and MULTI cases). - -pin ("YES"|"CORE") / "HYBRID_AWARE" / ("NO"|"NONE") / "NUMA" Optional. Explicit inference threads binding options (leave empty to let the OpenVINO make a choice): - enabling threads->cores pinning("YES", which is already default for any conventional CPU), - letting the runtime to decide on the threads->different core types("HYBRID_AWARE", which is default on the hybrid CPUs) - threads->(NUMA)nodes("NUMA") or - completely disable("NO") CPU inference threads pinning - -Statistics dumping options: - -latency_percentile Optional. Defines the percentile to be reported in latency metric. The valid range is [1, 100]. The default value is 50 (median). - -report_type Optional. Enable collecting statistics report. "no_counters" report contains configuration options specified, resulting FPS and latency. "average_counters" report extends "no_counters" report and additionally includes average PM counters values for each layer from the model. "detailed_counters" report extends "average_counters" report and additionally includes per-layer PM counters and latency for each executed infer request. - -report_folder Optional. Path to a folder where statistics report is stored. - -json_stats Optional. Enables JSON-based statistics output (by default reporting system will use CSV format). Should be used together with -report_folder option. - -pc Optional. Report performance counters. - -pcsort Optional. Report performance counters and analysis the sort hotpoint opts. "sort" Analysis opts time cost, print by hotpoint order "no_sort" Analysis opts time cost, print by normal order "simple_sort" Analysis opts time cost, only print EXECUTED opts by normal order - -pcseq Optional. Report latencies for each shape in -data_shape sequence. - -exec_graph_path Optional. Path to a file where to store executable graph information serialized. - -dump_config Optional. Path to JSON file to dump IE parameters, which were set by application. - -load_config Optional. Path to JSON file to load custom IE parameters. Please note, command line parameters have higher priority then parameters from configuration file. - Example 1: a simple JSON file for HW device with primary properties. - { - "CPU": {"NUM_STREAMS": "3", "PERF_COUNT": "NO"} - } - Example 2: a simple JSON file for meta device(AUTO/MULTI) with HW device properties. - { - "AUTO": { - "PERFORMANCE_HINT": "", - "PERF_COUNT": "NO", - "DEVICE_PROPERTIES": { - "CPU": { - "INFERENCE_PRECISION_HINT": "f32", - "NUM_STREAMS": "3" - }, - "GPU": { - "INFERENCE_PRECISION_HINT": "f32", - "NUM_STREAMS": "5" - } - } - } - } -``` +.. _all-configuration-options-cpp-benchmark: + +All configuration options ++++++++++++++++++++++++++ + +Running the application with the ``-h`` or ``--help`` option yields the following usage message: + +.. scrollbox:: + + .. code-block:: + + [Step 1/11] Parsing and validating input arguments + [ INFO ] Parsing input parameters + usage: benchmark_app [OPTION] + + Options: + -h, --help Print the usage message + -m Required. Path to an .xml/.onnx file with a trained model or to a .blob files with a trained compiled model. + -i Optional. Path to a folder with images and/or binaries or to specific image or binary file. + In case of dynamic shapes models with several inputs provide the same number of files for each input (except cases with single file for any input) :"input1:1.jpg input2:1.bin", "input1:1.bin,2.bin input2:3.bin input3:4.bin,5.bin ". Also you can pass specific keys for inputs: "random" - for fillling input with random data, "image_info" - for filling input with image size. + You should specify either one files set to be used for all inputs (without providing input names) or separate files sets for every input of model (providing inputs names). + Currently supported data types: bmp, bin, npy. + If OPENCV is enabled, this functionality is extended with the following data types: + dib, jpeg, jpg, jpe, jp2, png, pbm, pgm, ppm, sr, ras, tiff, tif. + -d Optional. Specify a target device to infer on (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify HETERO plugin. Use "-d MULTI:" format to specify MULTI plugin. The application looks for a suitable plugin for the specified device. + -hint (latency or throughput or cumulative_throughput or none) Optional. Performance hint allows the OpenVINO device to select the right model-specific settings. + 'throughput' or 'tput': device performance mode will be set to THROUGHPUT. + 'cumulative_throughput' or 'ctput': device performance mode will be set to CUMULATIVE_THROUGHPUT. + 'latency': device performance mode will be set to LATENCY. + 'none': device performance mode will be set to UNDEFINED. + Using explicit 'nstreams' or other device-specific options, please set hint to 'none' + -niter Optional. Number of iterations. If not specified, the number of iterations is calculated depending on a device. + -t Optional. Time in seconds to execute topology. + + Input shapes + -b Optional. Batch size value. If not specified, the batch size value is determined from Intermediate Representation. + -shape Optional. Set shape for model input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size. This parameter affect model input shape and can be dynamic. For dynamic dimensions use symbol `?` or '-1'. Ex. [?,3,?,?]. For bounded dimensions specify range 'min..max'. Ex. [1..10,3,?,?]. + -data_shape Required for models with dynamic shapes. Set shape for input blobs. In case of one input size: "[1,3,224,224]" or "input1[1,3,224,224],input2[1,4] ". In case of several input sizes provide the same number for each input (except cases with single shape for any input): "[1,3,128,128][3,3,128,128][1,3,320,320]", "input1[1,1, 128,128][1,1,256,256],input2[80,1]" or "input1[1,192][1,384],input2[1,192][1,384],input3[1,192][1,384],input4[1,192][1,384]". If model shapes are all static specifying the option will cause an exception. + -layout Optional. Prompts how model layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. + + Advanced options + -extensions Required for custom layers (extensions). Absolute path to a shared library with the kernels implementations. + -c Required for GPU custom kernels. Absolute path to an .xml file with the kernels description. + -cache_dir Optional. Enables caching of loaded models to specified directory. List of devices which support caching is shown at the end of this message. + -load_from_file Optional. Loads model from file directly without read_model. All CNNNetwork options (like re-shape) will be ignored + -api Optional (deprecated). Enable Sync/Async API. Default value is "async". + -nireq Optional. Number of infer requests. Default value is determined automatically for device. + -nstreams Optional. Number of streams to use for inference on the CPU or GPU devices (for HETERO and MULTI device cases use format :, : or just ). Default value is determined automatically for a device.Please note that although the automatic selection usually provides a reasonable performance, it still may be non - optimal for some cases, especially for very small models. See sample's README for more details. Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency estimations the number of streams should be set to 1. + -inference_only Optional. Measure only inference stage. Default option for static models. Dynamic models are measured in full mode which includes inputs setup stage, inference only mode available for them with single input data shape only. To enable full mode for static models pass "false" value to this argument: ex. "-inference_only=false". + -infer_precision Optional. Specifies the inference precision. Example #1: '-infer_precision bf16'. Example #2: '-infer_precision CPU:bf16,GPU:f32' + + Preprocessing options: + -ip Optional. Specifies precision for all input layers of the model. + -op Optional. Specifies precision for all output layers of the model. + -iop Optional. Specifies precision for input and output layers by name. + Example: -iop "input:FP16, output:FP16". + Notice that quotes are required. + Overwrites precision from ip and op options for specified layers. + -mean_values [R,G,B] Optional. Mean values to be used for the input image per channel. Values to be provided in the [R,G,B] format. Can be defined for desired input of the model, for example: "--mean_values data[255,255,255],info[255,255,255]". The exact meaning and order of channels depend on how the original model was trained. Applying the values affects performance and may cause type conversion + -scale_values [R,G,B] Optional. Scale values to be used for the input image per channel. Values are provided in the [R,G,B] format. Can be defined for desired input of the model, for example: "--scale_values data[255,255,255],info[255,255,255]". The exact meaning and order of channels depend on how the original model was trained. If both --mean_values and --scale_values are specified, the mean is subtracted first and then scale is applied regardless of the order of options in command line. Applying the values affects performance and may cause type conversion + + Device-specific performance options: + -nthreads Optional. Number of threads to use for inference on the CPU (including HETERO and MULTI cases). + -pin ("YES"|"CORE") / "HYBRID_AWARE" / ("NO"|"NONE") / "NUMA" Optional. Explicit inference threads binding options (leave empty to let the OpenVINO make a choice): + enabling threads->cores pinning("YES", which is already default for any conventional CPU), + letting the runtime to decide on the threads->different core types("HYBRID_AWARE", which is default on the hybrid CPUs) + threads->(NUMA)nodes("NUMA") or + completely disable("NO") CPU inference threads pinning + + Statistics dumping options: + -latency_percentile Optional. Defines the percentile to be reported in latency metric. The valid range is [1, 100]. The default value is 50 (median). + -report_type Optional. Enable collecting statistics report. "no_counters" report contains configuration options specified, resulting FPS and latency. "average_counters" report extends "no_counters" report and additionally includes average PM counters values for each layer from the model. "detailed_counters" report extends "average_counters" report and additionally includes per-layer PM counters and latency for each executed infer request. + -report_folder Optional. Path to a folder where statistics report is stored. + -json_stats Optional. Enables JSON-based statistics output (by default reporting system will use CSV format). Should be used together with -report_folder option. + -pc Optional. Report performance counters. + -pcsort Optional. Report performance counters and analysis the sort hotpoint opts. "sort" Analysis opts time cost, print by hotpoint order "no_sort" Analysis opts time cost, print by normal order "simple_sort" Analysis opts time cost, only print EXECUTED opts by normal order + -pcseq Optional. Report latencies for each shape in -data_shape sequence. + -exec_graph_path Optional. Path to a file where to store executable graph information serialized. + -dump_config Optional. Path to JSON file to dump IE parameters, which were set by application. + -load_config Optional. Path to JSON file to load custom IE parameters. Please note, command line parameters have higher priority then parameters from configuration file. + Example 1: a simple JSON file for HW device with primary properties. + { + "CPU": {"NUM_STREAMS": "3", "PERF_COUNT": "NO"} + } + Example 2: a simple JSON file for meta device(AUTO/MULTI) with HW device properties. + { + "AUTO": { + "PERFORMANCE_HINT": "", + "PERF_COUNT": "NO", + "DEVICE_PROPERTIES": { + "CPU": { + "INFERENCE_PRECISION_HINT": "f32", + "NUM_STREAMS": "3" + }, + "GPU": { + "INFERENCE_PRECISION_HINT": "f32", + "NUM_STREAMS": "5" + } + } + } + } + Running the application with the empty list of options yields the usage message given above and an error message. -### More information on inputs +More information on inputs +++++++++++++++++++++++++++ + The benchmark tool supports topologies with one or more inputs. If a topology is not data sensitive, you can skip the input parameter, and the inputs will be filled with random values. If a model has only image input(s), provide a folder with images or a path to an image as input. If a model has some specific input(s) (besides images), please prepare a binary file(s) or numpy array(s) that is filled with data of appropriate precision and provide a path to it as input. If a model has mixed input types, the input folder should contain all required files. Image inputs are filled with image files one by one. Binary inputs are filled with binary inputs one by one. -## Examples of Running the Tool -This section provides step-by-step instructions on how to run the Benchmark Tool with the `asl-recognition` model from the [Open Model Zoo](@ref model_zoo) on CPU or GPU devices. It uses random data as the input. +.. _examples-of-running-the-tool-cpp: + +Examples of Running the Tool +############################ + +This section provides step-by-step instructions on how to run the Benchmark Tool with the ``asl-recognition`` model from the :doc:`Open Model Zoo ` on CPU or GPU devices. It uses random data as the input. + +.. note:: + + Internet access is required to execute the following steps successfully. If you have access to the Internet through a proxy server only, please make sure that it is configured in your OS environment. -> **NOTE**: Internet access is required to execute the following steps successfully. If you have access to the Internet through a proxy server only, please make sure that it is configured in your OS environment. 1. Install OpenVINO Development Tools (if it hasn't been installed already): - ```sh - pip install openvino-dev - ``` -2. Download the model using `omz_downloader`, specifying the model name and directory to download the model to: - ```sh - omz_downloader --name asl-recognition-0004 --precisions FP16 --output_dir omz_models - ``` + .. code-block:: sh + + pip install openvino-dev + + +2. Download the model using ``omz_downloader``, specifying the model name and directory to download the model to: + + .. code-block:: sh + + omz_downloader --name asl-recognition-0004 --precisions FP16 --output_dir omz_models + 3. Run the tool, specifying the location of the model .xml file, the device to perform inference on, and with a performance hint. The following commands demonstrate examples of how to run the Benchmark Tool in latency mode on CPU and throughput mode on GPU devices: * On CPU (latency mode): - ```sh - ./benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d CPU -hint latency - ``` + + .. code-block:: sh + + ./benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d CPU -hint latency + * On GPU (throughput mode): - ```sh - ./benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d GPU -hint throughput - ``` + + .. code-block:: sh + + ./benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d GPU -hint throughput + The application outputs the number of executed iterations, total duration of execution, latency, and throughput. -Additionally, if you set the `-report_type` parameter, the application outputs a statistics report. If you set the `-pc` parameter, the application outputs performance counters. If you set `-exec_graph_path`, the application reports executable graph information serialized. All measurements including per-layer PM counters are reported in milliseconds. +Additionally, if you set the ``-report_type`` parameter, the application outputs a statistics report. If you set the ``-pc`` parameter, the application outputs performance counters. If you set ``-exec_graph_path``, the application reports executable graph information serialized. All measurements including per-layer PM counters are reported in milliseconds. An example of the information output when running benchmark_app on CPU in latency mode is shown below: - ```sh +.. code-block:: sh + ./benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d CPU -hint latency - ``` - ```sh + +.. code-block:: sh + [Step 1/11] Parsing and validating input arguments [ INFO ] Parsing input parameters [ INFO ] Input command: /home/openvino/bin/intel64/DEBUG/benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d CPU -hint latency @@ -300,14 +360,18 @@ An example of the information output when running benchmark_app on CPU in latenc [ INFO ] Min: 20.60 ms [ INFO ] Max: 37.19 ms [ INFO ] Throughput: 91.12 FPS - ``` -The Benchmark Tool can also be used with dynamically shaped networks to measure expected inference time for various input data shapes. See the `-shape` and `-data_shape` argument descriptions in the All configuration options section to learn more about using dynamic shapes. Here is a command example for using benchmark_app with dynamic networks and a portion of the resulting output: - ```sh + + +The Benchmark Tool can also be used with dynamically shaped networks to measure expected inference time for various input data shapes. See the ``-shape`` and ``-data_shape`` argument descriptions in the :ref:`All configuration options ` section to learn more about using dynamic shapes. Here is a command example for using benchmark_app with dynamic networks and a portion of the resulting output: + +.. code-block:: sh + ./benchmark_app -m omz_models/intel/asl-recognition-0004/FP16/asl-recognition-0004.xml -d CPU -shape [-1,3,16,224,224] -data_shape [1,3,16,224,224][2,3,16,224,224][4,3,16,224,224] -pcseq - ``` - ```sh + +.. code-block:: sh + [Step 9/11] Creating infer requests and preparing input tensors [ INFO ] Test Config 0 [ INFO ] input ([N,C,D,H,W], f32, {1, 3, 16, 224, 224}, dyn:{?,3,16,224,224}): random (binary data is expected) @@ -343,9 +407,13 @@ The Benchmark Tool can also be used with dynamically shaped networks to measure [ INFO ] Min: 327.50 ms [ INFO ] Max: 743.46 ms [ INFO ] Throughput: 107.61 FPS - ``` -## See Also -* [Using OpenVINO Runtime Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -* [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) -* [Model Downloader](@ref omz_tools_downloader) + +See Also +#################### + +* :doc:`Using OpenVINO Samples ` +* :doc:`Model Optimizer ` +* :doc:`Model Downloader ` + +@endsphinxdirective \ No newline at end of file diff --git a/samples/cpp/benchmark_app/benchmark_app.hpp b/samples/cpp/benchmark_app/benchmark_app.hpp index 50fe8e8dac1c28..1631861b18c123 100644 --- a/samples/cpp/benchmark_app/benchmark_app.hpp +++ b/samples/cpp/benchmark_app/benchmark_app.hpp @@ -327,7 +327,7 @@ DEFINE_string(nstreams, "", infer_num_streams_message); /// @brief Define flag for inference only mode
DEFINE_bool(inference_only, true, inference_only_message); -/// @brief Define flag for inference precision +/// @brief Define flag for inference precision hint DEFINE_string(infer_precision, "", inference_precision_message); /// @brief Specify precision for all input layers of the network diff --git a/samples/cpp/benchmark_app/infer_request_wrap.hpp b/samples/cpp/benchmark_app/infer_request_wrap.hpp index 6e72b7559dadb3..3383bd08019fe3 100644 --- a/samples/cpp/benchmark_app/infer_request_wrap.hpp +++ b/samples/cpp/benchmark_app/infer_request_wrap.hpp @@ -26,8 +26,7 @@ typedef std::function QueueCallbackFunction; -/// @brief Wrapper class for InferenceEngine::InferRequest. Handles asynchronous callbacks and calculates execution -/// time. +/// @brief Handles asynchronous callbacks and calculates execution time class InferReqWrap final { public: using Ptr = std::shared_ptr; diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 9706012a827b44..9e142b2ae84d38 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -120,23 +120,27 @@ void next_step(const std::string additional_info = "") { } ov::hint::PerformanceMode get_performance_hint(const std::string& device, const ov::Core& core) { + OPENVINO_SUPPRESS_DEPRECATED_START ov::hint::PerformanceMode ov_perf_hint = ov::hint::PerformanceMode::UNDEFINED; + OPENVINO_SUPPRESS_DEPRECATED_END auto supported_properties = core.get_property(device, ov::supported_properties); if (std::find(supported_properties.begin(), supported_properties.end(), ov::hint::performance_mode) != supported_properties.end()) { if (FLAGS_hint != "") { if (FLAGS_hint == "throughput" || FLAGS_hint == "tput") { - slog::warn << "Device(" << device << ") performance hint is set to THROUGHPUT" << slog::endl; ov_perf_hint = ov::hint::PerformanceMode::THROUGHPUT; } else if (FLAGS_hint == "latency") { - slog::warn << "Device(" << device << ") performance hint is set to LATENCY" << slog::endl; ov_perf_hint = ov::hint::PerformanceMode::LATENCY; } else if (FLAGS_hint == "cumulative_throughput" || FLAGS_hint == "ctput") { - slog::warn << "Device(" << device << ") performance hint is set to CUMULATIVE_THROUGHPUT" << slog::endl; ov_perf_hint = ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT; } else if (FLAGS_hint == "none") { - slog::warn << "No device(" << device << ") performance hint is set" << slog::endl; + OPENVINO_SUPPRESS_DEPRECATED_START ov_perf_hint = ov::hint::PerformanceMode::UNDEFINED; + OPENVINO_SUPPRESS_DEPRECATED_END + } else { + throw std::logic_error( + "Incorrect performance hint. Please set -hint option to" + "`throughput`(tput), `latency', 'cumulative_throughput'(ctput) value or 'none'."); } } else { ov_perf_hint = @@ -416,6 +420,7 @@ int main(int argc, char* argv[]) { return std::find(std::begin(supported_properties), std::end(supported_properties), key) != std::end(supported_properties); }; + OPENVINO_SUPPRESS_DEPRECATED_START // the rest are individual per-device settings (overriding the values set with perf modes) auto set_throughput_streams = [&]() { std::string key = getDeviceTypeFromName(device) + "_THROUGHPUT_STREAMS"; @@ -476,22 +481,23 @@ int main(int argc, char* argv[]) { if (it_streams != device_config.end()) device_nstreams[device] = it_streams->second.as(); }; + OPENVINO_SUPPRESS_DEPRECATED_END auto set_infer_precision = [&] { auto it_device_infer_precision = device_infer_precision.find(device); if (it_device_infer_precision != device_infer_precision.end()) { // set to user defined value - if (supported(ov::inference_precision.name())) { - device_config.emplace(ov::inference_precision(it_device_infer_precision->second)); + if (supported(ov::hint::inference_precision.name())) { + device_config.emplace(ov::hint::inference_precision(it_device_infer_precision->second)); } else if (is_virtual_device(device)) { update_device_config_for_virtual_device(it_device_infer_precision->second, device_config, - ov::inference_precision, + ov::hint::inference_precision, is_dev_set_property, is_load_config); } else { throw std::logic_error("Device " + device + " doesn't support config key '" + - ov::inference_precision.name() + "'! " + + ov::hint::inference_precision.name() + "'! " + "Please specify -infer_precision for correct devices in format " ":,:" + " or via configuration file."); diff --git a/samples/cpp/benchmark_app/remote_tensors_filling.cpp b/samples/cpp/benchmark_app/remote_tensors_filling.cpp index fa139d7485f141..9301a8d113210f 100644 --- a/samples/cpp/benchmark_app/remote_tensors_filling.cpp +++ b/samples/cpp/benchmark_app/remote_tensors_filling.cpp @@ -40,8 +40,10 @@ void fill_buffer_random(void* inputBuffer, void fill_buffer(void* inputBuffer, size_t elementsNum, const ov::element::Type& type) { if (type == ov::element::f32) { fill_buffer_random(inputBuffer, elementsNum); + } else if (type == ov::element::f64) { + fill_buffer_random(inputBuffer, elementsNum); } else if (type == ov::element::f16) { - fill_buffer_random(inputBuffer, elementsNum); + fill_buffer_random(inputBuffer, elementsNum); } else if (type == ov::element::i32) { fill_buffer_random(inputBuffer, elementsNum); } else if (type == ov::element::i64) { diff --git a/samples/cpp/benchmark_app/statistics_report.hpp b/samples/cpp/benchmark_app/statistics_report.hpp index d26db4e050d682..d346d89944405f 100644 --- a/samples/cpp/benchmark_app/statistics_report.hpp +++ b/samples/cpp/benchmark_app/statistics_report.hpp @@ -98,6 +98,8 @@ class StatisticsReport { enum class Category { COMMAND_LINE_PARAMETERS, RUNTIME_CONFIG, EXECUTION_RESULTS, EXECUTION_RESULTS_GROUPPED }; + virtual ~StatisticsReport() = default; + explicit StatisticsReport(Config config) : _config(std::move(config)) { _separator = #if defined _WIN32 || defined __CYGWIN__ diff --git a/samples/cpp/benchmark_app/utils.cpp b/samples/cpp/benchmark_app/utils.cpp index 8c53f3d192430a..48940059dfebd4 100644 --- a/samples/cpp/benchmark_app/utils.cpp +++ b/samples/cpp/benchmark_app/utils.cpp @@ -200,7 +200,7 @@ void update_device_config_for_virtual_device(const std::string& value, const auto& device_value = it.second; if (device_config.find(ov::device::properties.name()) == device_config.end() || (is_load_config && is_dev_set_property[device_name])) { - // Create ov::device::properties with ov::num_stream/ov::inference_precision and + // Create ov::device::properties with ov::num_stream/ov::hint::inference_precision and // 1. Insert this ov::device::properties into device config if this // ov::device::properties isn't existed. Otherwise, // 2. Replace the existed ov::device::properties within device config. diff --git a/samples/cpp/classification_sample_async/README.md b/samples/cpp/classification_sample_async/README.md index 3038a1e4291df4..4b460b40db4058 100644 --- a/samples/cpp/classification_sample_async/README.md +++ b/samples/cpp/classification_sample_async/README.md @@ -1,173 +1,200 @@ # Image Classification Async C++ Sample {#openvino_inference_engine_samples_classification_sample_async_README} -This sample demonstrates how to do inference of image classification models using Asynchronous Inference Request API. +@sphinxdirective + +This sample demonstrates how to do inference of image classification models using Asynchronous Inference Request API. + Models with only one input and output are supported. -In addition to regular images, the sample also supports single-channel `ubyte` images as an input for LeNet model. +In addition to regular images, the sample also supports single-channel ``ubyte`` images as an input for LeNet model. The following C++ API is used in the application: -| Feature | API | Description | -| :--- | :--- | :--- | -| Asynchronous Infer | `ov::InferRequest::start_async`, `ov::InferRequest::set_callback` | Do asynchronous inference with callback. | -| Model Operations | `ov::Output::get_shape`, `ov::set_batch` | Manage the model, operate with its batch size. Set batch size using input image count. | -| Infer Request Operations | `ov::InferRequest::get_input_tensor` | Get an input tensor. | -| Tensor Operations | `ov::shape_size`, `ov::Tensor::data` | Get a tensor shape size and its data. | - -Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). - -| Options | Values | -| :--- | :--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../samples/python/classification_sample_async/README.md) | - -## How It Works ++--------------------------+-----------------------------------------------------------------------+----------------------------------------------------------------------------------------+ +| Feature | API | Description | ++==========================+=======================================================================+========================================================================================+ +| Asynchronous Infer | ``ov::InferRequest::start_async``, ``ov::InferRequest::set_callback`` | Do asynchronous inference with callback. | ++--------------------------+-----------------------------------------------------------------------+----------------------------------------------------------------------------------------+ +| Model Operations | ``ov::Output::get_shape``, ``ov::set_batch`` | Manage the model, operate with its batch size. Set batch size using input image count. | ++--------------------------+-----------------------------------------------------------------------+----------------------------------------------------------------------------------------+ +| Infer Request Operations | ``ov::InferRequest::get_input_tensor`` | Get an input tensor. | ++--------------------------+-----------------------------------------------------------------------+----------------------------------------------------------------------------------------+ +| Tensor Operations | ``ov::shape_size``, ``ov::Tensor::data`` | Get a tensor shape size and its data. | ++--------------------------+-----------------------------------------------------------------------+----------------------------------------------------------------------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification C++ sample `. + ++----------------------------+-------------------------------------------------------------------------------------------------------+ +| Options | Values | ++============================+=======================================================================================================+ +| Validated Models | :doc:`alexnet `, :doc:`googlenet-v1 ` | ++----------------------------+-------------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | ++----------------------------+-------------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++----------------------------+-------------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`Python ` | ++----------------------------+-------------------------------------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application reads command line parameters and loads the specified model and input images (or a folder with images) to the OpenVINO™ Runtime plugin. The batch size of the model is set according to the number of read images. The batch mode is an independent attribute on the asynchronous mode. Asynchronous mode works efficiently with any batch size. -Then, the sample creates an inference request object and assigns completion callback for it. In scope of the completion callback -handling the inference request is executed again. +Then, the sample creates an inference request object and assigns completion callback for it. In scope of the completion callback handling the inference request is executed again. After that, the application starts inference for the first infer request and waits of 10th inference request execution being completed. The asynchronous mode might increase the throughput of the pictures. When inference is done, the application outputs data to the standard output stream. You can place labels in .labels file near the model to get pretty output. -You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +####### -Run the application with the `-h` option to see the usage instructions: +Run the application with the ``-h`` option to see the usage instructions: -``` -classification_sample_async -h -``` +.. code-block:: sh -Usage instructions: + classification_sample_async -h -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... +Usage instructions: -classification_sample_async [OPTION] -Options: +.. code-block:: sh + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + + classification_sample_async [OPTION] + Options: + + -h Print usage instructions. + -m "" Required. Path to an .xml file with a trained model. + -i "" Required. Path to a folder with images or path to image files: a .ubyte file for LeNet and a .bmp file for other models. + -d "" Optional. Specify the target device to infer on (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify the HETERO plugin. Sample will look for a suitable plugin for the device specified. + + Available target devices: - -h Print usage instructions. - -m "" Required. Path to an .xml file with a trained model. - -i "" Required. Path to a folder with images or path to image files: a .ubyte file for LeNet and a .bmp file for other models. - -d "" Optional. Specify the target device to infer on (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:" format to specify the HETERO plugin. Sample will look for a suitable plugin for the device specified. +To run the sample, you need to specify a model and image: -Available target devices: -``` +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available `here `. -To run the sample, you need to specify a model and image: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- You can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. +.. note:: + + - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +Example ++++++++ -### Example +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev[caffe] - ``` + .. code-block:: sh + + python -m pip install openvino-dev[caffe] + 2. Download a pre-trained model using: - ``` - omz_downloader --name googlenet-v1 - ``` + + .. code-block:: sh + + omz_downloader --name googlenet-v1 + 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: - ``` - omz_converter --name googlenet-v1 - ``` - -4. Perform inference of `dog.bmp` using `googlenet-v1` model on a `GPU`, for example: - ``` - classification_sample_async -m googlenet-v1.xml -i dog.bmp -d GPU - ``` - -## Sample Output - -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Parsing input parameters -[ INFO ] Files were added: 1 -[ INFO ] /images/dog.bmp -[ INFO ] Loading model files: -[ INFO ] /models/googlenet-v1.xml -[ INFO ] model name: GoogleNet -[ INFO ] inputs -[ INFO ] input name: data -[ INFO ] input type: f32 -[ INFO ] input shape: {1, 3, 224, 224} -[ INFO ] outputs -[ INFO ] output name: prob -[ INFO ] output type: f32 -[ INFO ] output shape: {1, 1000} -[ INFO ] Read input images -[ INFO ] Set batch size 1 -[ INFO ] model name: GoogleNet -[ INFO ] inputs -[ INFO ] input name: data -[ INFO ] input type: u8 -[ INFO ] input shape: {1, 224, 224, 3} -[ INFO ] outputs -[ INFO ] output name: prob -[ INFO ] output type: f32 -[ INFO ] output shape: {1, 1000} -[ INFO ] Loading model to the device GPU -[ INFO ] Create infer request -[ INFO ] Start inference (asynchronous executions) -[ INFO ] Completed 1 async request execution -[ INFO ] Completed 2 async request execution -[ INFO ] Completed 3 async request execution -[ INFO ] Completed 4 async request execution -[ INFO ] Completed 5 async request execution -[ INFO ] Completed 6 async request execution -[ INFO ] Completed 7 async request execution -[ INFO ] Completed 8 async request execution -[ INFO ] Completed 9 async request execution -[ INFO ] Completed 10 async request execution -[ INFO ] Completed async requests execution - -Top 10 results: - -Image /images/dog.bmp - -classid probability -------- ----------- -156 0.8935547 -218 0.0608215 -215 0.0217133 -219 0.0105667 -212 0.0018835 -217 0.0018730 -152 0.0018730 -157 0.0015745 -154 0.0012817 -220 0.0010099 -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) + + .. code-block:: sh + + omz_converter --name googlenet-v1 + +4. Perform inference of ``dog.bmp`` using ``googlenet-v1`` model on a ``GPU``, for example: + + .. code-block:: sh + + classification_sample_async -m googlenet-v1.xml -i dog.bmp -d GPU + +Sample Output +############# + +.. code-block:: sh + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Parsing input parameters + [ INFO ] Files were added: 1 + [ INFO ] /images/dog.bmp + [ INFO ] Loading model files: + [ INFO ] /models/googlenet-v1.xml + [ INFO ] model name: GoogleNet + [ INFO ] inputs + [ INFO ] input name: data + [ INFO ] input type: f32 + [ INFO ] input shape: {1, 3, 224, 224} + [ INFO ] outputs + [ INFO ] output name: prob + [ INFO ] output type: f32 + [ INFO ] output shape: {1, 1000} + [ INFO ] Read input images + [ INFO ] Set batch size 1 + [ INFO ] model name: GoogleNet + [ INFO ] inputs + [ INFO ] input name: data + [ INFO ] input type: u8 + [ INFO ] input shape: {1, 224, 224, 3} + [ INFO ] outputs + [ INFO ] output name: prob + [ INFO ] output type: f32 + [ INFO ] output shape: {1, 1000} + [ INFO ] Loading model to the device GPU + [ INFO ] Create infer request + [ INFO ] Start inference (asynchronous executions) + [ INFO ] Completed 1 async request execution + [ INFO ] Completed 2 async request execution + [ INFO ] Completed 3 async request execution + [ INFO ] Completed 4 async request execution + [ INFO ] Completed 5 async request execution + [ INFO ] Completed 6 async request execution + [ INFO ] Completed 7 async request execution + [ INFO ] Completed 8 async request execution + [ INFO ] Completed 9 async request execution + [ INFO ] Completed 10 async request execution + [ INFO ] Completed async requests execution + + Top 10 results: + + Image /images/dog.bmp + + classid probability + ------- ----------- + 156 0.8935547 + 218 0.0608215 + 215 0.0217133 + 219 0.0105667 + 212 0.0018835 + 217 0.0018730 + 152 0.0018730 + 157 0.0015745 + 154 0.0012817 + 220 0.0010099 + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/cpp/common/utils/include/samples/common.hpp b/samples/cpp/common/utils/include/samples/common.hpp index f5884163df5f49..bee4be89063a71 100644 --- a/samples/cpp/common/utils/include/samples/common.hpp +++ b/samples/cpp/common/utils/include/samples/common.hpp @@ -146,146 +146,6 @@ inline slog::LogStream& operator<<(slog::LogStream& os, const std::map> data, size_t classesNum, std::ostream& outFile) { - unsigned int seed = (unsigned int)time(NULL); - // Known colors for training classes from Cityscape dataset - static std::vector colors = { - {128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, {153, 153, 190}, {153, 153, 153}, - {30, 170, 250}, {0, 220, 220}, {35, 142, 107}, {152, 251, 152}, {180, 130, 70}, {60, 20, 220}, - {0, 0, 255}, {142, 0, 0}, {70, 0, 0}, {100, 60, 0}, {90, 0, 0}, {230, 0, 0}, - {32, 11, 119}, {0, 74, 111}, {81, 0, 81}}; - - while (classesNum > colors.size()) { - static std::mt19937 rng(seed); - std::uniform_int_distribution dist(0, 255); - Color color(dist(rng), dist(rng), dist(rng)); - colors.push_back(color); - } - - unsigned char file[14] = { - 'B', - 'M', // magic - 0, - 0, - 0, - 0, // size in bytes - 0, - 0, // app data - 0, - 0, // app data - 40 + 14, - 0, - 0, - 0 // start of data offset - }; - unsigned char info[40] = { - 40, 0, 0, 0, // info hd size - 0, 0, 0, 0, // width - 0, 0, 0, 0, // height - 1, 0, // number color planes - 24, 0, // bits per pixel - 0, 0, 0, 0, // compression is none - 0, 0, 0, 0, // image bits size - 0x13, 0x0B, 0, 0, // horz resolution in pixel / m - 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) - 0, 0, 0, 0, // #colors in palette - 0, 0, 0, 0, // #important colors - }; - - auto height = data.size(); - auto width = data.at(0).size(); - - OPENVINO_ASSERT( - height < (size_t)std::numeric_limits::max && width < (size_t)std::numeric_limits::max, - "File size is too big: ", - height, - " X ", - width); - - int padSize = static_cast(4 - (width * 3) % 4) % 4; - int sizeData = static_cast(width * height * 3 + height * padSize); - int sizeAll = sizeData + sizeof(file) + sizeof(info); - - file[2] = (unsigned char)(sizeAll); - file[3] = (unsigned char)(sizeAll >> 8); - file[4] = (unsigned char)(sizeAll >> 16); - file[5] = (unsigned char)(sizeAll >> 24); - - info[4] = (unsigned char)(width); - info[5] = (unsigned char)(width >> 8); - info[6] = (unsigned char)(width >> 16); - info[7] = (unsigned char)(width >> 24); - - int32_t negativeHeight = -(int32_t)height; - info[8] = (unsigned char)(negativeHeight); - info[9] = (unsigned char)(negativeHeight >> 8); - info[10] = (unsigned char)(negativeHeight >> 16); - info[11] = (unsigned char)(negativeHeight >> 24); - - info[20] = (unsigned char)(sizeData); - info[21] = (unsigned char)(sizeData >> 8); - info[22] = (unsigned char)(sizeData >> 16); - info[23] = (unsigned char)(sizeData >> 24); - - outFile.write(reinterpret_cast(file), sizeof(file)); - outFile.write(reinterpret_cast(info), sizeof(info)); - - unsigned char pad[3] = {0, 0, 0}; - - for (size_t y = 0; y < height; y++) { - for (size_t x = 0; x < width; x++) { - unsigned char pixel[3]; - size_t index = data.at(y).at(x); - pixel[0] = colors.at(index).red(); - pixel[1] = colors.at(index).green(); - pixel[2] = colors.at(index).blue(); - outFile.write(reinterpret_cast(pixel), 3); - } - outFile.write(reinterpret_cast(pad), padSize); - } -} - /** * @brief Writes output data to BMP image * @param name - image name @@ -396,7 +256,12 @@ static UNUSED void addRectangles(unsigned char* data, size_t width, std::vector rectangles, std::vector classes, - int thickness = 1) { + int thickness) { + struct Color { + unsigned char red; + unsigned char green; + unsigned char blue; + }; std::vector colors = {// colors to be used for bounding boxes {128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, {153, 153, 190}, {153, 153, 153}, {30, 170, 250}, {0, 220, 220}, {35, 142, 107}, {152, 251, 152}, @@ -451,12 +316,12 @@ static UNUSED void addRectangles(unsigned char* data, shift_first = (y + t) * width * 3; shift_second = (y + h - t) * width * 3; for (int ii = x; ii < x + w + 1; ii++) { - data[shift_first + ii * 3] = colors.at(cls).red(); - data[shift_first + ii * 3 + 1] = colors.at(cls).green(); - data[shift_first + ii * 3 + 2] = colors.at(cls).blue(); - data[shift_second + ii * 3] = colors.at(cls).red(); - data[shift_second + ii * 3 + 1] = colors.at(cls).green(); - data[shift_second + ii * 3 + 2] = colors.at(cls).blue(); + data[shift_first + ii * 3] = colors.at(cls).red; + data[shift_first + ii * 3 + 1] = colors.at(cls).green; + data[shift_first + ii * 3 + 2] = colors.at(cls).blue; + data[shift_second + ii * 3] = colors.at(cls).red; + data[shift_second + ii * 3 + 1] = colors.at(cls).green; + data[shift_second + ii * 3 + 2] = colors.at(cls).blue; } } @@ -464,510 +329,17 @@ static UNUSED void addRectangles(unsigned char* data, shift_first = (x + t) * 3; shift_second = (x + w - t) * 3; for (int ii = y; ii < y + h + 1; ii++) { - data[shift_first + ii * width * 3] = colors.at(cls).red(); - data[shift_first + ii * width * 3 + 1] = colors.at(cls).green(); - data[shift_first + ii * width * 3 + 2] = colors.at(cls).blue(); - data[shift_second + ii * width * 3] = colors.at(cls).red(); - data[shift_second + ii * width * 3 + 1] = colors.at(cls).green(); - data[shift_second + ii * width * 3 + 2] = colors.at(cls).blue(); + data[shift_first + ii * width * 3] = colors.at(cls).red; + data[shift_first + ii * width * 3 + 1] = colors.at(cls).green; + data[shift_first + ii * width * 3 + 2] = colors.at(cls).blue; + data[shift_second + ii * width * 3] = colors.at(cls).red; + data[shift_second + ii * width * 3 + 1] = colors.at(cls).green; + data[shift_second + ii * width * 3 + 2] = colors.at(cls).blue; } } } } -/** - * Write output data to image - * \param name - image name - * \param data - output data - * \param classesNum - the number of classes - * \return false if error else true - */ - -static UNUSED bool writeOutputBmp(unsigned char* data, size_t height, size_t width, std::ostream& outFile) { - unsigned char file[14] = { - 'B', - 'M', // magic - 0, - 0, - 0, - 0, // size in bytes - 0, - 0, // app data - 0, - 0, // app data - 40 + 14, - 0, - 0, - 0 // start of data offset - }; - unsigned char info[40] = { - 40, 0, 0, 0, // info hd size - 0, 0, 0, 0, // width - 0, 0, 0, 0, // height - 1, 0, // number color planes - 24, 0, // bits per pixel - 0, 0, 0, 0, // compression is none - 0, 0, 0, 0, // image bits size - 0x13, 0x0B, 0, 0, // horz resolution in pixel / m - 0x13, 0x0B, 0, 0, // vert resolution (0x03C3 = 96 dpi, 0x0B13 = 72 dpi) - 0, 0, 0, 0, // #colors in palette - 0, 0, 0, 0, // #important colors - }; - - OPENVINO_ASSERT( - height < (size_t)std::numeric_limits::max && width < (size_t)std::numeric_limits::max, - "File size is too big: ", - height, - " X ", - width); - - int padSize = static_cast(4 - (width * 3) % 4) % 4; - int sizeData = static_cast(width * height * 3 + height * padSize); - int sizeAll = sizeData + sizeof(file) + sizeof(info); - - file[2] = (unsigned char)(sizeAll); - file[3] = (unsigned char)(sizeAll >> 8); - file[4] = (unsigned char)(sizeAll >> 16); - file[5] = (unsigned char)(sizeAll >> 24); - - info[4] = (unsigned char)(width); - info[5] = (unsigned char)(width >> 8); - info[6] = (unsigned char)(width >> 16); - info[7] = (unsigned char)(width >> 24); - - int32_t negativeHeight = -(int32_t)height; - info[8] = (unsigned char)(negativeHeight); - info[9] = (unsigned char)(negativeHeight >> 8); - info[10] = (unsigned char)(negativeHeight >> 16); - info[11] = (unsigned char)(negativeHeight >> 24); - - info[20] = (unsigned char)(sizeData); - info[21] = (unsigned char)(sizeData >> 8); - info[22] = (unsigned char)(sizeData >> 16); - info[23] = (unsigned char)(sizeData >> 24); - - outFile.write(reinterpret_cast(file), sizeof(file)); - outFile.write(reinterpret_cast(info), sizeof(info)); - - unsigned char pad[3] = {0, 0, 0}; - - for (size_t y = 0; y < height; y++) { - for (size_t x = 0; x < width; x++) { - unsigned char pixel[3]; - pixel[0] = data[y * width * 3 + x * 3]; - pixel[1] = data[y * width * 3 + x * 3 + 1]; - pixel[2] = data[y * width * 3 + x * 3 + 2]; - outFile.write(reinterpret_cast(pixel), 3); - } - outFile.write(reinterpret_cast(pad), padSize); - } - - return true; -} - -static UNUSED void printPerformanceCounts(const std::map& performanceMap, - std::ostream& stream, - std::string deviceName, - bool bshowHeader = true) { - std::chrono::microseconds totalTime = std::chrono::microseconds::zero(); - // Print performance counts - if (bshowHeader) { - stream << std::endl << "performance counts:" << std::endl << std::endl; - } - std::ios::fmtflags fmt(std::cout.flags()); - - for (const auto& it : performanceMap) { - std::string toPrint(it.first); - const int maxLayerName = 30; - - if (it.first.length() >= maxLayerName) { - toPrint = it.first.substr(0, maxLayerName - 4); - toPrint += "..."; - } - - stream << std::setw(maxLayerName) << std::left << toPrint; - switch (it.second.status) { - case ov::ProfilingInfo::Status::EXECUTED: - stream << std::setw(15) << std::left << "EXECUTED"; - break; - case ov::ProfilingInfo::Status::NOT_RUN: - stream << std::setw(15) << std::left << "NOT_RUN"; - break; - case ov::ProfilingInfo::Status::OPTIMIZED_OUT: - stream << std::setw(15) << std::left << "OPTIMIZED_OUT"; - break; - } - stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.node_type) + " "; - stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.real_time.count()); - stream << std::setw(20) << std::left << "cpu: " + std::to_string(it.second.cpu_time.count()); - stream << " execType: " << it.second.exec_type << std::endl; - if (it.second.real_time.count() > 0) { - totalTime += it.second.real_time; - } - } - stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime.count()) << " microseconds" - << std::endl; - stream << std::endl; - stream << "Full device name: " << deviceName << std::endl; - stream << std::endl; - stream.flags(fmt); -} - -/** - * @brief This class represents an object that is found by an object detection net - */ -class DetectedObject { -public: - int objectType; - float xmin, xmax, ymin, ymax, prob; - bool difficult; - - DetectedObject(int _objectType, - float _xmin, - float _ymin, - float _xmax, - float _ymax, - float _prob, - bool _difficult = false) - : objectType(_objectType), - xmin(_xmin), - xmax(_xmax), - ymin(_ymin), - ymax(_ymax), - prob(_prob), - difficult(_difficult) {} - - DetectedObject(const DetectedObject& other) = default; - - static float ioU(const DetectedObject& detectedObject1_, const DetectedObject& detectedObject2_) { - // Add small space to eliminate empty squares - float epsilon = 0; // 1e-5f; - - DetectedObject detectedObject1(detectedObject1_.objectType, - (detectedObject1_.xmin - epsilon), - (detectedObject1_.ymin - epsilon), - (detectedObject1_.xmax - epsilon), - (detectedObject1_.ymax - epsilon), - detectedObject1_.prob); - DetectedObject detectedObject2(detectedObject2_.objectType, - (detectedObject2_.xmin + epsilon), - (detectedObject2_.ymin + epsilon), - (detectedObject2_.xmax), - (detectedObject2_.ymax), - detectedObject2_.prob); - - if (detectedObject1.objectType != detectedObject2.objectType) { - // objects are different, so the result is 0 - return 0.0f; - } - - if (detectedObject1.xmax < detectedObject1.xmin) - return 0.0; - if (detectedObject1.ymax < detectedObject1.ymin) - return 0.0; - if (detectedObject2.xmax < detectedObject2.xmin) - return 0.0; - if (detectedObject2.ymax < detectedObject2.ymin) - return 0.0; - - float xmin = (std::max)(detectedObject1.xmin, detectedObject2.xmin); - float ymin = (std::max)(detectedObject1.ymin, detectedObject2.ymin); - float xmax = (std::min)(detectedObject1.xmax, detectedObject2.xmax); - float ymax = (std::min)(detectedObject1.ymax, detectedObject2.ymax); - - // Caffe adds 1 to every length if the box isn't normalized. So do we... - float addendum; - if (xmax > 1 || ymax > 1) - addendum = 1; - else - addendum = 0; - - // intersection - float intr; - if ((xmax >= xmin) && (ymax >= ymin)) { - intr = (addendum + xmax - xmin) * (addendum + ymax - ymin); - } else { - intr = 0.0f; - } - - // union - float square1 = (addendum + detectedObject1.xmax - detectedObject1.xmin) * - (addendum + detectedObject1.ymax - detectedObject1.ymin); - float square2 = (addendum + detectedObject2.xmax - detectedObject2.xmin) * - (addendum + detectedObject2.ymax - detectedObject2.ymin); - - float unn = square1 + square2 - intr; - - return static_cast(intr) / unn; - } - - DetectedObject scale(float scale_x, float scale_y) const { - return DetectedObject(objectType, - xmin * scale_x, - ymin * scale_y, - xmax * scale_x, - ymax * scale_y, - prob, - difficult); - } -}; - -class ImageDescription { -public: - const std::list alist; - const bool check_probs; - - explicit ImageDescription(const std::list& _alist, bool _check_probs = false) - : alist(_alist), - check_probs(_check_probs) {} - - static float ioUMultiple(const ImageDescription& detectedObjects, const ImageDescription& desiredObjects) { - const ImageDescription *detectedObjectsSmall, *detectedObjectsBig; - bool check_probs = desiredObjects.check_probs; - - if (detectedObjects.alist.size() < desiredObjects.alist.size()) { - detectedObjectsSmall = &detectedObjects; - detectedObjectsBig = &desiredObjects; - } else { - detectedObjectsSmall = &desiredObjects; - detectedObjectsBig = &detectedObjects; - } - - std::list doS = detectedObjectsSmall->alist; - std::list doB = detectedObjectsBig->alist; - - float fullScore = 0.0f; - while (doS.size() > 0) { - float score = 0.0f; - std::list::iterator bestJ = doB.end(); - for (auto j = doB.begin(); j != doB.end(); j++) { - float curscore = DetectedObject::ioU(*doS.begin(), *j); - if (score < curscore) { - score = curscore; - bestJ = j; - } - } - - float coeff = 1.0; - if (check_probs) { - if (bestJ != doB.end()) { - float mn = std::min((*bestJ).prob, (*doS.begin()).prob); - float mx = std::max((*bestJ).prob, (*doS.begin()).prob); - - coeff = mn / mx; - } - } - - doS.pop_front(); - if (bestJ != doB.end()) - doB.erase(bestJ); - fullScore += coeff * score; - } - fullScore /= detectedObjectsBig->alist.size(); - - return fullScore; - } - - ImageDescription scale(float scale_x, float scale_y) const { - std::list slist; - for (auto& dob : alist) { - slist.push_back(dob.scale(scale_x, scale_y)); - } - return ImageDescription(slist, check_probs); - } -}; - -struct AveragePrecisionCalculator { -private: - enum MatchKind { TruePositive, FalsePositive }; - - /** - * Here we count all TP and FP matches for all the classes in all the images. - */ - std::map>> matches; - - std::map N; - - double threshold; - - static bool SortBBoxDescend(const DetectedObject& bbox1, const DetectedObject& bbox2) { - return bbox1.prob > bbox2.prob; - } - - static bool SortPairDescend(const std::pair& p1, const std::pair& p2) { - return p1.first > p2.first; - } - -public: - explicit AveragePrecisionCalculator(double _threshold) : threshold(_threshold) {} - - // gt_bboxes -> des - // bboxes -> det - - void consumeImage(const ImageDescription& detectedObjects, const ImageDescription& desiredObjects) { - // Collecting IoU values - std::vector visited(desiredObjects.alist.size(), false); - std::vector bboxes{std::begin(detectedObjects.alist), std::end(detectedObjects.alist)}; - std::sort(bboxes.begin(), bboxes.end(), SortBBoxDescend); - - for (auto&& detObj : bboxes) { - // Searching for the best match to this detection - // Searching for desired object - float overlap_max = -1; - int jmax = -1; - auto desmax = desiredObjects.alist.end(); - - int j = 0; - for (auto desObj = desiredObjects.alist.begin(); desObj != desiredObjects.alist.end(); desObj++, j++) { - double iou = DetectedObject::ioU(detObj, *desObj); - if (iou > overlap_max) { - overlap_max = static_cast(iou); - jmax = j; - desmax = desObj; - } - } - - MatchKind mk; - if (overlap_max >= threshold) { - if (!desmax->difficult) { - if (!visited[jmax]) { - mk = TruePositive; - visited[jmax] = true; - } else { - mk = FalsePositive; - } - matches[detObj.objectType].push_back(std::make_pair(detObj.prob, mk)); - } - } else { - mk = FalsePositive; - matches[detObj.objectType].push_back(std::make_pair(detObj.prob, mk)); - } - } - - for (auto desObj = desiredObjects.alist.begin(); desObj != desiredObjects.alist.end(); desObj++) { - if (!desObj->difficult) { - N[desObj->objectType]++; - } - } - } - - std::map calculateAveragePrecisionPerClass() const { - /** - * Precision-to-TP curve per class (a variation of precision-to-recall curve without - * dividing into N) - */ - std::map> precisionToTP; - - std::map res; - - for (auto m : matches) { - // Sorting - std::sort(m.second.begin(), m.second.end(), SortPairDescend); - - int clazz = m.first; - int TP = 0, FP = 0; - - std::vector prec; - std::vector rec; - - for (auto mm : m.second) { - // Here we are descending in a probability value - MatchKind mk = mm.second; - if (mk == TruePositive) - TP++; - else if (mk == FalsePositive) - FP++; - - double precision = static_cast(TP) / (TP + FP); - double recall = 0; - if (N.find(clazz) != N.end()) { - recall = static_cast(TP) / N.at(clazz); - } - - prec.push_back(precision); - rec.push_back(recall); - } - - int num = static_cast(rec.size()); - - // 11point from Caffe - double ap = 0; - std::vector max_precs(11, 0.); - int start_idx = num - 1; - for (int j = 10; j >= 0; --j) { - for (int i = start_idx; i >= 0; --i) { - if (rec[i] < j / 10.) { - start_idx = i; - if (j > 0) { - max_precs[j - 1] = max_precs[j]; - } - break; - } else { - if (max_precs[j] < prec[i]) { - max_precs[j] = static_cast(prec[i]); - } - } - } - } - for (int j = 10; j >= 0; --j) { - ap += max_precs[j] / 11; - } - res[clazz] = ap; - } - - return res; - } -}; - -/** - * @brief Adds colored rectangles to the image - * @param data - data where rectangles are put - * @param height - height of the rectangle - * @param width - width of the rectangle - * @param detectedObjects - vector of detected objects - */ -static UNUSED void addRectangles(unsigned char* data, - size_t height, - size_t width, - std::vector detectedObjects) { - std::vector colors = {{128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, {153, 153, 190}, - {153, 153, 153}, {30, 170, 250}, {0, 220, 220}, {35, 142, 107}, {152, 251, 152}, - {180, 130, 70}, {60, 20, 220}, {0, 0, 255}, {142, 0, 0}, {70, 0, 0}, - {100, 60, 0}, {90, 0, 0}, {230, 0, 0}, {32, 11, 119}, {0, 74, 111}, - {81, 0, 81}}; - - for (size_t i = 0; i < detectedObjects.size(); i++) { - int cls = detectedObjects[i].objectType % colors.size(); - - int xmin = static_cast(detectedObjects[i].xmin * width); - int xmax = static_cast(detectedObjects[i].xmax * width); - int ymin = static_cast(detectedObjects[i].ymin * height); - int ymax = static_cast(detectedObjects[i].ymax * height); - - size_t shift_first = ymin * width * 3; - size_t shift_second = ymax * width * 3; - for (int x = xmin; x < xmax; x++) { - data[shift_first + x * 3] = colors.at(cls).red(); - data[shift_first + x * 3 + 1] = colors.at(cls).green(); - data[shift_first + x * 3 + 2] = colors.at(cls).blue(); - data[shift_second + x * 3] = colors.at(cls).red(); - data[shift_second + x * 3 + 1] = colors.at(cls).green(); - data[shift_second + x * 3 + 2] = colors.at(cls).blue(); - } - - shift_first = xmin * 3; - shift_second = xmax * 3; - for (int y = ymin; y < ymax; y++) { - data[shift_first + y * width * 3] = colors.at(cls).red(); - data[shift_first + y * width * 3 + 1] = colors.at(cls).green(); - data[shift_first + y * width * 3 + 2] = colors.at(cls).blue(); - data[shift_second + y * width * 3] = colors.at(cls).red(); - data[shift_second + y * width * 3 + 1] = colors.at(cls).green(); - data[shift_second + y * width * 3 + 2] = colors.at(cls).blue(); - } - } -} - inline void showAvailableDevices() { ov::Core core; std::vector devices = core.get_available_devices(); @@ -980,16 +352,6 @@ inline void showAvailableDevices() { std::cout << std::endl; } -/** - * @brief Parse text config file. The file must have the following format (with space a delimeter): - * CONFIG_NAME1 CONFIG_VALUE1 - * CONFIG_NAME2 CONFIG_VALUE2 - * - * @param configName - filename for a file with config options - * @param comment - lines starting with symbol `comment` are skipped - */ -std::map parseConfig(const std::string& configName, char comment = '#'); - inline std::string getFullDeviceName(ov::Core& core, std::string device) { try { return core.get_property(device, ov::device::full_name); @@ -1066,15 +428,6 @@ static UNUSED void printPerformanceCounts(std::vector perform stream.flags(fmt); } -static UNUSED void printPerformanceCounts(ov::InferRequest request, - std::ostream& stream, - std::string deviceName, - bool bshowHeader = true, - int precision = 3) { - auto performanceMap = request.get_profiling_info(); - printPerformanceCounts(performanceMap, stream, deviceName, bshowHeader, precision); -} - static inline std::string double_to_string(const double number) { std::stringstream ss; ss << std::fixed << std::setprecision(2) << number; diff --git a/samples/cpp/common/utils/include/samples/ocv_common.hpp b/samples/cpp/common/utils/include/samples/ocv_common.hpp deleted file mode 100644 index a1198760e25ce8..00000000000000 --- a/samples/cpp/common/utils/include/samples/ocv_common.hpp +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief a header file with common samples functionality using OpenCV - * @file ocv_common.hpp - */ - -#pragma once - -#include - -#include "openvino/openvino.hpp" -#include "samples/common.hpp" - -/** - * @brief Sets image data stored in cv::Mat object to a given Blob object. - * @param orig_image - given cv::Mat object with an image data. - * @param blob - Blob object which to be filled by an image data. - * @param batchIndex - batch index of an image inside of the blob. - */ -template -void matU8ToBlob(const cv::Mat& orig_image, InferenceEngine::Blob::Ptr& blob, int batchIndex = 0) { - InferenceEngine::SizeVector blobSize = blob->getTensorDesc().getDims(); - const size_t width = blobSize[3]; - const size_t height = blobSize[2]; - const size_t channels = blobSize[1]; - InferenceEngine::MemoryBlob::Ptr mblob = InferenceEngine::as(blob); - OPENVINO_ASSERT(mblob, - "We expect blob to be inherited from MemoryBlob in matU8ToBlob, " - "but by fact we were not able to cast inputBlob to MemoryBlob"); - // locked memory holder should be alive all time while access to its buffer happens - auto mblobHolder = mblob->wmap(); - - T* blob_data = mblobHolder.as(); - - cv::Mat resized_image(orig_image); - if (static_cast(width) != orig_image.size().width || static_cast(height) != orig_image.size().height) { - cv::resize(orig_image, resized_image, cv::Size(width, height)); - } - - int batchOffset = batchIndex * width * height * channels; - - for (size_t c = 0; c < channels; c++) { - for (size_t h = 0; h < height; h++) { - for (size_t w = 0; w < width; w++) { - blob_data[batchOffset + c * width * height + h * width + w] = resized_image.at(h, w)[c]; - } - } - } -} - -/** - * @brief Wraps data stored inside of a passed cv::Mat object by new Blob pointer. - * @note: No memory allocation is happened. The blob just points to already existing - * cv::Mat data. - * @param mat - given cv::Mat object with an image data. - * @return resulting Blob pointer. - */ -static UNUSED InferenceEngine::Blob::Ptr wrapMat2Blob(const cv::Mat& mat) { - size_t channels = mat.channels(); - size_t height = mat.size().height; - size_t width = mat.size().width; - - size_t strideH = mat.step.buf[0]; - size_t strideW = mat.step.buf[1]; - - bool is_dense = strideW == channels && strideH == channels * width; - - OPENVINO_ASSERT(is_dense, "Doesn't support conversion from not dense cv::Mat"); - - InferenceEngine::TensorDesc tDesc(InferenceEngine::Precision::U8, - {1, channels, height, width}, - InferenceEngine::Layout::NHWC); - - return InferenceEngine::make_shared_blob(tDesc, mat.data); -} - -static UNUSED ov::Tensor wrapMat2Tensor(const cv::Mat& mat) { - const size_t channels = mat.channels(); - const size_t height = mat.size().height; - const size_t width = mat.size().width; - - const size_t strideH = mat.step.buf[0]; - const size_t strideW = mat.step.buf[1]; - - const bool is_dense = strideW == channels && strideH == channels * width; - OPENVINO_ASSERT(is_dense, "Doesn't support conversion from not dense cv::Mat"); - - return ov::Tensor(ov::element::u8, ov::Shape{1, height, width, channels}, mat.data); -} diff --git a/samples/cpp/common/utils/include/samples/vpu/vpu_tools_common.hpp b/samples/cpp/common/utils/include/samples/vpu/vpu_tools_common.hpp deleted file mode 100644 index f31d6e374f66e0..00000000000000 --- a/samples/cpp/common/utils/include/samples/vpu/vpu_tools_common.hpp +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -static std::map parseConfig(const std::string& configName, char comment = '#') { - std::map config = {}; - - std::ifstream file(configName); - if (!file.is_open()) { - return config; - } - - std::string key, value; - while (file >> key >> value) { - if (key.empty() || key[0] == comment) { - continue; - } - config[key] = value; - } - - return config; -} diff --git a/samples/cpp/common/utils/src/common.cpp b/samples/cpp/common/utils/src/common.cpp deleted file mode 100644 index 1cf43fddf2a40d..00000000000000 --- a/samples/cpp/common/utils/src/common.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "samples/common.hpp" - -std::map parseConfig(const std::string& configName, char comment) { - std::map config = {}; - - std::ifstream file(configName); - if (!file.is_open()) { - return config; - } - - std::string key, value; - while (file >> key >> value) { - if (key.empty() || key[0] == comment) { - continue; - } - config[key] = value; - } - - return config; -} diff --git a/samples/cpp/hello_classification/README.md b/samples/cpp/hello_classification/README.md index 6c129904a7a6ff..51e05bb077c6fc 100644 --- a/samples/cpp/hello_classification/README.md +++ b/samples/cpp/hello_classification/README.md @@ -1,117 +1,155 @@ # Hello Classification C++ Sample {#openvino_inference_engine_samples_hello_classification_README} -This sample demonstrates how to do inference of image classification models using Synchronous Inference Request API. -Models with only one input and output are supported. +@sphinxdirective -The following C++ API is used in the application: +This sample demonstrates how to do inference of image classification models using Synchronous Inference Request API. -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | `ov::get_openvino_version` | Get Openvino API version | -| Basic Infer Flow | `ov::Core::read_model`, `ov::Core::compile_model`, `ov::CompiledModel::create_infer_request`, `ov::InferRequest::set_input_tensor`, `ov::InferRequest::get_output_tensor` | Common API to do inference: read and compile a model, create an infer request, configure input and output tensors | -| Synchronous Infer | `ov::InferRequest::infer` | Do synchronous inference | -| Model Operations | `ov::Model::inputs`, `ov::Model::outputs` | Get inputs and outputs of a model | -| Tensor Operations | `ov::Tensor::get_shape` | Get a tensor shape | -| Preprocessing | `ov::preprocess::InputTensorInfo::set_element_type`, `ov::preprocess::InputTensorInfo::set_layout`, `ov::preprocess::InputTensorInfo::set_spatial_static_shape`, `ov::preprocess::PreProcessSteps::resize`, `ov::preprocess::InputModelInfo::set_layout`, `ov::preprocess::OutputTensorInfo::set_element_type`, `ov::preprocess::PrePostProcessor::build` | Set image of the original size as input for a model with other input size. Resize and layout conversions are performed automatically by the corresponding plugin just before inference. | +Models with only one input and output are supported. -| Options | Values | -| :--- | :--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C](../../../samples/c/hello_classification/README.md), [Python](../../../samples/python/hello_classification/README.md) | +The following C++ API is used in the application: -## How It Works ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Feature | API | Description | ++=====================================+================================================================+=========================================================================================================================================================================================+ +| OpenVINO Runtime Version | ``ov::get_openvino_version`` | Get Openvino API version | ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Basic Infer Flow | ``ov::Core::read_model``, | Common API to do inference: read and compile a model, create an infer request, configure input and output tensors | +| | ``ov::Core::compile_model``, | | +| | ``ov::CompiledModel::create_infer_request``, | | +| | ``ov::InferRequest::set_input_tensor``, | | +| | ``ov::InferRequest::get_output_tensor`` | | ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Synchronous Infer | ``ov::InferRequest::infer`` | Do synchronous inference | ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Operations | ``ov::Model::inputs``, | Get inputs and outputs of a model | +| | ``ov::Model::outputs`` | | ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Tensor Operations | ``ov::Tensor::get_shape`` | Get a tensor shape | ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Preprocessing | ``ov::preprocess::InputTensorInfo::set_element_type``, | Set image of the original size as input for a model with other input size. Resize and layout conversions are performed automatically by the corresponding plugin just before inference. | +| | ``ov::preprocess::InputTensorInfo::set_layout``, | | +| | ``ov::preprocess::InputTensorInfo::set_spatial_static_shape``, | | +| | ``ov::preprocess::PreProcessSteps::resize``, | | +| | ``ov::preprocess::InputModelInfo::set_layout``, | | +| | ``ov::preprocess::OutputTensorInfo::set_element_type``, | | +| | ``ov::preprocess::PrePostProcessor::build`` | | ++-------------------------------------+----------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + ++-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Options | Values | ++=====================================+=======================================================================================================================================================================================+ +| Validated Models | :doc:`alexnet `, :doc:`googlenet-v1 ` | ++-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | ++-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`C `, :doc:`Python ` | ++-------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application reads command line parameters, prepares input data, loads a specified model and image to the OpenVINO™ Runtime plugin and performs synchronous inference. Then processes output data and write it to a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +####### -``` -hello_classification -``` +.. code-block:: console + + hello_classification To run the sample, you need to specify a model and image: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- You can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `__. + +.. note:: + + - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. -### Example +Example ++++++++ -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev[caffe] - ``` +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: + + .. code-block:: console + + python -m pip install openvino-dev[caffe] 2. Download a pre-trained model using: - ``` - omz_downloader --name googlenet-v1 - ``` + + .. code-block:: console + + omz_downloader --name googlenet-v1 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: - ``` - omz_converter --name googlenet-v1 - ``` + + .. code-block:: console + + omz_converter --name googlenet-v1 -4. Perform inference of `car.bmp` using the `googlenet-v1` model on a `GPU`, for example: - ``` - hello_classification googlenet-v1.xml car.bmp GPU - ``` +4. Perform inference of ``car.bmp`` using the ``googlenet-v1`` model on a ``GPU``, for example: + + .. code-block:: console + + hello_classification googlenet-v1.xml car.bmp GPU -## Sample Output +Sample Output +############# The application outputs top-10 inference results. -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Loading model files: /models/googlenet-v1.xml -[ INFO ] model name: GoogleNet -[ INFO ] inputs -[ INFO ] input name: data -[ INFO ] input type: f32 -[ INFO ] input shape: {1, 3, 224, 224} -[ INFO ] outputs -[ INFO ] output name: prob -[ INFO ] output type: f32 -[ INFO ] output shape: {1, 1000} - -Top 10 results: - -Image /images/car.bmp - -classid probability -------- ----------- -656 0.8139648 -654 0.0550537 -468 0.0178375 -436 0.0165405 -705 0.0111694 -817 0.0105820 -581 0.0086823 -575 0.0077515 -734 0.0064468 -785 0.0043983 -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: console + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Loading model files: /models/googlenet-v1.xml + [ INFO ] model name: GoogleNet + [ INFO ] inputs + [ INFO ] input name: data + [ INFO ] input type: f32 + [ INFO ] input shape: {1, 3, 224, 224} + [ INFO ] outputs + [ INFO ] output name: prob + [ INFO ] output type: f32 + [ INFO ] output shape: {1, 1000} + + Top 10 results: + + Image /images/car.bmp + + classid probability + ------- ----------- + 656 0.8139648 + 654 0.0550537 + 468 0.0178375 + 436 0.0165405 + 705 0.0111694 + 817 0.0105820 + 581 0.0086823 + 575 0.0077515 + 734 0.0064468 + 785 0.0043983 + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/cpp/hello_nv12_input_classification/README.md b/samples/cpp/hello_nv12_input_classification/README.md index 18ddc5474333a9..5c3c89b168da3c 100644 --- a/samples/cpp/hello_nv12_input_classification/README.md +++ b/samples/cpp/hello_nv12_input_classification/README.md @@ -1,132 +1,153 @@ # Hello NV12 Input Classification C++ Sample {#openvino_inference_engine_samples_hello_nv12_input_classification_README} +@sphinxdirective + This sample demonstrates how to execute an inference of image classification models with images in NV12 color format using Synchronous Inference Request API. The following C++ API is used in the application: -| Feature | API | Description | -| :--- | :--- | :--- | -| Node Operations | `ov::Output::get_any_name` | Get a layer name | -| Infer Request Operations | `ov::InferRequest::set_tensor`, `ov::InferRequest::get_tensor` | Operate with tensors | -| Preprocessing | `ov::preprocess::InputTensorInfo::set_color_format`, `ov::preprocess::PreProcessSteps::convert_element_type`, `ov::preprocess::PreProcessSteps::convert_color` | Change the color format of the input data | ++-------------------------------------+-------------------------------------------------------------+-------------------------------------------+ +| Feature | API | Description | ++=====================================+=============================================================+===========================================+ +| Node Operations | ``ov::Output::get_any_name`` | Get a layer name | ++-------------------------------------+-------------------------------------------------------------+-------------------------------------------+ +| Infer Request Operations | ``ov::InferRequest::set_tensor``, | Operate with tensors | +| | ``ov::InferRequest::get_tensor`` | | ++-------------------------------------+-------------------------------------------------------------+-------------------------------------------+ +| Preprocessing | ``ov::preprocess::InputTensorInfo::set_color_format``, | Change the color format of the input data | +| | ``ov::preprocess::PreProcessSteps::convert_element_type``, | | +| | ``ov::preprocess::PreProcessSteps::convert_color`` | | ++-------------------------------------+-------------------------------------------------------------+-------------------------------------------+ + -Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification C++ sample `. -| Options | Values | -| :--- |:--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Validated images | An uncompressed image in the NV12 color format - \*.yuv | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C](../../../samples/c/hello_nv12_input_classification/README.md) | ++-------------------------------------+--------------------------------------------------------------------------------------------------+ +| Options | Values | ++=====================================+==================================================================================================+ +| Validated Models | :doc:`alexnet ` | ++-------------------------------------+--------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | ++-------------------------------------+--------------------------------------------------------------------------------------------------+ +| Validated images | An uncompressed image in the NV12 color format - \*.yuv | ++-------------------------------------+--------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++-------------------------------------+--------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`C ` | ++-------------------------------------+--------------------------------------------------------------------------------------------------+ -## How It Works +How It Works +############ -At startup, the sample application reads command line parameters, loads the specified model and an -image in the NV12 color format to an OpenVINO™ Runtime plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application outputs data to the standard output stream. You can place labels in .labels file near the model to get pretty output. +At startup, the sample application reads command line parameters, loads the specified model and an image in the NV12 color format to an OpenVINO™ Runtime plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application outputs data to the standard output stream. You can place labels in .labels file near the model to get pretty output. -You can see the explicit description of each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +####### -``` -hello_nv12_input_classification -``` +.. code-block:: console + + hello_nv12_input_classification To run the sample, you need to specify a model and image: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- You can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. - -The sample accepts an uncompressed image in the NV12 color format. To run the sample, you need to -convert your BGR/RGB image to NV12. To do this, you can use one of the widely available tools such -as FFmpeg\* or GStreamer\*. The following command shows how to convert an ordinary image into an -uncompressed NV12 image using FFmpeg: - -```sh -ffmpeg -i cat.jpg -pix_fmt nv12 car.yuv -``` - -> **NOTES**: -> -> - Because the sample reads raw image files, you should provide a correct image size along with the -> image path. The sample expects the logical size of the image, not the buffer size. For example, -> for 640x480 BGR/RGB image the corresponding NV12 logical image size is also 640x480, whereas the -> buffer size is 640x720. -> - By default, this sample expects that model input has BGR channels order. If you trained your -> model to work with RGB order, you need to reconvert your model using the Model Optimizer tool -> with `--reverse_input_channels` argument specified. For more information about the argument, -> refer to **When to Reverse Input Channels** section of -> [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. - -### Example + +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `__. + +The sample accepts an uncompressed image in the NV12 color format. To run the sample, you need to convert your BGR/RGB image to NV12. To do this, you can use one of the widely available tools such as FFmpeg\* or GStreamer\*. The following command shows how to convert an ordinary image into an uncompressed NV12 image using FFmpeg: + +.. code-block:: bash + + ffmpeg -i cat.jpg -pix_fmt nv12 car.yuv + + +.. note:: + + - Because the sample reads raw image files, you should provide a correct image size along with the image path. The sample expects the logical size of the image, not the buffer size. For example, for 640x480 BGR/RGB image the corresponding NV12 logical image size is also 640x480, whereas the buffer size is 640x720. + - By default, this sample expects that model input has BGR channels order. If you trained your model to work with RGB order, you need to reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + +Example ++++++++ 1. Install openvino-dev python package if you don't have it to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev[caffe] - ``` + + .. code-block:: console + + python -m pip install openvino-dev[caffe] 2. Download a pre-trained model: - ``` - omz_downloader --name alexnet - ``` + + .. code-block:: console + + omz_downloader --name alexnet 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: - ``` - omz_converter --name alexnet - ``` + + .. code-block:: console + + omz_converter --name alexnet -4. Perform inference of NV12 image using `alexnet` model on a `CPU`, for example: - ``` - hello_nv12_input_classification alexnet.xml car.yuv 300x300 CPU - ``` +4. Perform inference of NV12 image using ``alexnet`` model on a ``CPU``, for example: + + .. code-block:: console + + hello_nv12_input_classification alexnet.xml car.yuv 300x300 CPU -## Sample Output + +Sample Output +############# The application outputs top-10 inference results. -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Loading model files: \models\alexnet.xml -[ INFO ] model name: AlexNet -[ INFO ] inputs -[ INFO ] input name: data -[ INFO ] input type: f32 -[ INFO ] input shape: {1, 3, 227, 227} -[ INFO ] outputs -[ INFO ] output name: prob -[ INFO ] output type: f32 -[ INFO ] output shape: {1, 1000} - -Top 10 results: - -Image \images\car.yuv - -classid probability -------- ----------- -656 0.6668988 -654 0.1125269 -581 0.0679280 -874 0.0340229 -436 0.0257744 -817 0.0169367 -675 0.0110199 -511 0.0106134 -569 0.0083373 -717 0.0061734 -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: console + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Loading model files: \models\alexnet.xml + [ INFO ] model name: AlexNet + [ INFO ] inputs + [ INFO ] input name: data + [ INFO ] input type: f32 + [ INFO ] input shape: {1, 3, 227, 227} + [ INFO ] outputs + [ INFO ] output name: prob + [ INFO ] output type: f32 + [ INFO ] output shape: {1, 1000} + + Top 10 results: + + Image \images\car.yuv + + classid probability + ------- ----------- + 656 0.6668988 + 654 0.1125269 + 581 0.0679280 + 874 0.0340229 + 436 0.0257744 + 817 0.0169367 + 675 0.0110199 + 511 0.0106134 + 569 0.0083373 + 717 0.0061734 + + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/cpp/hello_query_device/README.md b/samples/cpp/hello_query_device/README.md index 202f733542becc..b6ce6fa099bbcf 100644 --- a/samples/cpp/hello_query_device/README.md +++ b/samples/cpp/hello_query_device/README.md @@ -1,94 +1,110 @@ # Hello Query Device C++ Sample {#openvino_inference_engine_samples_hello_query_device_README} -This sample demonstrates how to execute an query OpenVINO™ Runtime devices, prints their metrics and default configuration values, using [Properties API](../../../docs/OV_Runtime_UG/supported_plugins/config_properties.md). +@sphinxdirective + +This sample demonstrates how to execute an query OpenVINO™ Runtime devices, prints their metrics and default configuration values, using :doc:`Properties API `. The following C++ API is used in the application: -| Feature | API | Description | -| :--- | :--- | :--- | -| Available Devices | `ov::Core::get_available_devices`, `ov::Core::get_property` | Get available devices information and configuration for inference | ++----------------------------------------+---------------------------------------+-------------------------------------------------------------------+ +| Feature | API | Description | ++========================================+=======================================+===================================================================+ +| Available Devices | ``ov::Core::get_available_devices``, | Get available devices information and configuration for inference | +| | ``ov::Core::get_property`` | | ++----------------------------------------+---------------------------------------+-------------------------------------------------------------------+ -Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification C++ sample `. -| Options | Values | -| :--- |:--- -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../python/hello_query_device/README.md) | ++----------------------------------------+----------------------------------------------------------------------------------------------+ +| Options | Values | ++========================================+==============================================================================================+ +| Supported devices | :doc:`All ` | ++----------------------------------------+----------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`Python ` | ++----------------------------------------+----------------------------------------------------------------------------------------------+ -## How It Works +How It Works +############ The sample queries all available OpenVINO™ Runtime devices, prints their supported metrics and plugin configuration parameters. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +####### To see quired information, run the following: -``` -hello_query_device -``` +.. code-block:: console + + hello_query_device -## Sample Output +Sample Output +############# The application prints all available devices with their supported metrics and default values for configuration parameters: -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Available devices: -[ INFO ] CPU -[ INFO ] SUPPORTED_METRICS: -[ INFO ] AVAILABLE_DEVICES : [ ] -[ INFO ] FULL_DEVICE_NAME : Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz -[ INFO ] OPTIMIZATION_CAPABILITIES : [ FP32 FP16 INT8 BIN ] -[ INFO ] RANGE_FOR_ASYNC_INFER_REQUESTS : { 1, 1, 1 } -[ INFO ] RANGE_FOR_STREAMS : { 1, 8 } -[ INFO ] IMPORT_EXPORT_SUPPORT : true -[ INFO ] SUPPORTED_CONFIG_KEYS (default values): -[ INFO ] CACHE_DIR : "" -[ INFO ] CPU_BIND_THREAD : NO -[ INFO ] CPU_THREADS_NUM : 0 -[ INFO ] CPU_THROUGHPUT_STREAMS : 1 -[ INFO ] DUMP_EXEC_GRAPH_AS_DOT : "" -[ INFO ] DYN_BATCH_ENABLED : NO -[ INFO ] DYN_BATCH_LIMIT : 0 -[ INFO ] ENFORCE_BF16 : NO -[ INFO ] EXCLUSIVE_ASYNC_REQUESTS : NO -[ INFO ] PERFORMANCE_HINT : "" -[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS : 0 -[ INFO ] PERF_COUNT : NO -[ INFO ] -[ INFO ] GNA -[ INFO ] SUPPORTED_METRICS: -[ INFO ] AVAILABLE_DEVICES : [ GNA_SW_EXACT ] -[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS : 1 -[ INFO ] FULL_DEVICE_NAME : GNA_SW_EXACT -[ INFO ] GNA_LIBRARY_FULL_VERSION : 3.0.0.1455 -[ INFO ] IMPORT_EXPORT_SUPPORT : true -[ INFO ] SUPPORTED_CONFIG_KEYS (default values): -[ INFO ] EXCLUSIVE_ASYNC_REQUESTS : NO -[ INFO ] GNA_COMPACT_MODE : YES -[ INFO ] GNA_COMPILE_TARGET : "" -[ INFO ] GNA_DEVICE_MODE : GNA_SW_EXACT -[ INFO ] GNA_EXEC_TARGET : "" -[ INFO ] GNA_FIRMWARE_MODEL_IMAGE : "" -[ INFO ] GNA_FIRMWARE_MODEL_IMAGE_GENERATION : "" -[ INFO ] GNA_LIB_N_THREADS : 1 -[ INFO ] GNA_PRECISION : I16 -[ INFO ] GNA_PWL_MAX_ERROR_PERCENT : 1.000000 -[ INFO ] GNA_PWL_UNIFORM_DESIGN : NO -[ INFO ] GNA_SCALE_FACTOR : 1.000000 -[ INFO ] GNA_SCALE_FACTOR_0 : 1.000000 -[ INFO ] LOG_LEVEL : LOG_NONE -[ INFO ] PERF_COUNT : NO -[ INFO ] SINGLE_THREAD : YES -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) +.. code-block:: console + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Available devices: + [ INFO ] CPU + [ INFO ] SUPPORTED_METRICS: + [ INFO ] AVAILABLE_DEVICES : [ ] + [ INFO ] FULL_DEVICE_NAME : Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz + [ INFO ] OPTIMIZATION_CAPABILITIES : [ FP32 FP16 INT8 BIN ] + [ INFO ] RANGE_FOR_ASYNC_INFER_REQUESTS : { 1, 1, 1 } + [ INFO ] RANGE_FOR_STREAMS : { 1, 8 } + [ INFO ] IMPORT_EXPORT_SUPPORT : true + [ INFO ] SUPPORTED_CONFIG_KEYS (default values): + [ INFO ] CACHE_DIR : "" + [ INFO ] CPU_BIND_THREAD : NO + [ INFO ] CPU_THREADS_NUM : 0 + [ INFO ] CPU_THROUGHPUT_STREAMS : 1 + [ INFO ] DUMP_EXEC_GRAPH_AS_DOT : "" + [ INFO ] DYN_BATCH_ENABLED : NO + [ INFO ] DYN_BATCH_LIMIT : 0 + [ INFO ] ENFORCE_BF16 : NO + [ INFO ] EXCLUSIVE_ASYNC_REQUESTS : NO + [ INFO ] PERFORMANCE_HINT : "" + [ INFO ] PERFORMANCE_HINT_NUM_REQUESTS : 0 + [ INFO ] PERF_COUNT : NO + [ INFO ] + [ INFO ] GNA + [ INFO ] SUPPORTED_METRICS: + [ INFO ] AVAILABLE_DEVICES : [ GNA_SW_EXACT ] + [ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS : 1 + [ INFO ] FULL_DEVICE_NAME : GNA_SW_EXACT + [ INFO ] GNA_LIBRARY_FULL_VERSION : 3.0.0.1455 + [ INFO ] IMPORT_EXPORT_SUPPORT : true + [ INFO ] SUPPORTED_CONFIG_KEYS (default values): + [ INFO ] EXCLUSIVE_ASYNC_REQUESTS : NO + [ INFO ] GNA_COMPACT_MODE : YES + [ INFO ] GNA_COMPILE_TARGET : "" + [ INFO ] GNA_DEVICE_MODE : GNA_SW_EXACT + [ INFO ] GNA_EXEC_TARGET : "" + [ INFO ] GNA_FIRMWARE_MODEL_IMAGE : "" + [ INFO ] GNA_FIRMWARE_MODEL_IMAGE_GENERATION : "" + [ INFO ] GNA_LIB_N_THREADS : 1 + [ INFO ] GNA_PRECISION : I16 + [ INFO ] GNA_PWL_MAX_ERROR_PERCENT : 1.000000 + [ INFO ] GNA_PWL_UNIFORM_DESIGN : NO + [ INFO ] GNA_SCALE_FACTOR : 1.000000 + [ INFO ] GNA_SCALE_FACTOR_0 : 1.000000 + [ INFO ] LOG_LEVEL : LOG_NONE + [ INFO ] PERF_COUNT : NO + [ INFO ] SINGLE_THREAD : YES + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` + +@endsphinxdirective + diff --git a/samples/cpp/hello_reshape_ssd/README.md b/samples/cpp/hello_reshape_ssd/README.md index d683f542fa0335..36c5e526978d82 100644 --- a/samples/cpp/hello_reshape_ssd/README.md +++ b/samples/cpp/hello_reshape_ssd/README.md @@ -1,117 +1,143 @@ # Hello Reshape SSD C++ Sample {#openvino_inference_engine_samples_hello_reshape_ssd_README} -This sample demonstrates how to do synchronous inference of object detection models using [input reshape feature](../../../docs/OV_Runtime_UG/ShapeInference.md). +@sphinxdirective + +This sample demonstrates how to do synchronous inference of object detection models using :doc:`input reshape feature `. Models with only one input and output are supported. The following C++ API is used in the application: -| Feature | API | Description | -| :--- | :--- | :--- | -| Node operations | `ov::Node::get_type_info`, `ngraph::op::DetectionOutput::get_type_info_static`, `ov::Output::get_any_name`, `ov::Output::get_shape` | Get a node info | -| Model Operations | `ov::Model::get_ops`, `ov::Model::reshape` | Get model nodes, reshape input | -| Tensor Operations | `ov::Tensor::data` | Get a tensor data | -| Preprocessing | `ov::preprocess::PreProcessSteps::convert_element_type`, `ov::preprocess::PreProcessSteps::convert_layout` | Model input preprocessing | - -Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). - -| Options | Values | -| :--- | :--- | -| Validated Models | [person-detection-retail-0013](@ref omz_models_model_person_detection_retail_0013) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../samples/python/hello_reshape_ssd/README.md) | - -## How It Works ++----------------------------------+-------------------------------------------------------------+------------------------------------------------+ +| Feature | API | Description | ++==================================+=============================================================+================================================+ +| Node operations | ``ov::Node::get_type_info``, | Get a node info | +| | ``ngraph::op::DetectionOutput::get_type_info_static``, | | +| | ``ov::Output::get_any_name``, | | +| | ``ov::Output::get_shape`` | | ++----------------------------------+-------------------------------------------------------------+------------------------------------------------+ +| Model Operations | ``ov::Model::get_ops``, | Get model nodes, reshape input | +| | ``ov::Model::reshape`` | | ++----------------------------------+-------------------------------------------------------------+------------------------------------------------+ +| Tensor Operations | ``ov::Tensor::data`` | Get a tensor data | ++----------------------------------+-------------------------------------------------------------+------------------------------------------------+ +| Preprocessing | ``ov::preprocess::PreProcessSteps::convert_element_type``, | Model input preprocessing | +| | ``ov::preprocess::PreProcessSteps::convert_layout`` | | ++----------------------------------+-------------------------------------------------------------+------------------------------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification C++ sample `. + ++----------------------------------+---------------------------------------------------------------------------------------------+ +| Options | Values | ++==================================+=============================================================================================+ +| Validated Models | :doc:`person-detection-retail-0013 ` | ++----------------------------------+---------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | ++----------------------------------+---------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++----------------------------------+---------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`Python ` | ++----------------------------------+---------------------------------------------------------------------------------------------+ + +How It Works +############ Upon the start-up the sample application reads command line parameters, loads specified network and image to the Inference Engine plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application creates output image and output data to the standard output stream. -You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +####### -``` -hello_reshape_ssd -``` +.. code-block:: console + + hello_reshape_ssd To run the sample, you need to specify a model and image: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- You can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (\*.onnx) that do not require preprocessing. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `__. -### Example +.. note:: + + - By default, OpenVINO™ Toolkit Samples and Demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (\*.onnx) that do not require preprocessing. + +Example ++++++++ 1. Install openvino-dev python package if you don't have it to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev - ``` + + .. code-block:: console + + python -m pip install openvino-dev 2. Download a pre-trained model using: - ``` - omz_downloader --name person-detection-retail-0013 - ``` - -3. `person-detection-retail-0013` does not need to be converted, because it is already in necessary format, so you can skip this step. If you want to use another model that is not in the IR or ONNX format, you can convert it using the model converter script: - ``` - omz_converter --name - ``` - -4. Perform inference of `person_detection.bmp` using `person-detection-retail-0013` model on a `GPU`, for example: - ``` - hello_reshape_ssd person-detection-retail-0013.xml person_detection.bmp GPU - ``` - -## Sample Output - -The application renders an image with detected objects enclosed in rectangles. It outputs the list of classes -of the detected objects along with the respective confidence values and the coordinates of the -rectangles to the standard output stream. - -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Loading model files: \models\person-detection-retail-0013.xml -[ INFO ] model name: ResMobNet_v4 (LReLU) with single SSD head -[ INFO ] inputs -[ INFO ] input name: data -[ INFO ] input type: f32 -[ INFO ] input shape: {1, 3, 320, 544} -[ INFO ] outputs -[ INFO ] output name: detection_out -[ INFO ] output type: f32 -[ INFO ] output shape: {1, 1, 200, 7} -Reshape network to the image size = [960x1699] -[ INFO ] model name: ResMobNet_v4 (LReLU) with single SSD head -[ INFO ] inputs -[ INFO ] input name: data -[ INFO ] input type: f32 -[ INFO ] input shape: {1, 3, 960, 1699} -[ INFO ] outputs -[ INFO ] output name: detection_out -[ INFO ] output type: f32 -[ INFO ] output shape: {1, 1, 200, 7} -[0,1] element, prob = 0.716309, (852,187)-(983,520) -The resulting image was saved in the file: hello_reshape_ssd_output.bmp - -This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) + + .. code-block:: console + + omz_downloader --name person-detection-retail-0013 + +3. ``person-detection-retail-0013`` does not need to be converted, because it is already in necessary format, so you can skip this step. If you want to use another model that is not in the IR or ONNX format, you can convert it using the model converter script: + + .. code-block:: console + + omz_converter --name + +4. Perform inference of ``person_detection.bmp`` using ``person-detection-retail-0013`` model on a ``GPU``, for example: + + .. code-block:: console + + hello_reshape_ssd person-detection-retail-0013.xml person_detection.bmp GPU + +Sample Output +############# + +The application renders an image with detected objects enclosed in rectangles. It outputs the list of classes of the detected objects along with the respective confidence values and the coordinates of the rectangles to the standard output stream. + +.. code-block:: console + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Loading model files: \models\person-detection-retail-0013.xml + [ INFO ] model name: ResMobNet_v4 (LReLU) with single SSD head + [ INFO ] inputs + [ INFO ] input name: data + [ INFO ] input type: f32 + [ INFO ] input shape: {1, 3, 320, 544} + [ INFO ] outputs + [ INFO ] output name: detection_out + [ INFO ] output type: f32 + [ INFO ] output shape: {1, 1, 200, 7} + Reshape network to the image size = [960x1699] + [ INFO ] model name: ResMobNet_v4 (LReLU) with single SSD head + [ INFO ] inputs + [ INFO ] input name: data + [ INFO ] input type: f32 + [ INFO ] input shape: {1, 3, 960, 1699} + [ INFO ] outputs + [ INFO ] output name: detection_out + [ INFO ] output type: f32 + [ INFO ] output shape: {1, 1, 200, 7} + [0,1] element, prob = 0.716309, (852,187)-(983,520) + The resulting image was saved in the file: hello_reshape_ssd_output.bmp + + This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/cpp/model_creation_sample/README.md b/samples/cpp/model_creation_sample/README.md index 542d6d82ec0077..f270db56758fe2 100644 --- a/samples/cpp/model_creation_sample/README.md +++ b/samples/cpp/model_creation_sample/README.md @@ -1,183 +1,220 @@ # Model Creation C++ Sample {#openvino_inference_engine_samples_model_creation_sample_README} -This sample demonstrates how to execute an synchronous inference using [model](../../../docs/OV_Runtime_UG/model_representation.md) built on the fly which uses weights from LeNet classification model, which is known to work well on digit classification tasks. +@sphinxdirective + +This sample demonstrates how to execute an synchronous inference using :doc:`model ` built on the fly which uses weights from LeNet classification model, which is known to work well on digit classification tasks. You do not need an XML file to create a model. The API of ov::Model allows creating a model on the fly from the source code. The following C++ API is used in the application: -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Info | `ov::Core::get_versions` | Get device plugins versions | -| Shape Operations | `ov::Output::get_shape`, `ov::Shape::size`, `ov::shape_size`| Operate with shape | -| Tensor Operations | `ov::Tensor::get_byte_size`, `ov::Tensor:data` | Get tensor byte size and its data | -| Model Operations | `ov::set_batch` | Operate with model batch size | -| Infer Request Operations | `ov::InferRequest::get_input_tensor` | Get a input tensor | -| Model creation objects | `ov::opset8::Parameter`, `ov::Node::output`, `ov::opset8::Constant`, `ov::opset8::Convolution`, `ov::opset8::Add`, `ov::opset1::MaxPool`, `ov::opset8::Reshape`, `ov::opset8::MatMul`, `ov::opset8::Relu`, `ov::opset8::Softmax`, `ov::descriptor::Tensor::set_names`, `ov::opset8::Result`, `ov::Model`, `ov::ParameterVector::vector` | Used to construct an OpenVINO model | - -Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). - -| Options | Values | -| :--- | :--- | -| Validated Models | LeNet | -| Model Format | model weights file (\*.bin) | -| Validated images | single-channel `MNIST ubyte` images | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../samples/python/model_creation_sample/README.md) | - -## How It Works ++------------------------------------------+-----------------------------------------+---------------------------------------+ +| Feature | API | Description | ++==========================================+=========================================+=======================================+ +| OpenVINO Runtime Info | ``ov::Core::get_versions`` | Get device plugins versions | ++------------------------------------------+-----------------------------------------+---------------------------------------+ +| Shape Operations | ``ov::Output::get_shape``, | Operate with shape | +| | ``ov::Shape::size``, | | +| | ``ov::shape_size`` | | ++------------------------------------------+-----------------------------------------+---------------------------------------+ +| Tensor Operations | ``ov::Tensor::get_byte_size``, | Get tensor byte size and its data | +| | ``ov::Tensor:data`` | | ++------------------------------------------+-----------------------------------------+---------------------------------------+ +| Model Operations | ``ov::set_batch`` | Operate with model batch size | ++------------------------------------------+-----------------------------------------+---------------------------------------+ +| Infer Request Operations | ``ov::InferRequest::get_input_tensor`` | Get a input tensor | ++------------------------------------------+-----------------------------------------+---------------------------------------+ +| Model creation objects | ``ov::opset8::Parameter``, | Used to construct an OpenVINO model | +| | ``ov::Node::output``, | | +| | ``ov::opset8::Constant``, | | +| | ``ov::opset8::Convolution``, | | +| | ``ov::opset8::Add``, | | +| | ``ov::opset1::MaxPool``, | | +| | ``ov::opset8::Reshape``, | | +| | ``ov::opset8::MatMul``, | | +| | ``ov::opset8::Relu``, | | +| | ``ov::opset8::Softmax``, | | +| | ``ov::descriptor::Tensor::set_names``, | | +| | ``ov::opset8::Result``, | | +| | ``ov::Model``, | | +| | ``ov::ParameterVector::vector`` | | ++------------------------------------------+-----------------------------------------+---------------------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification C++ sample `. + ++---------------------------------------------------------+-------------------------------------------------------------------------------------------------+ +| Options | Values | ++=========================================================+=================================================================================================+ +| Validated Models | LeNet | ++---------------------------------------------------------+-------------------------------------------------------------------------------------------------+ +| Model Format | model weights file (\*.bin) | ++---------------------------------------------------------+-------------------------------------------------------------------------------------------------+ +| Validated images | single-channel ``MNIST ubyte`` images | ++---------------------------------------------------------+-------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++---------------------------------------------------------+-------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`Python ` | ++---------------------------------------------------------+-------------------------------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application does the following: + - Reads command line parameters -- [Build a Model](../../../docs/OV_Runtime_UG/model_representation.md) and passed weights file +- :doc:`Build a Model ` and passed weights file - Loads the model and input data to the OpenVINO™ Runtime plugin - Performs synchronous inference and processes output data, logging each step in a standard output stream -You can see the explicit description of each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. + +Building +######## -## Building +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +Running +####### -## Running +.. code-block:: console -``` -model_creation_sample -``` + model_creation_sample -> **NOTES**: -> -> - you can use LeNet model weights in the sample folder: `lenet.bin` with FP32 weights file -> - The `lenet.bin` with FP32 weights file was generated by the [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) tool from the public LeNet model with the `--input_shape [64,1,28,28]` parameter specified. -> -> The original model is available in the [Caffe* repository](https://github.com/BVLC/caffe/tree/master/examples/mnist) on GitHub\*. +.. note:: + + - you can use LeNet model weights in the sample folder: ``lenet.bin`` with FP32 weights file + - The ``lenet.bin`` with FP32 weights file was generated by the :doc:`Model Optimizer ` tool from the public LeNet model with the ``--input_shape [64,1,28,28]`` parameter specified. + + The original model is available in the `Caffe* repository `__ on GitHub\*. You can do inference of an image using a pre-trained model on a GPU using the following command: -``` -model_creation_sample lenet.bin GPU -``` +.. code-block:: console + + model_creation_sample lenet.bin GPU -## Sample Output +Sample Output +############# The sample application logs each step in a standard output stream and outputs top-10 inference results. -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Device info: -[ INFO ] GPU -[ INFO ] Intel GPU plugin version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] -[ INFO ] Create model from weights: lenet.bin -[ INFO ] model name: lenet -[ INFO ] inputs -[ INFO ] input name: NONE -[ INFO ] input type: f32 -[ INFO ] input shape: {64, 1, 28, 28} -[ INFO ] outputs -[ INFO ] output name: output_tensor -[ INFO ] output type: f32 -[ INFO ] output shape: {64, 10} -[ INFO ] Batch size is 10 -[ INFO ] model name: lenet -[ INFO ] inputs -[ INFO ] input name: NONE -[ INFO ] input type: u8 -[ INFO ] input shape: {10, 28, 28, 1} -[ INFO ] outputs -[ INFO ] output name: output_tensor -[ INFO ] output type: f32 -[ INFO ] output shape: {10, 10} -[ INFO ] Compiling a model for the GPU device -[ INFO ] Create infer request -[ INFO ] Combine images in batch and set to input tensor -[ INFO ] Start sync inference -[ INFO ] Processing output tensor - -Top 1 results: - -Image 0 - -classid probability label -------- ----------- ----- -0 1.0000000 0 - -Image 1 - -classid probability label -------- ----------- ----- -1 1.0000000 1 - -Image 2 - -classid probability label -------- ----------- ----- -2 1.0000000 2 - -Image 3 - -classid probability label -------- ----------- ----- -3 1.0000000 3 - -Image 4 - -classid probability label -------- ----------- ----- -4 1.0000000 4 - -Image 5 - -classid probability label -------- ----------- ----- -5 1.0000000 5 - -Image 6 - -classid probability label -------- ----------- ----- -6 1.0000000 6 - -Image 7 - -classid probability label -------- ----------- ----- -7 1.0000000 7 - -Image 8 - -classid probability label -------- ----------- ----- -8 1.0000000 8 - -Image 9 - -classid probability label -------- ----------- ----- -9 1.0000000 9 - -``` - -## Deprecation Notice - - - - - - - - - - -
Deprecation BeginsJune 1, 2020
Removal DateDecember 1, 2020
- -## See Also +.. code-block:: console + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Device info: + [ INFO ] GPU + [ INFO ] Intel GPU plugin version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] + [ INFO ] Create model from weights: lenet.bin + [ INFO ] model name: lenet + [ INFO ] inputs + [ INFO ] input name: NONE + [ INFO ] input type: f32 + [ INFO ] input shape: {64, 1, 28, 28} + [ INFO ] outputs + [ INFO ] output name: output_tensor + [ INFO ] output type: f32 + [ INFO ] output shape: {64, 10} + [ INFO ] Batch size is 10 + [ INFO ] model name: lenet + [ INFO ] inputs + [ INFO ] input name: NONE + [ INFO ] input type: u8 + [ INFO ] input shape: {10, 28, 28, 1} + [ INFO ] outputs + [ INFO ] output name: output_tensor + [ INFO ] output type: f32 + [ INFO ] output shape: {10, 10} + [ INFO ] Compiling a model for the GPU device + [ INFO ] Create infer request + [ INFO ] Combine images in batch and set to input tensor + [ INFO ] Start sync inference + [ INFO ] Processing output tensor + + Top 1 results: + + Image 0 + + classid probability label + ------- ----------- ----- + 0 1.0000000 0 + + Image 1 + + classid probability label + ------- ----------- ----- + 1 1.0000000 1 + + Image 2 + + classid probability label + ------- ----------- ----- + 2 1.0000000 2 + + Image 3 + + classid probability label + ------- ----------- ----- + 3 1.0000000 3 + + Image 4 + + classid probability label + ------- ----------- ----- + 4 1.0000000 4 + + Image 5 + + classid probability label + ------- ----------- ----- + 5 1.0000000 5 + + Image 6 + + classid probability label + ------- ----------- ----- + 6 1.0000000 6 + + Image 7 + + classid probability label + ------- ----------- ----- + 7 1.0000000 7 + + Image 8 + + classid probability label + ------- ----------- ----- + 8 1.0000000 8 + + Image 9 + + classid probability label + ------- ----------- ----- + 9 1.0000000 9 + + + +Deprecation Notice +################## + ++--------------------+------------------+ +| Deprecation Begins | June 1, 2020 | ++====================+==================+ +| Removal Date | December 1, 2020 | ++--------------------+------------------+ + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Optimizer ` + +@endsphinxdirective -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/speech_sample/README.md b/samples/cpp/speech_sample/README.md index dfd12b70c15b33..23e8e421a62e19 100644 --- a/samples/cpp/speech_sample/README.md +++ b/samples/cpp/speech_sample/README.md @@ -1,134 +1,164 @@ # Automatic Speech Recognition C++ Sample {#openvino_inference_engine_samples_speech_sample_README} +@sphinxdirective + This sample demonstrates how to execute an Asynchronous Inference of acoustic model based on Kaldi\* neural networks and speech feature vectors. The sample works with Kaldi ARK or Numpy* uncompressed NPZ files, so it does not cover an end-to-end speech recognition scenario (speech to text), requiring additional preprocessing (feature extraction) to get a feature vector from a speech signal, as well as postprocessing (decoding) to produce text from scores. The following C++ API is used in the application: -| Feature | API | Description | -| :--- | :--- | :--- | -| Available Devices | `ov::Core::get_available_devices`, `ov::Core::get_property` | Get information of the devices for inference | -| Import/Export Model | `ov::Core::import_model`, `ov::CompiledModel::export_model` | The GNA plugin supports loading and saving of the GNA-optimized model | -| Model Operations | `ov::set_batch`, `ov::Model::add_output`, `ov::CompiledModel::inputs`, `ov::CompiledModel::outputs` | Managing of model: configure batch_size, input and output tensors | -| Node Operations | `ov::OutputVector::size`, `ov::Output::get_shape` | Get node shape | -| Asynchronous Infer | `ov::InferRequest::start_async`, `ov::InferRequest::wait` | Do asynchronous inference and waits until inference result becomes available | -| InferRequest Operations | `ov::InferRequest::query_state`, `ov::VariableState::reset` | Gets and resets CompiledModel state control | -| Tensor Operations | `ov::Tensor::get_size`, `ov::Tensor::data`, `ov::InferRequest::get_tensor` | Get a tensor, its size and data | -| Profiling | `ov::InferRequest::get_profiling_info` | Get infer request profiling info | - -Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). - -| Options | Values | -| :--- | :--- | -| Validated Models | Acoustic model based on Kaldi\* neural networks (see [Model Preparation](#model-preparation-speech) section) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin) | -| Supported devices | See [Execution Modes](#execution-modes-speech) section below and [List Supported Devices](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | - -## How It Works ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Feature | API | Description | ++=============================================================+=============================================================================================================+==============================================================================+ +| Available Devices | ``ov::Core::get_available_devices``, ``ov::Core::get_property`` | Get information of the devices for inference | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Import/Export Model | ``ov::Core::import_model``, ``ov::CompiledModel::export_model`` | The GNA plugin supports loading and saving of the GNA-optimized model | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Model Operations | ``ov::set_batch``, ``ov::Model::add_output``, ``ov::CompiledModel::inputs``, ``ov::CompiledModel::outputs`` | Managing of model: configure batch_size, input and output tensors | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Node Operations | ``ov::OutputVector::size``, ``ov::Output::get_shape`` | Get node shape | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Asynchronous Infer | ``ov::InferRequest::start_async``, ``ov::InferRequest::wait`` | Do asynchronous inference and waits until inference result becomes available | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| InferRequest Operations | ``ov::InferRequest::query_state``, ``ov::VariableState::reset`` | Gets and resets CompiledModel state control | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Tensor Operations | ``ov::Tensor::get_size``, ``ov::Tensor::data``, ``ov::InferRequest::get_tensor`` | Get a tensor, its size and data | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ +| Profiling | ``ov::InferRequest::get_profiling_info`` | Get infer request profiling info | ++-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------+ + + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification C++ sample `. + ++-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Options | Values | ++=============================================================+===============================================================================================================================================================+ +| Validated Models | Acoustic model based on Kaldi\* neural networks (see :ref:`Model Preparation ` section) | ++-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin) | ++-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Supported devices | See :ref:`Execution Modes ` section below and :doc:`List Supported Devices ` | ++-------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application reads command-line parameters, loads a specified model and input data to the OpenVINO™ Runtime plugin, performs inference on all speech utterances stored in the input file(s), logging each step in a standard output stream. -If the `-r` option is given, error statistics are provided for each speech utterance as shown above. +If the ``-r`` option is given, error statistics are provided for each speech utterance as shown above. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -### GNA-specific details +GNA-specific details +++++++++++++++++++++ -#### Quantization +Quantization +------------ -If the GNA device is selected (for example, using the `-d` GNA flag), the GNA OpenVINO™ Runtime plugin quantizes the model and input feature vector sequence to integer representation before performing inference. -Several parameters control neural network quantization. The `-q` flag determines the quantization mode. +If the GNA device is selected (for example, using the ``-d`` GNA flag), the GNA OpenVINO™ Runtime plugin quantizes the model and input feature vector sequence to integer representation before performing inference. +Several parameters control neural network quantization. The ``-q`` flag determines the quantization mode. Two modes are supported: - *static* - The first utterance in the input file is scanned for dynamic range. The scale factor (floating point scalar multiplier) required to scale the maximum input value of the first utterance to 16384 (15 bits) is used for all subsequent inputs. The neural network is quantized to accommodate the scaled input dynamic range. -- *user-defined* - The user may specify a scale factor via the `-sf` flag that will be used for static quantization. +- *user-defined* - The user may specify a scale factor via the ``-sf`` flag that will be used for static quantization. -The `-qb` flag provides a hint to the GNA plugin regarding the preferred target weight resolution for all layers. For example, when `-qb 8` is specified, the plugin will use 8-bit weights wherever possible in the +The ``-qb`` flag provides a hint to the GNA plugin regarding the preferred target weight resolution for all layers. For example, when ``-qb 8`` is specified, the plugin will use 8-bit weights wherever possible in the network. -> **NOTE**: -> -> - It is not always possible to use 8-bit weights due to GNA hardware limitations. For example, convolutional layers always use 16-bit weights (GNA hardware version 1 and 2). This limitation will be removed in GNA hardware version 3 and higher. +.. note:: + + It is not always possible to use 8-bit weights due to GNA hardware limitations. For example, convolutional layers always use 16-bit weights (GNA hardware version 1 and 2). This limitation will be removed in GNA hardware version 3 and higher. -#### Execution Modes -Several execution modes are supported via the `-d` flag: +.. _execution-modes-speech: -- `CPU` - All calculations are performed on CPU device using CPU Plugin. -- `GPU` - All calculations are performed on GPU device using GPU Plugin. -- `VPUX` - All calculations are performed on VPUX device using VPUX Plugin. -- `GNA_AUTO` - GNA hardware is used if available and the driver is installed. Otherwise, the GNA device is emulated in fast-but-not-bit-exact mode. -- `GNA_HW` - GNA hardware is used if available and the driver is installed. Otherwise, an error will occur. -- `GNA_SW` - Deprecated. The GNA device is emulated in fast-but-not-bit-exact mode. -- `GNA_SW_FP32` - Substitutes parameters and calculations from low precision to floating point (FP32). -- `GNA_SW_EXACT` - GNA device is emulated in bit-exact mode. +Execution Modes +--------------- -#### Loading and Saving Models +Several execution modes are supported via the ``-d`` flag: -The GNA plugin supports loading and saving of the GNA-optimized model (non-IR) via the `-rg` and `-wg` flags. Thereby, it is possible to avoid the cost of full model quantization at run time. The GNA plugin also supports export of firmware-compatible embedded model images for the Intel® Speech Enabling Developer Kit and Amazon Alexa* Premium Far-Field Voice Development Kit via the `-we` flag (save only). +- ``CPU`` - All calculations are performed on CPU device using CPU Plugin. +- ``GPU`` - All calculations are performed on GPU device using GPU Plugin. +- ``VPUX`` - All calculations are performed on VPUX device using VPUX Plugin. +- ``GNA_AUTO`` - GNA hardware is used if available and the driver is installed. Otherwise, the GNA device is emulated in fast-but-not-bit-exact mode. +- ``GNA_HW`` - GNA hardware is used if available and the driver is installed. Otherwise, an error will occur. +- ``GNA_SW`` - Deprecated. The GNA device is emulated in fast-but-not-bit-exact mode. +- ``GNA_SW_FP32`` - Substitutes parameters and calculations from low precision to floating point (FP32). +- ``GNA_SW_EXACT`` - GNA device is emulated in bit-exact mode. + +Loading and Saving Models +------------------------- + +The GNA plugin supports loading and saving of the GNA-optimized model (non-IR) via the ``-rg`` and ``-wg`` flags. Thereby, it is possible to avoid the cost of full model quantization at run time. The GNA plugin also supports export of firmware-compatible embedded model images for the Intel® Speech Enabling Developer Kit and Amazon Alexa* Premium Far-Field Voice Development Kit via the ``-we`` flag (save only). In addition to performing inference directly from a GNA model file, these combinations of options make it possible to: -- Convert from IR format to GNA format model file (`-m`, `-wg`) -- Convert from IR format to embedded format model file (`-m`, `-we`) -- Convert from GNA format to embedded format model file (`-rg`, `-we`) +- Convert from IR format to GNA format model file (``-m``, ``-wg``) +- Convert from IR format to embedded format model file (``-m``, ``-we``) +- Convert from GNA format to embedded format model file (``-rg``, ``-we``) -## Building +Building +######## -To build the sample, please use instructions available at [Build the Sample Applications](../../../docs/OV_Runtime_UG/Samples_Overview.md) section in OpenVINO™ Toolkit Samples guide. +To build the sample, please use instructions available at :doc:`Build the Sample Applications ` section in OpenVINO™ Toolkit Samples guide. -## Running +Running +####### Run the application with the -h option to see the usage message: -``` -speech_sample -h -``` +.. code-block:: sh + + speech_sample -h Usage message: -``` -[ INFO ] OpenVINO Runtime version ......... -[ INFO ] Build ........... -[ INFO ] -[ INFO ] Parsing input parameters - -speech_sample [OPTION] -Options: - - -h Print a usage message. - -i "" Required. Path(s) to input file(s). Usage for a single file/layer: or . Example of usage for several files/layers: :=,:=. - -m "" Required. Path to an .xml file with a trained model (required if -rg is missing). - -o "" Optional. Output file name(s) to save scores (inference results). Example of usage for a single file/layer: or . Example of usage for several files/layers: :=,:=. - -d "" Optional. Specify a target device to infer on. CPU, GPU, VPUX, GNA_AUTO, GNA_HW, GNA_HW_WITH_SW_FBACK, GNA_SW_FP32, GNA_SW_EXACT and HETERO with combination of GNA as the primary device and CPU as a secondary (e.g. HETERO:GNA,CPU) are supported. The sample will look for a suitable plugin for device specified. - -pc Optional. Enables per-layer performance report. - -q "" Optional. Input quantization mode for GNA: static (default) or user defined (use with -sf). - -qb "" Optional. Weight resolution in bits for GNA quantization: 8 or 16 (default) - -sf "" Optional. User-specified input scale factor for GNA quantization (use with -q user). If the model contains multiple inputs, provide scale factors by separating them with commas. For example: :,: or just to be applied to all inputs. - -bs "" Optional. Batch size 1-8 (default 1) - -r "" Optional. Read reference score file(s) and compare inference results with reference scores. Usage for a single file/layer: or . Example of usage for several files/layers: :=,:=. - -rg "" Read GNA model from file using path/filename provided (required if -m is missing). - -wg "" Optional. Write GNA model to file using path/filename provided. - -we "" Optional. Write GNA embedded model to file using path/filename provided. - -cw_l "" Optional. Number of frames for left context windows (default is 0). Works only with context window networks. If you use the cw_l or cw_r flag, then batch size argument is ignored. - -cw_r "" Optional. Number of frames for right context windows (default is 0). Works only with context window networks. If you use the cw_r or cw_l flag, then batch size argument is ignored. - -layout "" Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. - -pwl_me "" Optional. The maximum percent of error for PWL function.The value must be in <0, 100> range. The default value is 1.0. - -exec_target "" Optional. Specify GNA execution target generation. May be one of GNA_TARGET_2_0, GNA_TARGET_3_0. By default, generation corresponds to the GNA HW available in the system or the latest fully supported generation by the software. See the GNA Plugin's GNA_EXEC_TARGET config option description. - -compile_target "" Optional. Specify GNA compile target generation. May be one of GNA_TARGET_2_0, GNA_TARGET_3_0. By default, generation corresponds to the GNA HW available in the system or the latest fully supported generation by the software. See the GNA Plugin's GNA_COMPILE_TARGET config option description. - -memory_reuse_off Optional. Disables memory optimizations for compiled model. - -Available target devices: CPU GNA GPU VPUX -``` - -### Model Preparation +.. code-block:: sh + + [ INFO ] OpenVINO Runtime version ......... + [ INFO ] Build ........... + [ INFO ] + [ INFO ] Parsing input parameters + + speech_sample [OPTION] + Options: + + -h Print a usage message. + -i "" Required. Path(s) to input file(s). Usage for a single file/layer: or . Example of usage for several files/layers: :=,:=. + -m "" Required. Path to an .xml file with a trained model (required if -rg is missing). + -o "" Optional. Output file name(s) to save scores (inference results). Example of usage for a single file/layer: or . Example of usage for several files/layers: :=,:=. + -d "" Optional. Specify a target device to infer on. CPU, GPU, VPUX, GNA_AUTO, GNA_HW, GNA_HW_WITH_SW_FBACK, GNA_SW_FP32, GNA_SW_EXACT and HETERO with combination of GNA as the primary device and CPU as a secondary (e.g. HETERO:GNA,CPU) are supported. The sample will look for a suitable plugin for device specified. + -pc Optional. Enables per-layer performance report. + -q "" Optional. Input quantization mode for GNA: static (default) or user defined (use with -sf). + -qb "" Optional. Weight resolution in bits for GNA quantization: 8 or 16 (default) + -sf "" Optional. User-specified input scale factor for GNA quantization (use with -q user). If the model contains multiple inputs, provide scale factors by separating them with commas. For example: :,: or just to be applied to all inputs. + -bs "" Optional. Batch size 1-8 (default 1) + -r "" Optional. Read reference score file(s) and compare inference results with reference scores. Usage for a single file/layer: or . Example of usage for several files/layers: :=,:=. + -rg "" Read GNA model from file using path/filename provided (required if -m is missing). + -wg "" Optional. Write GNA model to file using path/filename provided. + -we "" Optional. Write GNA embedded model to file using path/filename provided. + -cw_l "" Optional. Number of frames for left context windows (default is 0). Works only with context window networks. If you use the cw_l or cw_r flag, then batch size argument is ignored. + -cw_r "" Optional. Number of frames for right context windows (default is 0). Works only with context window networks. If you use the cw_r or cw_l flag, then batch size argument is ignored. + -layout "" Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. + -pwl_me "" Optional. The maximum percent of error for PWL function.The value must be in <0, 100> range. The default value is 1.0. + -exec_target "" Optional. Specify GNA execution target generation. May be one of GNA_TARGET_2_0, GNA_TARGET_3_0. By default, generation corresponds to the GNA HW available in the system or the latest fully supported generation by the software. See the GNA Plugin's GNA_EXEC_TARGET config option description. + -compile_target "" Optional. Specify GNA compile target generation. May be one of GNA_TARGET_2_0, GNA_TARGET_3_0. By default, generation corresponds to the GNA HW available in the system or the latest fully supported generation by the software. See the GNA Plugin's GNA_COMPILE_TARGET config option description. + -memory_reuse_off Optional. Disables memory optimizations for compiled model. + + Available target devices: CPU GNA GPU VPUX + + +.. _model-preparation-speech: + +Model Preparation ++++++++++++++++++ You can use the following model optimizer command to convert a Kaldi nnet1 or nnet2 neural model to OpenVINO™ toolkit Intermediate Representation format: -``` -mo --framework kaldi --input_model wsj_dnn5b.nnet --counts wsj_dnn5b.counts --remove_output_softmax --output_dir -``` +.. code-block:: sh + + mo --framework kaldi --input_model wsj_dnn5b.nnet --counts wsj_dnn5b.counts --remove_output_softmax --output_dir The following pre-trained models are available: @@ -136,103 +166,114 @@ The following pre-trained models are available: - rm_lstm4f - wsj_dnn5b_smbr -All of them can be downloaded from [https://storage.openvinotoolkit.org/models_contrib/speech/2021.2](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2). +All of them can be downloaded from `the storage `__. -### Speech Inference +Speech Inference +++++++++++++++++ Once the IR is created, you can do inference on Intel® Processors with the GNA co-processor (or emulation library): -``` -speech_sample -m wsj_dnn5b.xml -i dev93_10.ark -r dev93_scores_10.ark -d GNA_AUTO -o result.ark -``` +.. code-block:: sh + + speech_sample -m wsj_dnn5b.xml -i dev93_10.ark -r dev93_scores_10.ark -d GNA_AUTO -o result.ark -Here, the floating point Kaldi-generated reference neural network scores (`dev93_scores_10.ark`) corresponding to the input feature file (`dev93_10.ark`) are assumed to be available for comparison. +Here, the floating point Kaldi-generated reference neural network scores (``dev93_scores_10.ark``) corresponding to the input feature file (``dev93_10.ark``) are assumed to be available for comparison. -> **NOTES**: -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample supports input and output in numpy file format (.npz) +.. note:: -## Sample Output + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + + - The sample supports input and output in numpy file format (.npz) + +Sample Output +############# The sample application logs each step in a standard output stream. -``` -[ INFO ] OpenVINO runtime: OpenVINO Runtime version ......... 2022.1.0 -[ INFO ] Build ........... 2022.1.0-6311-a90bb1ff017 -[ INFO ] -[ INFO ] Parsing input parameters -[ INFO ] Loading model files: -[ INFO ] \test_data\models\wsj_dnn5b_smbr_fp32\wsj_dnn5b_smbr_fp32.xml -[ INFO ] Using scale factor of 2175.43 calculated from first utterance. -[ INFO ] Model loading time 0.0034 ms -[ INFO ] Loading model to the device GNA_AUTO -[ INFO ] Loading model to the device -[ INFO ] Number scores per frame : 3425 -Utterance 0: -Total time in Infer (HW and SW): 5687.53 ms -Frames in utterance: 1294 frames -Average Infer time per frame: 4.39531 ms - max error: 0.705184 - avg error: 0.0448388 - avg rms error: 0.0574098 - stdev error: 0.0371649 - - -End of Utterance 0 - -[ INFO ] Number scores per frame : 3425 -Utterance 1: -Total time in Infer (HW and SW): 4341.34 ms -Frames in utterance: 1005 frames -Average Infer time per frame: 4.31974 ms - max error: 0.757597 - avg error: 0.0452166 - avg rms error: 0.0578436 - stdev error: 0.0372769 - - -End of Utterance 1 - -... -End of Utterance X - -[ INFO ] Execution successful -``` - -## Use of Sample in Kaldi* Speech Recognition Pipeline - -The Wall Street Journal DNN model used in this example was prepared using the Kaldi s5 recipe and the Kaldi Nnet (nnet1) framework. It is possible to recognize speech by substituting the `speech_sample` for -Kaldi's nnet-forward command. Since the `speech_sample` does not yet use pipes, it is necessary to use temporary files for speaker-transformed feature vectors and scores when running the Kaldi speech recognition pipeline. The following operations assume that feature extraction was already performed according to the `s5` recipe and that the working directory within the Kaldi source tree is `egs/wsj/s5`. - -1. Prepare a speaker-transformed feature set given the feature transform specified in `final.feature_transform` and the feature files specified in `feats.scp`: - ```sh - nnet-forward --use-gpu=no final.feature_transform "ark,s,cs:copy-feats scp:feats.scp ark:- |" ark:feat.ark - ``` - -2. Score the feature set using the `speech_sample`: - ```sh - ./speech_sample -d GNA_AUTO -bs 8 -i feat.ark -m wsj_dnn5b.xml -o scores.ark - ``` - - OpenVINO™ toolkit Intermediate Representation `wsj_dnn5b.xml` file was generated in the previous [Model Preparation](#model-preparation-speech) section. - -3. Run the Kaldi decoder to produce n-best text hypotheses and select most likely text given the WFST (`HCLG.fst`), vocabulary (`words.txt`), and TID/PID mapping (`final.mdl`): - ```sh - latgen-faster-mapped --max-active=7000 --max-mem=50000000 --beam=13.0 --lattice-beam=6.0 --acoustic-scale=0.0833 --allow-partial=true --word-symbol-table=words.txt final.mdl HCLG.fst ark:scores.ark ark:-| lattice-scale --inv-acoustic-scale=13 ark:- ark:- | lattice-best-path --word-symbol-table=words.txt ark:- ark,t:- > out.txt & - ``` - -4. Run the word error rate tool to check accuracy given the vocabulary (`words.txt`) and reference transcript (`test_filt.txt`): - ```sh - cat out.txt | utils/int2sym.pl -f 2- words.txt | sed s:\::g | compute-wer --text --mode=present ark:test_filt.txt ark,p:- - ``` - - All of mentioned files can be downloaded from [https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/wsj_dnn5b_smbr](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/wsj_dnn5b_smbr) - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: sh + + [ INFO ] OpenVINO runtime: OpenVINO Runtime version ......... 2022.1.0 + [ INFO ] Build ........... 2022.1.0-6311-a90bb1ff017 + [ INFO ] + [ INFO ] Parsing input parameters + [ INFO ] Loading model files: + [ INFO ] \test_data\models\wsj_dnn5b_smbr_fp32\wsj_dnn5b_smbr_fp32.xml + [ INFO ] Using scale factor of 2175.43 calculated from first utterance. + [ INFO ] Model loading time 0.0034 ms + [ INFO ] Loading model to the device GNA_AUTO + [ INFO ] Loading model to the device + [ INFO ] Number scores per frame : 3425 + Utterance 0: + Total time in Infer (HW and SW): 5687.53 ms + Frames in utterance: 1294 frames + Average Infer time per frame: 4.39531 ms + max error: 0.705184 + avg error: 0.0448388 + avg rms error: 0.0574098 + stdev error: 0.0371649 + + + End of Utterance 0 + + [ INFO ] Number scores per frame : 3425 + Utterance 1: + Total time in Infer (HW and SW): 4341.34 ms + Frames in utterance: 1005 frames + Average Infer time per frame: 4.31974 ms + max error: 0.757597 + avg error: 0.0452166 + avg rms error: 0.0578436 + stdev error: 0.0372769 + + + End of Utterance 1 + + ... + End of Utterance X + + [ INFO ] Execution successful + +Use of Sample in Kaldi* Speech Recognition Pipeline +################################################### + +The Wall Street Journal DNN model used in this example was prepared using the Kaldi s5 recipe and the Kaldi Nnet (nnet1) framework. It is possible to recognize speech by substituting the ``speech_sample`` for +Kaldi's nnet-forward command. Since the ``speech_sample`` does not yet use pipes, it is necessary to use temporary files for speaker-transformed feature vectors and scores when running the Kaldi speech recognition pipeline. The following operations assume that feature extraction was already performed according to the ``s5`` recipe and that the working directory within the Kaldi source tree is ``egs/wsj/s5``. + +1. Prepare a speaker-transformed feature set given the feature transform specified in ``final.feature_transform`` and the feature files specified in ``feats.scp``: + + .. code-block:: sh + + nnet-forward --use-gpu=no final.feature_transform "ark,s,cs:copy-feats scp:feats.scp ark:- |" ark:feat.ark + +2. Score the feature set using the ``speech_sample``: + + .. code-block:: sh + + ./speech_sample -d GNA_AUTO -bs 8 -i feat.ark -m wsj_dnn5b.xml -o scores.ark + + OpenVINO™ toolkit Intermediate Representation ``wsj_dnn5b.xml`` file was generated in the previous :ref:`Model Preparation ` section. + +3. Run the Kaldi decoder to produce n-best text hypotheses and select most likely text given the WFST (``HCLG.fst``), vocabulary (``words.txt``), and TID/PID mapping (``final.mdl``): + + .. code-block:: sh + + latgen-faster-mapped --max-active=7000 --max-mem=50000000 --beam=13.0 --lattice-beam=6.0 --acoustic-scale=0.0833 --allow-partial=true --word-symbol-table=words.txt final.mdl HCLG.fst ark:scores.ark ark:-| lattice-scale --inv-acoustic-scale=13 ark:- ark:- | lattice-best-path --word-symbol-table=words.txt ark:- ark,t:- > out.txt & + +4. Run the word error rate tool to check accuracy given the vocabulary (``words.txt``) and reference transcript (``test_filt.txt``): + + .. code-block:: sh + + cat out.txt | utils/int2sym.pl -f 2- words.txt | sed s:\::g | compute-wer --text --mode=present ark:test_filt.txt ark,p:- + + All of mentioned files can be downloaded from `the storage `__ + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/cpp/speech_sample/main.cpp b/samples/cpp/speech_sample/main.cpp index 7553ee7bf221c7..3c17b8e34882c0 100644 --- a/samples/cpp/speech_sample/main.cpp +++ b/samples/cpp/speech_sample/main.cpp @@ -220,7 +220,7 @@ int main(int argc, char* argv[]) { gnaPluginConfig[ov::intel_gna::scale_factors_per_input.name()] = scale_factors_per_input; } } - gnaPluginConfig[ov::inference_precision.name()] = (FLAGS_qb == 8) ? ov::element::i8 : ov::element::i16; + gnaPluginConfig[ov::hint::inference_precision.name()] = (FLAGS_qb == 8) ? ov::element::i8 : ov::element::i16; const std::unordered_map StringHWGenerationMap{ {"GNA_TARGET_1_0", ov::intel_gna::HWGeneration::GNA_1_0}, {"GNA_TARGET_2_0", ov::intel_gna::HWGeneration::GNA_2_0}, diff --git a/samples/python/benchmark/bert_benchmark/README.md b/samples/python/benchmark/bert_benchmark/README.md index 84571ec446788d..5e8c6ca3365ad6 100644 --- a/samples/python/benchmark/bert_benchmark/README.md +++ b/samples/python/benchmark/bert_benchmark/README.md @@ -1,50 +1,69 @@ # Bert Benchmark Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_bert_benchmark_README} -This sample demonstrates how to estimate performace of a Bert model using Asynchronous Inference Request API. Unlike [demos](@ref omz_demos) this sample doesn't have configurable command line arguments. Feel free to modify sample's source code to try out different options. +@sphinxdirective -The following Python\* API is used in the application: +This sample demonstrates how to estimate performance of a Bert model using Asynchronous Inference Request API. Unlike :doc:`demos ` this sample doesn't have configurable command line arguments. Feel free to modify sample's source code to try out different options. -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version | -| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.compile_model] | Common API to do inference: compile a model | -| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], [openvino.runtime.AsyncInferQueue.start_async], [openvino.runtime.AsyncInferQueue.wait_all] | Do asynchronous inference | -| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model | +The following Python API is used in the application: -## How It Works ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Feature | API | Description | ++================================+=================================================+==============================================+ +| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Basic Infer Flow | [openvino.runtime.Core], | Common API to do inference: compile a model. | +| | [openvino.runtime.Core.compile_model] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], | Do asynchronous inference. | +| | [openvino.runtime.AsyncInferQueue.start_async], | | +| | [openvino.runtime.AsyncInferQueue.wait_all] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ -The sample downloads a model and a tokenizer, export the model to onnx, reads the exported model and reshapes it to enforce dynamic inpus shapes, compiles the resulting model, downloads a dataset and runs benchmarking on the dataset. +How It Works +#################### + +The sample downloads a model and a tokenizer, export the model to onnx, reads the exported model and reshapes it to enforce dynamic input shapes, compiles the resulting model, downloads a dataset and runs benchmarking on the dataset. You can see the explicit description of -each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. + +Running +#################### + +Install the ``openvino`` Python package: + +.. code-block:: sh -## Running + python -m pip install openvino -Install the `openvino` Python package: -``` -python -m pip install openvino -``` +Install packages from ``requirements.txt``: -Install packages from `requirements.txt`: +.. code-block:: sh + + python -m pip install -r requirements.txt -``` -python -m pip install -r requirements.txt -``` Run the sample -``` -python bert_benchmark.py -``` +.. code-block:: sh + + python bert_benchmark.py -## Sample Output + +Sample Output +#################### The sample outputs how long it takes to process a dataset. -## See Also +See Also +#################### + +* :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +* :doc:`Using OpenVINO Samples ` +* :doc:`Model Downloader ` +* :doc:`Model Optimizer ` -- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +@endsphinxdirective diff --git a/samples/python/benchmark/sync_benchmark/README.md b/samples/python/benchmark/sync_benchmark/README.md index 3d4bda76e8c08a..f8194fa0c427c7 100644 --- a/samples/python/benchmark/sync_benchmark/README.md +++ b/samples/python/benchmark/sync_benchmark/README.md @@ -1,92 +1,129 @@ # Sync Benchmark Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_sync_benchmark_README} -This sample demonstrates how to estimate performace of a model using Synchronous Inference Request API. It makes sence to use synchronous inference only in latency oriented scenarios. Models with static input shapes are supported. Unlike [demos](@ref omz_demos) this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. +@sphinxdirective + +This sample demonstrates how to estimate performance of a model using Synchronous Inference Request API. It makes sense to use synchronous inference only in latency oriented scenarios. Models with static input shapes are supported. Unlike :doc:`demos ` this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. + +The following Python API is used in the application: + + ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Feature | API | Description | ++================================+=================================================+==============================================+ +| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Basic Infer Flow | [openvino.runtime.Core], | Common API to do inference: compile a model, | +| | [openvino.runtime.Core.compile_model], | configure input tensors. | +| | [openvino.runtime.InferRequest.get_tensor] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Synchronous Infer | [openvino.runtime.InferRequest.infer], | Do synchronous inference. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Tensor Operations | [openvino.runtime.Tensor.get_shape], | Get a tensor shape and its data. | +| | [openvino.runtime.Tensor.data] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ + ++--------------------------------+------------------------------------------------------------------------------+ +| Options | Values | ++================================+==============================================================================+ +| Validated Models | :doc:`alexnet `, | +| | :doc:`googlenet-v1 `, | +| | :doc:`yolo-v3-tf `, | +| | :doc:`face-detection-0200 ` | ++--------------------------------+------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation | +| | (\*.xml + \*.bin), ONNX (\*.onnx) | ++--------------------------------+------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++--------------------------------+------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++--------------------------------+------------------------------------------------------------------------------+ + + +How It Works +#################### -The following Python\* API is used in the application: +The sample compiles a model for a given device, randomly generates input data, performs synchronous inference multiple times for a given number of seconds. Then processes and reports performance results. -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version | -| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.compile_mode], [openvino.runtime.InferRequest.get_tensor] | Common API to do inference: compile a model, configure input tensors | -| Synchronous Infer | [openvino.runtime.InferRequest.infer] | Do synchronous inference | -| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model | -| Tensor Operations | [openvino.runtime.Tensor.get_shape], [openvino.runtime.Tensor.data] | Get a tensor shape and its data. | +You can see the explicit description of +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -| Options | Values | -| :--- | :--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) [yolo-v3-tf](@ref omz_models_model_yolo_v3_tf), [face-detection-0200](@ref omz_models_model_face_detection_0200) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../cpp/benchmark/sync_benchmark/README.md) | +Running +#################### -## How It Works +.. code-block:: sh -The sample compiles a model for a given device, randomly generates input data, performs synchronous inference multiple times for a given number of seconds. Then processes and reports performance results. + python sync_benchmark.py -You can see the explicit description of -each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Running +To run the sample, you need to specify a model: -``` -python sync_benchmark.py -``` +- You can use :doc:`public ` or doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. -To run the sample, you need to specify a model: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). +.. note:: -> **NOTES**: -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. -### Example + The sample accepts models in ONNX format (.onnx) that do not require preprocessing. -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: -``` -python -m pip install openvino-dev[caffe] -``` +Example +++++++++++++++++++++ + +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: + + .. code-block:: sh + + python -m pip install openvino-dev[caffe] + 2. Download a pre-trained model using: -``` -omz_downloader --name googlenet-v1 -``` + .. code-block:: sh + + omz_downloader --name googlenet-v1 + 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: -``` -omz_converter --name googlenet-v1 -``` + .. code-block:: sh + + omz_converter --name googlenet-v1 + + +4. Perform benchmarking using the ``googlenet-v1`` model on a ``CPU``: + + .. code-block:: sh -4. Perform benchmarking using the `googlenet-v1` model on a `CPU`: + python sync_benchmark.py googlenet-v1.xml -``` -python sync_benchmark.py googlenet-v1.xml -``` -## Sample Output +Sample Output +#################### The application outputs performance results. -``` -[ INFO ] OpenVINO: -[ INFO ] Build ................................. -[ INFO ] Count: 2333 iterations -[ INFO ] Duration: 10003.59 ms -[ INFO ] Latency: -[ INFO ] Median: 3.90 ms -[ INFO ] Average: 4.29 ms -[ INFO ] Min: 3.30 ms -[ INFO ] Max: 10.11 ms -[ INFO ] Throughput: 233.22 FPS -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: sh + + [ INFO ] OpenVINO: + [ INFO ] Build ................................. + [ INFO ] Count: 2333 iterations + [ INFO ] Duration: 10003.59 ms + [ INFO ] Latency: + [ INFO ] Median: 3.90 ms + [ INFO ] Average: 4.29 ms + [ INFO ] Min: 3.30 ms + [ INFO ] Max: 10.11 ms + [ INFO ] Throughput: 233.22 FPS + + +See Also +#################### + +* :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +* :doc:`Using OpenVINO Samples ` +* :doc:`Model Downloader ` +* :doc:`Model Optimizer ` + +@endsphinxdirective \ No newline at end of file diff --git a/samples/python/benchmark/throughput_benchmark/README.md b/samples/python/benchmark/throughput_benchmark/README.md index 8e104248e497b0..9b5f18c7df7c53 100644 --- a/samples/python/benchmark/throughput_benchmark/README.md +++ b/samples/python/benchmark/throughput_benchmark/README.md @@ -1,94 +1,134 @@ # Throughput Benchmark Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_throughput_benchmark_README} -This sample demonstrates how to estimate performace of a model using Asynchronous Inference Request API in throughput mode. Unlike [demos](@ref omz_demos) this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. +@sphinxdirective + + +This sample demonstrates how to estimate performance of a model using Asynchronous Inference Request API in throughput mode. Unlike :doc:`demos ` this sample doesn't have other configurable command line arguments. Feel free to modify sample's source code to try out different options. + +The reported results may deviate from what :doc:`benchmark_app ` reports. One example is model input precision for computer vision tasks. benchmark_app sets uint8, while the sample uses default model precision which is usually float32. + +The following Python API is used in the application: + ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Feature | API | Description | ++================================+=================================================+==============================================+ +| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Basic Infer Flow | [openvino.runtime.Core], | Common API to do inference: compile a model, | +| | [openvino.runtime.Core.compile_model] | configure input tensors. | +| | [openvino.runtime.InferRequest.get_tensor] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], | Do asynchronous inference. | +| | [openvino.runtime.AsyncInferQueue.start_async], | | +| | [openvino.runtime.AsyncInferQueue.wait_all], | | +| | [openvino.runtime.InferRequest.results] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model. | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ +| Tensor Operations | [openvino.runtime.Tensor.get_shape], | Get a tensor shape and its data. | +| | [openvino.runtime.Tensor.data] | | ++--------------------------------+-------------------------------------------------+----------------------------------------------+ + ++--------------------------------+------------------------------------------------------------------------------+ +| Options | Values | ++================================+==============================================================================+ +| Validated Models | :doc:`alexnet `, | +| | :doc:`googlenet-v1 `, | +| | :doc:`yolo-v3-tf `, | +| | :doc:`face-detection-0200 ` | ++--------------------------------+------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation | +| | (\*.xml + \*.bin), ONNX (\*.onnx) | ++--------------------------------+------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++--------------------------------+------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++--------------------------------+------------------------------------------------------------------------------+ + + +How It Works +#################### -The reported results may deviate from what [benchmark_app](../../../../tools/benchmark_tool/README.md) reports. One example is model input precision for computer vision tasks. benchmark_app sets uint8, while the sample uses default model precision which is usually float32. +The sample compiles a model for a given device, randomly generates input data, performs asynchronous inference multiple times for a given number of seconds. Then processes and reports performance results. -The following Python\* API is used in the application: +You can see the explicit description of +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -| Feature | API | Description | -| :--- | :--- | :--- | -| OpenVINO Runtime Version | [openvino.runtime.get_version] | Get Openvino API version | -| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.compile_model], [openvino.runtime.InferRequest.get_tensor] | Common API to do inference: compile a model, configure input tensors | -| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], [openvino.runtime.AsyncInferQueue.start_async], [openvino.runtime.AsyncInferQueue.wait_all], [openvino.runtime.InferRequest.results] | Do asynchronous inference | -| Model Operations | [openvino.runtime.CompiledModel.inputs] | Get inputs of a model | -| Tensor Operations | [openvino.runtime.Tensor.get_shape], [openvino.runtime.Tensor.data] | Get a tensor shape and its data. | +Running +#################### -| Options | Values | -| :--- | :--- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) [yolo-v3-tf](@ref omz_models_model_yolo_v3_tf), [face-detection-0200](@ref omz_models_model_face_detection_0200) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (\*.xml + \*.bin), ONNX (\*.onnx) | -| Supported devices | [All](../../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../cpp/benchmark/throughput_benchmark/README.md) | +.. code-block:: sh -## How It Works + python throughput_benchmark.py -The sample compiles a model for a given device, randomly generates input data, performs asynchronous inference multiple times for a given number of seconds. Then processes and reports performance results. -You can see the explicit description of -each sample step at [Integration Steps](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +To run the sample, you need to specify a model: -## Running +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. -``` -python throughput_benchmark.py -``` +.. note:: + + Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + + The sample accepts models in ONNX format (.onnx) that do not require preprocessing. -To run the sample, you need to specify a model: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -> **NOTES**: -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +Example +++++++++++++++++++++ -### Example +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: + .. code-block:: sh + + python -m pip install openvino-dev[caffe] -``` -python -m pip install openvino-dev[caffe] -``` 2. Download a pre-trained model using: -``` -omz_downloader --name googlenet-v1 -``` + .. code-block:: sh + + omz_downloader --name googlenet-v1 + 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: -``` -omz_converter --name googlenet-v1 -``` + .. code-block:: sh + + omz_converter --name googlenet-v1 + + +4. Perform benchmarking using the ``googlenet-v1`` model on a ``CPU``: + + .. code-block:: sh -4. Perform benchmarking using the `googlenet-v1` model on a `CPU`: + python throughput_benchmark.py googlenet-v1.xml -``` -python throughput_benchmark.py googlenet-v1.xml -``` -## Sample Output +Sample Output +#################### The application outputs performance results. -``` -[ INFO ] OpenVINO: -[ INFO ] Build ................................. -[ INFO ] Count: 2817 iterations -[ INFO ] Duration: 10012.65 ms -[ INFO ] Latency: -[ INFO ] Median: 13.80 ms -[ INFO ] Average: 14.10 ms -[ INFO ] Min: 8.35 ms -[ INFO ] Max: 28.38 ms -[ INFO ] Throughput: 281.34 FPS -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +.. code-block:: sh + + [ INFO ] OpenVINO: + [ INFO ] Build ................................. + [ INFO ] Count: 2817 iterations + [ INFO ] Duration: 10012.65 ms + [ INFO ] Latency: + [ INFO ] Median: 13.80 ms + [ INFO ] Average: 14.10 ms + [ INFO ] Min: 8.35 ms + [ INFO ] Max: 28.38 ms + [ INFO ] Throughput: 281.34 FPS + + +See Also +#################### + +* :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +* :doc:`Using OpenVINO Samples ` +* :doc:`Model Downloader ` +* :doc:`Model Optimizer ` + +@endsphinxdirective diff --git a/samples/python/classification_sample_async/README.md b/samples/python/classification_sample_async/README.md index d2182839e24aa2..c705ef86ba206e 100644 --- a/samples/python/classification_sample_async/README.md +++ b/samples/python/classification_sample_async/README.md @@ -1,144 +1,166 @@ # Image Classification Async Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_classification_sample_async_README} -This sample demonstrates how to do inference of image classification models using Asynchronous Inference Request API. -Models with only 1 input and output are supported. - -The following Python API is used in the application: +@sphinxdirective -| Feature | API | Description | -| :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------ | -| Asynchronous Infer | [openvino.runtime.AsyncInferQueue], [openvino.runtime.AsyncInferQueue.set_callback], [openvino.runtime.AsyncInferQueue.start_async], [openvino.runtime.AsyncInferQueue.wait_all], [openvino.runtime.InferRequest.results] | Do asynchronous inference | +This sample demonstrates how to do inference of image classification models using Asynchronous Inference Request API. -Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample](../hello_classification/README.md). +Models with only 1 input and output are supported. -| Options | Values | -| :------------------------- | :----------------------------------------------------------------------- | -| Validated Models | [alexnet](@ref omz_models_model_alexnet) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin), ONNX (.onnx) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/classification_sample_async/README.md) | +The following Python API is used in the application: -## How It Works ++--------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------+ +| Feature | API | Description | ++====================+===========================================================================================================================================================================================================+===========================+ +| Asynchronous Infer | `openvino.runtime.AsyncInferQueue `__ , | Do asynchronous inference | +| | `openvino.runtime.AsyncInferQueue.set_callback `__ , | | +| | `openvino.runtime.AsyncInferQueue.start_async `__ , | | +| | `openvino.runtime.AsyncInferQueue.wait_all `__ , | | +| | `openvino.runtime.InferRequest.results `__ | | ++--------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification Python* Sample `. + ++----------------------------+-----------------------------------------------------------------------------------+ +| Options | Values | ++============================+===================================================================================+ +| Validated Models | :doc:`alexnet ` | ++----------------------------+-----------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin), ONNX (.onnx) | ++----------------------------+-----------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++----------------------------+-----------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++----------------------------+-----------------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application reads command-line parameters, prepares input data, loads a specified model and image(s) to the OpenVINO™ Runtime plugin, performs synchronous inference, and processes output data, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. + +Running +####### -## Running +Run the application with the ``-h`` option to see the usage message: -Run the application with the `-h` option to see the usage message: +.. code-block:: sh -``` -python classification_sample_async.py -h -``` + python classification_sample_async.py -h Usage message: -``` -usage: classification_sample_async.py [-h] -m MODEL -i INPUT [INPUT ...] - [-d DEVICE] - -Options: - -h, --help Show this help message and exit. - -m MODEL, --model MODEL - Required. Path to an .xml or .onnx file with a trained - model. - -i INPUT [INPUT ...], --input INPUT [INPUT ...] - Required. Path to an image file(s). - -d DEVICE, --device DEVICE - Optional. Specify the target device to infer on; CPU, - GPU or HETERO: is acceptable. The sample - will look for a suitable plugin for device specified. - Default value is CPU. -``` +.. code-block:: sh + + usage: classification_sample_async.py [-h] -m MODEL -i INPUT [INPUT ...] + [-d DEVICE] + + Options: + -h, --help Show this help message and exit. + -m MODEL, --model MODEL + Required. Path to an .xml or .onnx file with a trained + model. + -i INPUT [INPUT ...], --input INPUT [INPUT ...] + Required. Path to an image file(s). + -d DEVICE, --device DEVICE + Optional. Specify the target device to infer on; CPU, + GPU or HETERO: is acceptable. The sample + will look for a suitable plugin for device specified. + Default value is CPU. To run the sample, you need specify a model and image: -- you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- you can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available `here `__ . + +.. note:: + + - By default, OpenVINO™ Toolkit Samples and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + +Example ++++++++ -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: -### Example + .. code-block:: sh -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev[caffe] - ``` + python -m pip install openvino-dev[caffe] 2. Download a pre-trained model: - ``` - omz_downloader --name alexnet - ``` + + .. code-block:: sh + + omz_downloader --name alexnet + 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: - ``` - omz_converter --name alexnet - ``` -4. Perform inference of `banana.jpg` and `car.bmp` using the `alexnet` model on a `GPU`, for example: - ``` - python classification_sample_async.py -m alexnet.xml -i banana.jpg car.bmp -d GPU - ``` + .. code-block:: sh + + omz_converter --name alexnet + +4. Perform inference of ``banana.jpg`` and ``car.bmp`` using the ``alexnet`` model on a ``GPU``, for example: -## Sample Output + .. code-block:: sh + + python classification_sample_async.py -m alexnet.xml -i banana.jpg car.bmp -d GPU + +Sample Output +############# The sample application logs each step in a standard output stream and outputs top-10 inference results. -``` -[ INFO ] Creating OpenVINO Runtime Core -[ INFO ] Reading the model: C:/test_data/models/alexnet.xml -[ INFO ] Loading the model to the plugin -[ INFO ] Starting inference in asynchronous mode -[ INFO ] Image path: /test_data/images/banana.jpg -[ INFO ] Top 10 results: -[ INFO ] class_id probability -[ INFO ] -------------------- -[ INFO ] 954 0.9707602 -[ INFO ] 666 0.0216788 -[ INFO ] 659 0.0032558 -[ INFO ] 435 0.0008082 -[ INFO ] 809 0.0004359 -[ INFO ] 502 0.0003860 -[ INFO ] 618 0.0002867 -[ INFO ] 910 0.0002866 -[ INFO ] 951 0.0002410 -[ INFO ] 961 0.0002193 -[ INFO ] -[ INFO ] Image path: /test_data/images/car.bmp -[ INFO ] Top 10 results: -[ INFO ] class_id probability -[ INFO ] -------------------- -[ INFO ] 656 0.5120340 -[ INFO ] 874 0.1142275 -[ INFO ] 654 0.0697167 -[ INFO ] 436 0.0615163 -[ INFO ] 581 0.0552262 -[ INFO ] 705 0.0304179 -[ INFO ] 675 0.0151660 -[ INFO ] 734 0.0151582 -[ INFO ] 627 0.0148493 -[ INFO ] 757 0.0120964 -[ INFO ] -[ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) - -[openvino.runtime.AsyncInferQueue]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.AsyncInferQueue.html -[openvino.runtime.AsyncInferQueue.set_callback]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.AsyncInferQueue.html#openvino.runtime.AsyncInferQueue.set_callback -[openvino.runtime.AsyncInferQueue.start_async]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.AsyncInferQueue.html#openvino.runtime.AsyncInferQueue.start_async -[openvino.runtime.AsyncInferQueue.wait_all]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.AsyncInferQueue.html#openvino.runtime.AsyncInferQueue.wait_all -[openvino.runtime.InferRequest.results]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.results +.. code-block:: sh + + [ INFO ] Creating OpenVINO Runtime Core + [ INFO ] Reading the model: C:/test_data/models/alexnet.xml + [ INFO ] Loading the model to the plugin + [ INFO ] Starting inference in asynchronous mode + [ INFO ] Image path: /test_data/images/banana.jpg + [ INFO ] Top 10 results: + [ INFO ] class_id probability + [ INFO ] -------------------- + [ INFO ] 954 0.9707602 + [ INFO ] 666 0.0216788 + [ INFO ] 659 0.0032558 + [ INFO ] 435 0.0008082 + [ INFO ] 809 0.0004359 + [ INFO ] 502 0.0003860 + [ INFO ] 618 0.0002867 + [ INFO ] 910 0.0002866 + [ INFO ] 951 0.0002410 + [ INFO ] 961 0.0002193 + [ INFO ] + [ INFO ] Image path: /test_data/images/car.bmp + [ INFO ] Top 10 results: + [ INFO ] class_id probability + [ INFO ] -------------------- + [ INFO ] 656 0.5120340 + [ INFO ] 874 0.1142275 + [ INFO ] 654 0.0697167 + [ INFO ] 436 0.0615163 + [ INFO ] 581 0.0552262 + [ INFO ] 705 0.0304179 + [ INFO ] 675 0.0151660 + [ INFO ] 734 0.0151582 + [ INFO ] 627 0.0148493 + [ INFO ] 757 0.0120964 + [ INFO ] + [ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/python/hello_classification/README.md b/samples/python/hello_classification/README.md index ad342702c0a959..8ef1a3aeae8870 100644 --- a/samples/python/hello_classification/README.md +++ b/samples/python/hello_classification/README.md @@ -1,116 +1,134 @@ # Hello Classification Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_hello_classification_README} -This sample demonstrates how to do inference of image classification models using Synchronous Inference Request API. -Models with only 1 input and output are supported. +@sphinxdirective -The following Python API is used in the application: +This sample demonstrates how to do inference of image classification models using Synchronous Inference Request API. -| Feature | API | Description | -| :---------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Basic Infer Flow | [openvino.runtime.Core], [openvino.runtime.Core.read_model], [openvino.runtime.Core.compile_model] | Common API to do inference | -| Synchronous Infer | [openvino.runtime.CompiledModel.infer_new_request] | Do synchronous inference | -| Model Operations | [openvino.runtime.Model.inputs], [openvino.runtime.Model.outputs] | Managing of model | -| Preprocessing | [openvino.preprocess.PrePostProcessor], [openvino.preprocess.InputTensorInfo.set_element_type],[openvino.preprocess.InputTensorInfo.set_layout],[openvino.preprocess.InputTensorInfo.set_spatial_static_shape],[openvino.preprocess.PreProcessSteps.resize],[openvino.preprocess.InputModelInfo.set_layout],[openvino.preprocess.OutputTensorInfo.set_element_type],[openvino.preprocess.PrePostProcessor.build] | Set image of the original size as input for a model with other input size. Resize and layout conversions will be performed automatically by the corresponding plugin just before inference | +Models with only 1 input and output are supported. -| Options | Values | -| :------------------------- | :------------------------------------------------------------------------------------------------------ | -| Validated Models | [alexnet](@ref omz_models_model_alexnet), [googlenet-v1](@ref omz_models_model_googlenet_v1) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin), ONNX (.onnx) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/hello_classification/README.md), [C](../../c/hello_classification/README.md) | +The following Python API is used in the application: -## How It Works ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Feature | API | Description | ++=============================+===========================================================================================================================================================================================================================================+============================================================================================================================================================================================+ +| Basic Infer Flow | `openvino.runtime.Core `__ , | | +| | `openvino.runtime.Core.read_model `__ , | | +| | `openvino.runtime.Core.compile_model `__ | Common API to do inference | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Synchronous Infer | `openvino.runtime.CompiledModel.infer_new_request `__ | Do synchronous inference | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Operations | `openvino.runtime.Model.inputs `__ , | Managing of model | +| | `openvino.runtime.Model.outputs `__ | | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Preprocessing | `openvino.preprocess.PrePostProcessor `__ , | Set image of the original size as input for a model with other input size. Resize and layout conversions will be performed automatically by the corresponding plugin just before inference | +| | `openvino.preprocess.InputTensorInfo.set_element_type `__ , | | +| | `openvino.preprocess.InputTensorInfo.set_layout `__ , | | +| | `openvino.preprocess.InputTensorInfo.set_spatial_static_shape `__ , | | +| | `openvino.preprocess.PreProcessSteps.resize `__ , | | +| | `openvino.preprocess.InputModelInfo.set_layout `__ , | | +| | `openvino.preprocess.OutputTensorInfo.set_element_type `__ , | | +| | `openvino.preprocess.PrePostProcessor.build `__ | | ++-----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + ++-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Options | Values | ++===================================+===================================================================================================================================================================+ +| Validated Models | :doc:`alexnet `, :doc:`googlenet-v1 ` | ++-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin), ONNX (.onnx) | ++-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ `, :doc:`C ` | ++-----------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application reads command-line parameters, prepares input data, loads a specified model and image to the OpenVINO™ Runtime plugin, performs synchronous inference, and processes output data, logging each step in a standard output stream. -You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Running +Running +####### -``` -python hello_classification.py -``` +.. code-block:: console + + python hello_classification.py To run the sample, you need to specify a model and image: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- You can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `__. + +.. note:: + + - By default, OpenVINO™ Toolkit Samples and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. -### Example +Example ++++++++ -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev[caffe] - ``` +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: + + .. code-block:: console + + python -m pip install openvino-dev[caffe] 2. Download a pre-trained model: - ``` - omz_downloader --name alexnet - ``` + + .. code-block:: console + + omz_downloader --name alexnet 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: - ``` - omz_converter --name alexnet - ``` + + .. code-block:: console + + omz_converter --name alexnet -4. Perform inference of `banana.jpg` using the `alexnet` model on a `GPU`, for example: - ``` - python hello_classification.py alexnet.xml banana.jpg GPU - ``` +4. Perform inference of ``banana.jpg`` using the ``alexnet`` model on a ``GPU``, for example: + + .. code-block:: console + + python hello_classification.py alexnet.xml banana.jpg GPU -## Sample Output +Sample Output +############# The sample application logs each step in a standard output stream and outputs top-10 inference results. -``` -[ INFO ] Creating OpenVINO Runtime Core -[ INFO ] Reading the model: /models/alexnet/alexnet.xml -[ INFO ] Loading the model to the plugin -[ INFO ] Starting inference in synchronous mode -[ INFO ] Image path: /images/banana.jpg -[ INFO ] Top 10 results: -[ INFO ] class_id probability -[ INFO ] -------------------- -[ INFO ] 954 0.9703885 -[ INFO ] 666 0.0219518 -[ INFO ] 659 0.0033120 -[ INFO ] 435 0.0008246 -[ INFO ] 809 0.0004433 -[ INFO ] 502 0.0003852 -[ INFO ] 618 0.0002906 -[ INFO ] 910 0.0002848 -[ INFO ] 951 0.0002427 -[ INFO ] 961 0.0002213 -[ INFO ] -[ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) - -[openvino.runtime.Core]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Core.html -[openvino.runtime.Core.read_model]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Core.html#openvino.runtime.Core.read_model -[openvino.runtime.Core.compile_model]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Core.html#openvino.runtime.Core.compile_model -[openvino.runtime.CompiledModel.infer_new_request]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.CompiledModel.html#openvino.runtime.CompiledModel.infer_new_request -[openvino.runtime.Model.inputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html#openvino.runtime.Model.inputs -[openvino.runtime.Model.outputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html#openvino.runtime.Model.outputs -[openvino.preprocess.PrePostProcessor]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.PrePostProcessor.html -[openvino.preprocess.InputTensorInfo.set_element_type]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.InputTensorInfo.html#openvino.preprocess.InputTensorInfo.set_element_type -[openvino.preprocess.InputTensorInfo.set_layout]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.InputTensorInfo.html#openvino.preprocess.InputTensorInfo.set_layout -[openvino.preprocess.InputTensorInfo.set_spatial_static_shape]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.InputTensorInfo.html#openvino.preprocess.InputTensorInfo.set_spatial_static_shape -[openvino.preprocess.PreProcessSteps.resize]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.PreProcessSteps.html#openvino.preprocess.PreProcessSteps.resize -[openvino.preprocess.InputModelInfo.set_layout]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.InputModelInfo.html#openvino.preprocess.InputModelInfo.set_layout -[openvino.preprocess.OutputTensorInfo.set_element_type]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.OutputTensorInfo.html#openvino.preprocess.OutputTensorInfo.set_element_type -[openvino.preprocess.PrePostProcessor.build]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.preprocess.PrePostProcessor.html#openvino.preprocess.PrePostProcessor.build +.. code-block:: console + + [ INFO ] Creating OpenVINO Runtime Core + [ INFO ] Reading the model: /models/alexnet/alexnet.xml + [ INFO ] Loading the model to the plugin + [ INFO ] Starting inference in synchronous mode + [ INFO ] Image path: /images/banana.jpg + [ INFO ] Top 10 results: + [ INFO ] class_id probability + [ INFO ] -------------------- + [ INFO ] 954 0.9703885 + [ INFO ] 666 0.0219518 + [ INFO ] 659 0.0033120 + [ INFO ] 435 0.0008246 + [ INFO ] 809 0.0004433 + [ INFO ] 502 0.0003852 + [ INFO ] 618 0.0002906 + [ INFO ] 910 0.0002848 + [ INFO ] 951 0.0002427 + [ INFO ] 961 0.0002213 + [ INFO ] + [ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/python/hello_query_device/README.md b/samples/python/hello_query_device/README.md index 1b93fb6a0b055a..2d97a03875f658 100644 --- a/samples/python/hello_query_device/README.md +++ b/samples/python/hello_query_device/README.md @@ -1,93 +1,105 @@ # Hello Query Device Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_hello_query_device_README} -This sample demonstrates how to show OpenVINO™ Runtime devices and prints their metrics and default configuration values using [Query Device API feature](../../../docs/OV_Runtime_UG/supported_plugins/config_properties.md). +@sphinxdirective + +This sample demonstrates how to show OpenVINO™ Runtime devices and prints their metrics and default configuration values using :doc:`Query Device API feature `. The following Python API is used in the application: -| Feature | API | Description | -| :----------- | :---------------------------------------------------------------------------------------------------------------- | :-------------------- | -| Basic | [openvino.runtime.Core] | Common API | -| Query Device | [openvino.runtime.Core.available_devices], [openvino.runtime.Core.get_metric], [openvino.runtime.Core.get_config] | Get device properties | ++---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------+ +| Feature | API | Description | ++=======================================+============================================================================================================================================================================================+========================================+ +| Basic | `openvino.runtime.Core `__ | Common API | ++---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------+ +| Query Device | `openvino.runtime.Core.available_devices `__ , | Get device properties | +| | `openvino.runtime.Core.get_metric `__ , | | +| | `openvino.runtime.Core.get_config `__ | | ++---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------+ -| Options | Values | -| :------------------------- | :---------------------------------------------------------------- | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/hello_query_device/README.md) | ++-------------------------------------------------------+--------------------------------------------------------------------------+ +| Options | Values | ++=======================================================+==========================================================================+ +| Supported devices | :doc:`All ` | ++-------------------------------------------------------+--------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++-------------------------------------------------------+--------------------------------------------------------------------------+ -## How It Works +How It Works +############ The sample queries all available OpenVINO™ Runtime devices and prints their supported metrics and plugin configuration parameters. -## Running +Running +####### The sample has no command-line parameters. To see the report, run the following command: -``` -python hello_query_device.py -``` +.. code-block:: console + + python hello_query_device.py -## Sample Output +Sample Output +############# The application prints all available devices with their supported metrics and default values for configuration parameters. For example: -``` -[ INFO ] Available devices: -[ INFO ] CPU : -[ INFO ] SUPPORTED_METRICS: -[ INFO ] AVAILABLE_DEVICES: -[ INFO ] FULL_DEVICE_NAME: Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz -[ INFO ] OPTIMIZATION_CAPABILITIES: FP32, FP16, INT8, BIN -[ INFO ] RANGE_FOR_ASYNC_INFER_REQUESTS: 1, 1, 1 -[ INFO ] RANGE_FOR_STREAMS: 1, 8 -[ INFO ] IMPORT_EXPORT_SUPPORT: True -[ INFO ] -[ INFO ] SUPPORTED_CONFIG_KEYS (default values): -[ INFO ] CACHE_DIR: -[ INFO ] CPU_BIND_THREAD: NO -[ INFO ] CPU_THREADS_NUM: 0 -[ INFO ] CPU_THROUGHPUT_STREAMS: 1 -[ INFO ] DUMP_EXEC_GRAPH_AS_DOT: -[ INFO ] DYN_BATCH_ENABLED: NO -[ INFO ] DYN_BATCH_LIMIT: 0 -[ INFO ] ENFORCE_BF16: NO -[ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO -[ INFO ] PERFORMANCE_HINT: -[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0 -[ INFO ] PERF_COUNT: NO -[ INFO ] -[ INFO ] GNA : -[ INFO ] SUPPORTED_METRICS: -[ INFO ] AVAILABLE_DEVICES: GNA_SW -[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1 -[ INFO ] FULL_DEVICE_NAME: GNA_SW -[ INFO ] GNA_LIBRARY_FULL_VERSION: 3.0.0.1455 -[ INFO ] IMPORT_EXPORT_SUPPORT: True -[ INFO ] -[ INFO ] SUPPORTED_CONFIG_KEYS (default values): -[ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO -[ INFO ] GNA_COMPACT_MODE: YES -[ INFO ] GNA_COMPILE_TARGET: -[ INFO ] GNA_DEVICE_MODE: GNA_SW_EXACT -[ INFO ] GNA_EXEC_TARGET: -[ INFO ] GNA_FIRMWARE_MODEL_IMAGE: -[ INFO ] GNA_FIRMWARE_MODEL_IMAGE_GENERATION: -[ INFO ] GNA_LIB_N_THREADS: 1 -[ INFO ] GNA_PRECISION: I16 -[ INFO ] GNA_PWL_MAX_ERROR_PERCENT: 1.000000 -[ INFO ] GNA_PWL_UNIFORM_DESIGN: NO -[ INFO ] GNA_SCALE_FACTOR: 1.000000 -[ INFO ] GNA_SCALE_FACTOR_0: 1.000000 -[ INFO ] LOG_LEVEL: LOG_NONE -[ INFO ] PERF_COUNT: NO -[ INFO ] SINGLE_THREAD: YES -``` - -## See Also - -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - -[openvino.runtime.Core]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Core.html -[openvino.runtime.Core.available_devices]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Core.html#openvino.runtime.Core.available_devices -[openvino.runtime.Core.get_metric]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_metric -[openvino.runtime.Core.get_config]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_config +.. code-block:: console + + [ INFO ] Available devices: + [ INFO ] CPU : + [ INFO ] SUPPORTED_METRICS: + [ INFO ] AVAILABLE_DEVICES: + [ INFO ] FULL_DEVICE_NAME: Intel(R) Core(TM) i5-8350U CPU @ 1.70GHz + [ INFO ] OPTIMIZATION_CAPABILITIES: FP32, FP16, INT8, BIN + [ INFO ] RANGE_FOR_ASYNC_INFER_REQUESTS: 1, 1, 1 + [ INFO ] RANGE_FOR_STREAMS: 1, 8 + [ INFO ] IMPORT_EXPORT_SUPPORT: True + [ INFO ] + [ INFO ] SUPPORTED_CONFIG_KEYS (default values): + [ INFO ] CACHE_DIR: + [ INFO ] CPU_BIND_THREAD: NO + [ INFO ] CPU_THREADS_NUM: 0 + [ INFO ] CPU_THROUGHPUT_STREAMS: 1 + [ INFO ] DUMP_EXEC_GRAPH_AS_DOT: + [ INFO ] DYN_BATCH_ENABLED: NO + [ INFO ] DYN_BATCH_LIMIT: 0 + [ INFO ] ENFORCE_BF16: NO + [ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO + [ INFO ] PERFORMANCE_HINT: + [ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0 + [ INFO ] PERF_COUNT: NO + [ INFO ] + [ INFO ] GNA : + [ INFO ] SUPPORTED_METRICS: + [ INFO ] AVAILABLE_DEVICES: GNA_SW + [ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1 + [ INFO ] FULL_DEVICE_NAME: GNA_SW + [ INFO ] GNA_LIBRARY_FULL_VERSION: 3.0.0.1455 + [ INFO ] IMPORT_EXPORT_SUPPORT: True + [ INFO ] + [ INFO ] SUPPORTED_CONFIG_KEYS (default values): + [ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO + [ INFO ] GNA_COMPACT_MODE: YES + [ INFO ] GNA_COMPILE_TARGET: + [ INFO ] GNA_DEVICE_MODE: GNA_SW_EXACT + [ INFO ] GNA_EXEC_TARGET: + [ INFO ] GNA_FIRMWARE_MODEL_IMAGE: + [ INFO ] GNA_FIRMWARE_MODEL_IMAGE_GENERATION: + [ INFO ] GNA_LIB_N_THREADS: 1 + [ INFO ] GNA_PRECISION: I16 + [ INFO ] GNA_PWL_MAX_ERROR_PERCENT: 1.000000 + [ INFO ] GNA_PWL_UNIFORM_DESIGN: NO + [ INFO ] GNA_SCALE_FACTOR: 1.000000 + [ INFO ] GNA_SCALE_FACTOR_0: 1.000000 + [ INFO ] LOG_LEVEL: LOG_NONE + [ INFO ] PERF_COUNT: NO + [ INFO ] SINGLE_THREAD: YES + +See Also +######## + +- :doc:`Using OpenVINO™ Toolkit Samples ` + +@endsphinxdirective + diff --git a/samples/python/hello_reshape_ssd/README.md b/samples/python/hello_reshape_ssd/README.md index 0341bea096d84a..ab27d66072c8cf 100644 --- a/samples/python/hello_reshape_ssd/README.md +++ b/samples/python/hello_reshape_ssd/README.md @@ -1,95 +1,115 @@ # Hello Reshape SSD Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_hello_reshape_ssd_README} -This sample demonstrates how to do synchronous inference of object detection models using [Shape Inference feature](../../../docs/OV_Runtime_UG/ShapeInference.md). -Models with only 1 input and output are supported. - -The following Python API is used in the application: +@sphinxdirective -| Feature | API | Description | -| :--------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | :---------------- | -| Model Operations | [openvino.runtime.Model.reshape], [openvino.runtime.Model.input], [openvino.runtime.Output.get_any_name], [openvino.runtime.PartialShape] | Managing of model | +This sample demonstrates how to do synchronous inference of object detection models using :doc:`Shape Inference feature `. -Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample](../hello_classification/README.md). +Models with only 1 input and output are supported. -| Options | Values | -| :------------------------- | :----------------------------------------------------------------------- | -| Validated Models | [mobilenet-ssd](@ref omz_models_model_mobilenet_ssd) | -| Validated Layout | NCHW | -| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin), ONNX (.onnx) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/hello_reshape_ssd/README.md) | +The following Python API is used in the application: -## How It Works ++------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------+ +| Feature | API | Description | ++====================================+================================================================================================================================================================================+======================================+ +| Model Operations | `openvino.runtime.Model.reshape `__ , | Managing of model | +| | `openvino.runtime.Model.input `__ , | | +| | `openvino.runtime.Output.get_any_name `__ , | | +| | `openvino.runtime.PartialShape `__ | | ++------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification Python* Sample `. + ++------------------------------------+---------------------------------------------------------------------------+ +| Options | Values | ++====================================+===========================================================================+ +| Validated Models | :doc:`mobilenet-ssd ` | ++------------------------------------+---------------------------------------------------------------------------+ +| Validated Layout | NCHW | ++------------------------------------+---------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin), ONNX (.onnx) | ++------------------------------------+---------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++------------------------------------+---------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++------------------------------------+---------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application reads command-line parameters, prepares input data, loads a specified model and image to the OpenVINO™ Runtime plugin, performs synchronous inference, and processes output data. As a result, the program creates an output image, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Running +Running +####### -``` -python hello_reshape_ssd.py -``` +.. code-block:: console + + python hello_reshape_ssd.py To run the sample, you need to specify a model and image: -- You can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). -- You can use images from the media files collection available at https://storage.openvinotoolkit.org/data/test_data. -> **NOTES**: -> -> - By default, OpenVINO™ Toolkit Samples and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Embedding Preprocessing Computation](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. +- You can use :doc:`public ` or :doc:`Intel's ` pre-trained models from the Open Model Zoo. The models can be downloaded using the :doc:`Model Downloader `. +- You can use images from the media files collection available at `the storage `. -### Example +.. note:: + + - By default, OpenVINO™ Toolkit Samples and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with ``--reverse_input_channels`` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of :doc:`Embedding Preprocessing Computation `. + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample accepts models in ONNX format (.onnx) that do not require preprocessing. -1. Install the `openvino-dev` Python package to use Open Model Zoo Tools: - ``` - python -m pip install openvino-dev[caffe] - ``` +Example ++++++++ + +1. Install the ``openvino-dev`` Python package to use Open Model Zoo Tools: + + .. code-block:: console + + python -m pip install openvino-dev[caffe] 2. Download a pre-trained model: - ``` - omz_downloader --name mobilenet-ssd - ``` + + .. code-block:: console + + omz_downloader --name mobilenet-ssd 3. If a model is not in the IR or ONNX format, it must be converted. You can do this using the model converter: - ``` - omz_converter --name mobilenet-ssd - ``` + + .. code-block:: console + + omz_converter --name mobilenet-ssd -4. Perform inference of `banana.jpg` using `ssdlite_mobilenet_v2` model on a `GPU`, for example: - ``` - python hello_reshape_ssd.py mobilenet-ssd.xml banana.jpg GPU - ``` +4. Perform inference of ``banana.jpg`` using ``ssdlite_mobilenet_v2`` model on a ``GPU``, for example: + + .. code-block:: console + + python hello_reshape_ssd.py mobilenet-ssd.xml banana.jpg GPU -## Sample Output +Sample Output +############# The sample application logs each step in a standard output stream and creates an output image, drawing bounding boxes for inference results with an over 50% confidence. -``` -[ INFO ] Creating OpenVINO Runtime Core -[ INFO ] Reading the model: C:/test_data/models/mobilenet-ssd.xml -[ INFO ] Reshaping the model to the height and width of the input image -[ INFO ] Loading the model to the plugin -[ INFO ] Starting inference in synchronous mode -[ INFO ] Found: class_id = 52, confidence = 0.98, coords = (21, 98), (276, 210) -[ INFO ] Image out.bmp was created! -[ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) - -[openvino.runtime.Model.reshape]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html#openvino.runtime.Model.reshape -[openvino.runtime.Model.input]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html#openvino.runtime.Model.input -[openvino.runtime.Output.get_any_name]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Output.html#openvino.runtime.Output.get_any_name -[openvino.runtime.PartialShape]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.PartialShape.html +.. code-block:: console + + [ INFO ] Creating OpenVINO Runtime Core + [ INFO ] Reading the model: C:/test_data/models/mobilenet-ssd.xml + [ INFO ] Reshaping the model to the height and width of the input image + [ INFO ] Loading the model to the plugin + [ INFO ] Starting inference in synchronous mode + [ INFO ] Found: class_id = 52, confidence = 0.98, coords = (21, 98), (276, 210) + [ INFO ] Image out.bmp was created! + [ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/python/model_creation_sample/README.md b/samples/python/model_creation_sample/README.md index f8fef9427231b2..986ffc2587925b 100644 --- a/samples/python/model_creation_sample/README.md +++ b/samples/python/model_creation_sample/README.md @@ -1,146 +1,160 @@ # Model Creation Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_model_creation_sample_README} -This sample demonstrates how to run inference using a [model](../../../docs/OV_Runtime_UG/model_representation.md) built on the fly that uses weights from the LeNet classification model, which is known to work well on digit classification tasks. You do not need an XML file, the model is created from the source code on the fly. +@sphinxdirective -The following OpenVINO Python API is used in the application: - -| Feature | API | Description | -| :--------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------ | -| Model Operations | [openvino.runtime.Model], [openvino.runtime.set_batch], [openvino.runtime.Model.input] | Managing of model | -| Opset operations | [openvino.runtime.op.Parameter], [openvino.runtime.op.Constant], [openvino.runtime.opset8.convolution], [openvino.runtime.opset8.add], [openvino.runtime.opset1.max_pool], [openvino.runtime.opset8.reshape], [openvino.runtime.opset8.matmul], [openvino.runtime.opset8.relu], [openvino.runtime.opset8.softmax] | Description of a model topology using OpenVINO Python API | - -Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample](../hello_classification/README.md). +This sample demonstrates how to run inference using a :doc:`model ` built on the fly that uses weights from the LeNet classification model, which is known to work well on digit classification tasks. You do not need an XML file, the model is created from the source code on the fly. -| Options | Values | -| :------------------------- | :-------------------------------------------------------------------- | -| Validated Models | LeNet | -| Model Format | Model weights file (\*.bin) | -| Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/model_creation_sample/README.md) | +The following OpenVINO Python API is used in the application: -## How It Works ++------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| Feature | API | Description | ++==========================================+==============================================================================================================================================================+====================================================================================+ +| Model Operations | `openvino.runtime.Model `__ , | Managing of model | +| | `openvino.runtime.set_batch `__ , | | +| | `openvino.runtime.Model.input `__ | | ++------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ +| Opset operations | `openvino.runtime.op.Parameter `__ , | Description of a model topology using OpenVINO Python API | +| | `openvino.runtime.op.Constant `__ , | | +| | `openvino.runtime.opset8.convolution `__ , | | +| | `openvino.runtime.opset8.add `__ , | | +| | `openvino.runtime.opset1.max_pool `__ , | | +| | `openvino.runtime.opset8.reshape `__ , | | +| | `openvino.runtime.opset8.matmul `__ , | | +| | `openvino.runtime.opset8.relu `__ , | | +| | `openvino.runtime.opset8.softmax `__ | | ++------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification Python* Sample `. + ++------------------------------------------------+-----------------------------------------------------------------------------+ +| Options | Values | ++================================================+=============================================================================+ +| Validated Models | LeNet | ++------------------------------------------------+-----------------------------------------------------------------------------+ +| Model Format | Model weights file (\*.bin) | ++------------------------------------------------+-----------------------------------------------------------------------------+ +| Supported devices | :doc:`All ` | ++------------------------------------------------+-----------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++------------------------------------------------+-----------------------------------------------------------------------------+ + +How It Works +############ At startup, the sample application does the following: + - Reads command line parameters -- [Build a Model](../../../docs/OV_Runtime_UG/model_representation.md) and passed weights file +- :doc:`Build a Model ` and passed weights file - Loads the model and input data to the OpenVINO™ Runtime plugin - Performs synchronous inference and processes output data, logging each step in a standard output stream -You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## Running +Running +####### To run the sample, you need to specify model weights and device. -``` -python model_creation_sample.py -``` +.. code-block:: console + + python model_creation_sample.py -> **NOTE**: -> -> - This sample supports models with FP32 weights only. -> -> - The `lenet.bin` weights file was generated by the [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) tool from the public LeNet model with the `--input_shape [64,1,28,28]` parameter specified. -> -> - The original model is available in the [Caffe* repository](https://github.com/BVLC/caffe/tree/master/examples/mnist) on GitHub\*. +.. note:: + + - This sample supports models with FP32 weights only. + + - The ``lenet.bin`` weights file was generated by the :doc:`Model Optimizer ` tool from the public LeNet model with the ``--input_shape [64,1,28,28]`` parameter specified. + + - The original model is available in the `Caffe* repository `__ on GitHub\*. For example: -``` -python model_creation_sample.py lenet.bin GPU -``` +.. code-block:: console + + python model_creation_sample.py lenet.bin GPU -## Sample Output +Sample Output +############# The sample application logs each step in a standard output stream and outputs 10 inference results. -``` -[ INFO ] Creating OpenVINO Runtime Core -[ INFO ] Loading the model using ngraph function with weights from lenet.bin -[ INFO ] Loading the model to the plugin -[ INFO ] Starting inference in synchronous mode -[ INFO ] Top 1 results: -[ INFO ] Image 0 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 0 1.0000000 0 -[ INFO ] -[ INFO ] Image 1 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 1 1.0000000 1 -[ INFO ] -[ INFO ] Image 2 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 2 1.0000000 2 -[ INFO ] -[ INFO ] Image 3 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 3 1.0000000 3 -[ INFO ] -[ INFO ] Image 4 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 4 1.0000000 4 -[ INFO ] -[ INFO ] Image 5 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 5 1.0000000 5 -[ INFO ] -[ INFO ] Image 6 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 6 1.0000000 6 -[ INFO ] -[ INFO ] Image 7 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 7 1.0000000 7 -[ INFO ] -[ INFO ] Image 8 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 8 1.0000000 8 -[ INFO ] -[ INFO ] Image 9 -[ INFO ] -[ INFO ] classid probability label -[ INFO ] ------------------------- -[ INFO ] 9 1.0000000 9 -[ INFO ] -[ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) - -[openvino.runtime.Model]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html -[openvino.runtime.set_batch]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.set_batch.html -[openvino.runtime.Model.input]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html#openvino.runtime.Model.input -[openvino.runtime.op.Parameter]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.op.Parameter.html -[openvino.runtime.op.Constant]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.op.Constant.html -[openvino.runtime.opset8.convolution]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset8.convolution.html -[openvino.runtime.opset8.add]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset8.add.html -[openvino.runtime.opset1.max_pool]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset1.max_pool.html -[openvino.runtime.opset8.reshape]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset8.reshape.html -[openvino.runtime.opset8.matmul]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset8.matmul.html -[openvino.runtime.opset8.relu]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset8.relu.html -[openvino.runtime.opset8.softmax]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.opset8.softmax.html +.. code-block:: console + + [ INFO ] Creating OpenVINO Runtime Core + [ INFO ] Loading the model using ngraph function with weights from lenet.bin + [ INFO ] Loading the model to the plugin + [ INFO ] Starting inference in synchronous mode + [ INFO ] Top 1 results: + [ INFO ] Image 0 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 0 1.0000000 0 + [ INFO ] + [ INFO ] Image 1 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 1 1.0000000 1 + [ INFO ] + [ INFO ] Image 2 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 2 1.0000000 2 + [ INFO ] + [ INFO ] Image 3 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 3 1.0000000 3 + [ INFO ] + [ INFO ] Image 4 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 4 1.0000000 4 + [ INFO ] + [ INFO ] Image 5 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 5 1.0000000 5 + [ INFO ] + [ INFO ] Image 6 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 6 1.0000000 6 + [ INFO ] + [ INFO ] Image 7 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 7 1.0000000 7 + [ INFO ] + [ INFO ] Image 8 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 8 1.0000000 8 + [ INFO ] + [ INFO ] Image 9 + [ INFO ] + [ INFO ] classid probability label + [ INFO ] ------------------------- + [ INFO ] 9 1.0000000 9 + [ INFO ] + [ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/samples/python/speech_sample/README.md b/samples/python/speech_sample/README.md index 588e145459da13..dd3cc51ee87481 100644 --- a/samples/python/speech_sample/README.md +++ b/samples/python/speech_sample/README.md @@ -1,164 +1,190 @@ # Automatic Speech Recognition Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_speech_sample_README} +@sphinxdirective + This sample demonstrates how to do a Synchronous Inference of acoustic model based on Kaldi\* neural models and speech feature vectors. The sample works with Kaldi ARK or Numpy* uncompressed NPZ files, so it does not cover an end-to-end speech recognition scenario (speech to text), requiring additional preprocessing (feature extraction) to get a feature vector from a speech signal, as well as postprocessing (decoding) to produce text from scores. Automatic Speech Recognition Python sample application demonstrates how to use the following Python API in applications: -| Feature | API | Description | -| :---------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------- | -| Import/Export Model | [openvino.runtime.Core.import_model], [openvino.runtime.CompiledModel.export_model] | The GNA plugin supports loading and saving of the GNA-optimized model | -| Model Operations | [openvino.runtime.Model.add_outputs], [openvino.runtime.set_batch], [openvino.runtime.CompiledModel.inputs], [openvino.runtime.CompiledModel.outputs], [openvino.runtime.ConstOutput.any_name] | Managing of model: configure batch_size, input and output tensors | -| Synchronous Infer | [openvino.runtime.CompiledModel.create_infer_request], [openvino.runtime.InferRequest.infer] | Do synchronous inference | -| InferRequest Operations | [openvino.runtime.InferRequest.get_input_tensor], [openvino.runtime.InferRequest.model_outputs], [openvino.runtime.InferRequest.model_inputs], | Get info about model using infer request API | -| InferRequest Operations | [openvino.runtime.InferRequest.query_state], [openvino.runtime.VariableState.reset] | Gets and resets CompiledModel state control | -| Profiling | [openvino.runtime.InferRequest.profiling_info], [openvino.runtime.ProfilingInfo.real_time] | Get infer request profiling info | - -Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample](../hello_classification/README.md). - -| Options | Values | -| :------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- | -| Validated Models | Acoustic model based on Kaldi* neural models (see [Model Preparation](#model-preparation-speech-python) section) | -| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin) | -| Supported devices | See [Execution Modes](#execution-modes-speech-python) section below and [List Supported Devices](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/speech_sample/README.md) | - -## How It Works ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ +| Feature | API | Description | ++===================================================================+================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================+=======================================================================+ +| Import/Export Model | `openvino.runtime.Core.import_model `__ , `openvino.runtime.CompiledModel.export_model `__ | The GNA plugin supports loading and saving of the GNA-optimized model | ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ +| Model Operations | `openvino.runtime.Model.add_outputs `__ , `openvino.runtime.set_batch `__ , `openvino.runtime.CompiledModel.inputs `__ , `openvino.runtime.CompiledModel.outputs `__ , `openvino.runtime.ConstOutput.any_name `__ | Managing of model: configure batch_size, input and output tensors | ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ +| Synchronous Infer | `openvino.runtime.CompiledModel.create_infer_request `__ , `openvino.runtime.InferRequest.infer `__ | Do synchronous inference | ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ +| InferRequest Operations | `openvino.runtime.InferRequest.get_input_tensor `__ , `openvino.runtime.InferRequest.model_outputs `__ , `openvino.runtime.InferRequest.model_inputs `__ , | Get info about model using infer request API | ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ +| InferRequest Operations | `openvino.runtime.InferRequest.query_state `__ , `openvino.runtime.VariableState.reset `__ | Gets and resets CompiledModel state control | ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ +| Profiling | `openvino.runtime.InferRequest.profiling_info `__ , `openvino.runtime.ProfilingInfo.real_time `__ | Get infer request profiling info | ++-------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------+ + +Basic OpenVINO™ Runtime API is covered by :doc:`Hello Classification Python* Sample `. + ++----------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Options | Values | ++======================================================================+======================================================================================================================================================================+ +| Validated Models | Acoustic model based on Kaldi* neural models (see :ref:`Model Preparation ` section) | ++----------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Model Format | OpenVINO™ toolkit Intermediate Representation (.xml + .bin) | ++----------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Supported devices | See :ref:`Execution Modes ` section below and :doc:`List Supported Devices ` | ++----------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Other language realization | :doc:`C++ ` | ++----------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +How It Works +############ At startup, the sample application reads command-line parameters, loads a specified model and input data to the OpenVINO™ Runtime plugin, performs synchronous inference on all speech utterances stored in the input file, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. +each sample step at :doc:`Integration Steps ` section of "Integrate OpenVINO™ Runtime with Your Application" guide. -## GNA-specific details +GNA-specific details +#################### -### Quantization +Quantization +++++++++++++ -If the GNA device is selected (for example, using the `-d` GNA flag), the GNA OpenVINO™ Runtime plugin quantizes the model and input feature vector sequence to integer representation before performing inference. +If the GNA device is selected (for example, using the ``-d`` GNA flag), the GNA OpenVINO™ Runtime plugin quantizes the model and input feature vector sequence to integer representation before performing inference. Several neural model quantization modes: - *static* - The first utterance in the input file is scanned for dynamic range. The scale factor (floating point scalar multiplier) required to scale the maximum input value of the first utterance to 16384 (15 bits) is used for all subsequent inputs. The neural model is quantized to accommodate the scaled input dynamic range. -- *user-defined* - The user may specify a scale factor via the `-sf` flag that will be used for static quantization. +- *user-defined* - The user may specify a scale factor via the ``-sf`` flag that will be used for static quantization. -The `-qb` flag provides a hint to the GNA plugin regarding the preferred target weight resolution for all layers. -For example, when `-qb 8` is specified, the plugin will use 8-bit weights wherever possible in the +The ``-qb`` flag provides a hint to the GNA plugin regarding the preferred target weight resolution for all layers. +For example, when ``-qb 8`` is specified, the plugin will use 8-bit weights wherever possible in the model. -> **NOTE**: -> -> - It is not always possible to use 8-bit weights due to GNA hardware limitations. For example, convolutional layers always use 16-bit weights (GNA hardware version 1 and 2). This limitation will be removed in GNA hardware version 3 and higher. -> +.. note:: + + It is not always possible to use 8-bit weights due to GNA hardware limitations. For example, convolutional layers always use 16-bit weights (GNA hardware version 1 and 2). This limitation will be removed in GNA hardware version 3 and higher. -### Execution Modes +.. _execution-modes-speech-python: -Several execution modes are supported via the `-d` flag: +Execution Modes ++++++++++++++++ -- `CPU` - All calculations are performed on CPU device using CPU Plugin. -- `GPU` - All calculations are performed on GPU device using GPU Plugin. -- `VPUX` - All calculations are performed on VPUX device using VPUX Plugin. -- `GNA_AUTO` - GNA hardware is used if available and the driver is installed. Otherwise, the GNA device is emulated in fast-but-not-bit-exact mode. -- `GNA_HW` - GNA hardware is used if available and the driver is installed. Otherwise, an error will occur. -- `GNA_SW` - Deprecated. The GNA device is emulated in fast-but-not-bit-exact mode. -- `GNA_SW_FP32` - Substitutes parameters and calculations from low precision to floating point (FP32). -- `GNA_SW_EXACT` - GNA device is emulated in bit-exact mode. +Several execution modes are supported via the ``-d`` flag: -### Loading and Saving Models +- ``CPU`` - All calculations are performed on CPU device using CPU Plugin. +- ``GPU`` - All calculations are performed on GPU device using GPU Plugin. +- ``VPUX`` - All calculations are performed on VPUX device using VPUX Plugin. +- ``GNA_AUTO`` - GNA hardware is used if available and the driver is installed. Otherwise, the GNA device is emulated in fast-but-not-bit-exact mode. +- ``GNA_HW`` - GNA hardware is used if available and the driver is installed. Otherwise, an error will occur. +- ``GNA_SW`` - Deprecated. The GNA device is emulated in fast-but-not-bit-exact mode. +- ``GNA_SW_FP32`` - Substitutes parameters and calculations from low precision to floating point (FP32). +- ``GNA_SW_EXACT`` - GNA device is emulated in bit-exact mode. -The GNA plugin supports loading and saving of the GNA-optimized model (non-IR) via the `-rg` and `-wg` flags. +Loading and Saving Models ++++++++++++++++++++++++++ + +The GNA plugin supports loading and saving of the GNA-optimized model (non-IR) via the ``-rg`` and ``-wg`` flags. Thereby, it is possible to avoid the cost of full model quantization at run time. -The GNA plugin also supports export of firmware-compatible embedded model images for the Intel® Speech Enabling Developer Kit and Amazon Alexa* Premium Far-Field Voice Development Kit via the `-we` flag (save only). +The GNA plugin also supports export of firmware-compatible embedded model images for the Intel® Speech Enabling Developer Kit and Amazon Alexa* Premium Far-Field Voice Development Kit via the ``-we`` flag (save only). In addition to performing inference directly from a GNA model file, these options make it possible to: -- Convert from IR format to GNA format model file (`-m`, `-wg`) -- Convert from IR format to embedded format model file (`-m`, `-we`) -- Convert from GNA format to embedded format model file (`-rg`, `-we`) +- Convert from IR format to GNA format model file (``-m``, ``-wg``) +- Convert from IR format to embedded format model file (``-m``, ``-we``) +- Convert from GNA format to embedded format model file (``-rg``, ``-we``) -## Running +Running +####### -Run the application with the `-h` option to see the usage message: +Run the application with the ``-h`` option to see the usage message: -``` -python speech_sample.py -h -``` +.. code-block:: bash + + python speech_sample.py -h Usage message: -``` -usage: speech_sample.py [-h] (-m MODEL | -rg IMPORT_GNA_MODEL) -i INPUT [-o OUTPUT] [-r REFERENCE] [-d DEVICE] [-bs [1-8]] - [-layout LAYOUT] [-qb [8, 16]] [-sf SCALE_FACTOR] [-wg EXPORT_GNA_MODEL] - [-we EXPORT_EMBEDDED_GNA_MODEL] [-we_gen [GNA1, GNA3]] - [--exec_target [GNA_TARGET_2_0, GNA_TARGET_3_0]] [-pc] [-a [CORE, ATOM]] [-iname INPUT_LAYERS] - [-oname OUTPUT_LAYERS] [-cw_l CONTEXT_WINDOW_LEFT] [-cw_r CONTEXT_WINDOW_RIGHT] [-pwl_me PWL_ME] - -optional arguments: - -m MODEL, --model MODEL - Path to an .xml file with a trained model (required if -rg is missing). - -rg IMPORT_GNA_MODEL, --import_gna_model IMPORT_GNA_MODEL - Read GNA model from file using path/filename provided (required if -m is missing). - -Options: - -h, --help Show this help message and exit. - -i INPUT, --input INPUT - Required. Path(s) to input file(s). - Usage for a single file/layer: or . - Example of usage for several files/layers: :=,:=. - -o OUTPUT, --output OUTPUT - Optional. Output file name(s) to save scores (inference results). - Usage for a single file/layer: or . - Example of usage for several files/layers: :=,:=. - -r REFERENCE, --reference REFERENCE - Read reference score file(s) and compare inference results with reference scores. - Usage for a single file/layer: or . - Example of usage for several files/layers: :=,:=. - -d DEVICE, --device DEVICE - Optional. Specify a target device to infer on. CPU, GPU, VPUX, GNA_AUTO, GNA_HW, GNA_SW_FP32, - GNA_SW_EXACT and HETERO with combination of GNA as the primary device and CPU as a secondary (e.g. - HETERO:GNA,CPU) are supported. The sample will look for a suitable plugin for device specified. - Default value is CPU. - -bs [1-8], --batch_size [1-8] - Optional. Batch size 1-8. - -layout LAYOUT Optional. Custom layout in format: "input0[value0],input1[value1]" or "[value]" (applied to all - inputs) - -qb [8, 16], --quantization_bits [8, 16] - Optional. Weight resolution in bits for GNA quantization: 8 or 16 (default 16). - -sf SCALE_FACTOR, --scale_factor SCALE_FACTOR - Optional. User-specified input scale factor for GNA quantization. - If the model contains multiple inputs, provide scale factors by separating them with commas. - For example: :,: or just to be applied to all inputs. - -wg EXPORT_GNA_MODEL, --export_gna_model EXPORT_GNA_MODEL - Optional. Write GNA model to file using path/filename provided. - -we EXPORT_EMBEDDED_GNA_MODEL, --export_embedded_gna_model EXPORT_EMBEDDED_GNA_MODEL - Optional. Write GNA embedded model to file using path/filename provided. - -we_gen [GNA1, GNA3], --embedded_gna_configuration [GNA1, GNA3] - Optional. GNA generation configuration string for embedded export. Can be GNA1 (default) or GNA3. - --exec_target [GNA_TARGET_2_0, GNA_TARGET_3_0] - Optional. Specify GNA execution target generation. By default, generation corresponds to the GNA HW - available in the system or the latest fully supported generation by the software. See the GNA - Plugin's GNA_EXEC_TARGET config option description. - -pc, --performance_counter - Optional. Enables performance report (specify -a to ensure arch accurate results). - -a [CORE, ATOM], --arch [CORE, ATOM] - Optional. Specify architecture. CORE, ATOM with the combination of -pc. - -cw_l CONTEXT_WINDOW_LEFT, --context_window_left CONTEXT_WINDOW_LEFT - Optional. Number of frames for left context windows (default is 0). Works only with context window - models. If you use the cw_l or cw_r flag, then batch size argument is ignored. - -cw_r CONTEXT_WINDOW_RIGHT, --context_window_right CONTEXT_WINDOW_RIGHT - Optional. Number of frames for right context windows (default is 0). Works only with context window - models. If you use the cw_l or cw_r flag, then batch size argument is ignored. - -pwl_me PWL_ME Optional. The maximum percent of error for PWL function. The value must be in <0, 100> range. The - default value is 1.0. -``` - -## Model Preparation +.. code-block:: console + + usage: speech_sample.py [-h] (-m MODEL | -rg IMPORT_GNA_MODEL) -i INPUT [-o OUTPUT] [-r REFERENCE] [-d DEVICE] [-bs [1-8]] + [-layout LAYOUT] [-qb [8, 16]] [-sf SCALE_FACTOR] [-wg EXPORT_GNA_MODEL] + [-we EXPORT_EMBEDDED_GNA_MODEL] [-we_gen [GNA1, GNA3]] + [--exec_target [GNA_TARGET_2_0, GNA_TARGET_3_0]] [-pc] [-a [CORE, ATOM]] [-iname INPUT_LAYERS] + [-oname OUTPUT_LAYERS] [-cw_l CONTEXT_WINDOW_LEFT] [-cw_r CONTEXT_WINDOW_RIGHT] [-pwl_me PWL_ME] + + optional arguments: + -m MODEL, --model MODEL + Path to an .xml file with a trained model (required if -rg is missing). + -rg IMPORT_GNA_MODEL, --import_gna_model IMPORT_GNA_MODEL + Read GNA model from file using path/filename provided (required if -m is missing). + + Options: + -h, --help Show this help message and exit. + -i INPUT, --input INPUT + Required. Path(s) to input file(s). + Usage for a single file/layer: or . + Example of usage for several files/layers: :=,:=. + -o OUTPUT, --output OUTPUT + Optional. Output file name(s) to save scores (inference results). + Usage for a single file/layer: or . + Example of usage for several files/layers: :=,:=. + -r REFERENCE, --reference REFERENCE + Read reference score file(s) and compare inference results with reference scores. + Usage for a single file/layer: or . + Example of usage for several files/layers: :=,:=. + -d DEVICE, --device DEVICE + Optional. Specify a target device to infer on. CPU, GPU, VPUX, GNA_AUTO, GNA_HW, GNA_SW_FP32, + GNA_SW_EXACT and HETERO with combination of GNA as the primary device and CPU as a secondary (e.g. + HETERO:GNA,CPU) are supported. The sample will look for a suitable plugin for device specified. + Default value is CPU. + -bs [1-8], --batch_size [1-8] + Optional. Batch size 1-8. + -layout LAYOUT Optional. Custom layout in format: "input0[value0],input1[value1]" or "[value]" (applied to all + inputs) + -qb [8, 16], --quantization_bits [8, 16] + Optional. Weight resolution in bits for GNA quantization: 8 or 16 (default 16). + -sf SCALE_FACTOR, --scale_factor SCALE_FACTOR + Optional. User-specified input scale factor for GNA quantization. + If the model contains multiple inputs, provide scale factors by separating them with commas. + For example: :,: or just to be applied to all inputs. + -wg EXPORT_GNA_MODEL, --export_gna_model EXPORT_GNA_MODEL + Optional. Write GNA model to file using path/filename provided. + -we EXPORT_EMBEDDED_GNA_MODEL, --export_embedded_gna_model EXPORT_EMBEDDED_GNA_MODEL + Optional. Write GNA embedded model to file using path/filename provided. + -we_gen [GNA1, GNA3], --embedded_gna_configuration [GNA1, GNA3] + Optional. GNA generation configuration string for embedded export. Can be GNA1 (default) or GNA3. + --exec_target [GNA_TARGET_2_0, GNA_TARGET_3_0] + Optional. Specify GNA execution target generation. By default, generation corresponds to the GNA HW + available in the system or the latest fully supported generation by the software. See the GNA + Plugin's GNA_EXEC_TARGET config option description. + -pc, --performance_counter + Optional. Enables performance report (specify -a to ensure arch accurate results). + -a [CORE, ATOM], --arch [CORE, ATOM] + Optional. Specify architecture. CORE, ATOM with the combination of -pc. + -cw_l CONTEXT_WINDOW_LEFT, --context_window_left CONTEXT_WINDOW_LEFT + Optional. Number of frames for left context windows (default is 0). Works only with context window + models. If you use the cw_l or cw_r flag, then batch size argument is ignored. + -cw_r CONTEXT_WINDOW_RIGHT, --context_window_right CONTEXT_WINDOW_RIGHT + Optional. Number of frames for right context windows (default is 0). Works only with context window + models. If you use the cw_l or cw_r flag, then batch size argument is ignored. + -pwl_me PWL_ME Optional. The maximum percent of error for PWL function. The value must be in <0, 100> range. The + default value is 1.0. + + +.. _model-preparation-speech-python: + +Model Preparation +################# You can use the following model optimizer command to convert a Kaldi nnet1 or nnet2 neural model to OpenVINO™ toolkit Intermediate Representation format: -```sh -mo --framework kaldi --input_model wsj_dnn5b.nnet --counts wsj_dnn5b.counts --remove_output_softmax --output_dir -``` +.. code-block:: sh + + mo --framework kaldi --input_model wsj_dnn5b.nnet --counts wsj_dnn5b.counts --remove_output_softmax --output_dir The following pre-trained models are available: @@ -166,189 +192,179 @@ The following pre-trained models are available: - rm_lstm4f - wsj_dnn5b_smbr -All of them can be downloaded from [https://storage.openvinotoolkit.org/models_contrib/speech/2021.2](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2). +All of them can be downloaded from `the storage `. -## Speech Inference +Speech Inference +################ You can do inference on Intel® Processors with the GNA co-processor (or emulation library): -``` -python speech_sample.py -m wsj_dnn5b.xml -i dev93_10.ark -r dev93_scores_10.ark -d GNA_AUTO -o result.npz -``` +.. code-block:: sh + + python speech_sample.py -m wsj_dnn5b.xml -i dev93_10.ark -r dev93_scores_10.ark -d GNA_AUTO -o result.npz + -> **NOTES**: -> -> - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). -> -> - The sample supports input and output in numpy file format (.npz) +.. note:: -## Sample Output + - Before running the sample with a trained model, make sure the model is converted to the intermediate representation (IR) format (\*.xml + \*.bin) using the :doc:`Model Optimizer tool `. + - The sample supports input and output in numpy file format (.npz) + +Sample Output +############# The sample application logs each step in a standard output stream. -``` -[ INFO ] Creating OpenVINO Runtime Core -[ INFO ] Reading the model: /models/wsj_dnn5b_smbr_fp32.xml -[ INFO ] Using scale factor(s) calculated from first utterance -[ INFO ] For input 0 using scale factor of 2175.4322418 -[ INFO ] Loading the model to the plugin -[ INFO ] Starting inference in synchronous mode -[ INFO ] -[ INFO ] Utterance 0: -[ INFO ] Total time in Infer (HW and SW): 6326.06ms -[ INFO ] Frames in utterance: 1294 -[ INFO ] Average Infer time per frame: 4.89ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7051840 -[ INFO ] avg error: 0.0448388 -[ INFO ] avg rms error: 0.0582387 -[ INFO ] stdev error: 0.0371650 -[ INFO ] -[ INFO ] Utterance 1: -[ INFO ] Total time in Infer (HW and SW): 4526.57ms -[ INFO ] Frames in utterance: 1005 -[ INFO ] Average Infer time per frame: 4.50ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7575974 -[ INFO ] avg error: 0.0452166 -[ INFO ] avg rms error: 0.0586013 -[ INFO ] stdev error: 0.0372769 -[ INFO ] -[ INFO ] Utterance 2: -[ INFO ] Total time in Infer (HW and SW): 6636.56ms -[ INFO ] Frames in utterance: 1471 -[ INFO ] Average Infer time per frame: 4.51ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7191710 -[ INFO ] avg error: 0.0472226 -[ INFO ] avg rms error: 0.0612991 -[ INFO ] stdev error: 0.0390846 -[ INFO ] -[ INFO ] Utterance 3: -[ INFO ] Total time in Infer (HW and SW): 3927.01ms -[ INFO ] Frames in utterance: 845 -[ INFO ] Average Infer time per frame: 4.65ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7436461 -[ INFO ] avg error: 0.0477581 -[ INFO ] avg rms error: 0.0621334 -[ INFO ] stdev error: 0.0397457 -[ INFO ] -[ INFO ] Utterance 4: -[ INFO ] Total time in Infer (HW and SW): 3891.49ms -[ INFO ] Frames in utterance: 855 -[ INFO ] Average Infer time per frame: 4.55ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7071600 -[ INFO ] avg error: 0.0449147 -[ INFO ] avg rms error: 0.0585048 -[ INFO ] stdev error: 0.0374897 -[ INFO ] -[ INFO ] Utterance 5: -[ INFO ] Total time in Infer (HW and SW): 3378.61ms -[ INFO ] Frames in utterance: 699 -[ INFO ] Average Infer time per frame: 4.83ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.8870468 -[ INFO ] avg error: 0.0479243 -[ INFO ] avg rms error: 0.0625490 -[ INFO ] stdev error: 0.0401951 -[ INFO ] -[ INFO ] Utterance 6: -[ INFO ] Total time in Infer (HW and SW): 4034.31ms -[ INFO ] Frames in utterance: 790 -[ INFO ] Average Infer time per frame: 5.11ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7648273 -[ INFO ] avg error: 0.0482702 -[ INFO ] avg rms error: 0.0629734 -[ INFO ] stdev error: 0.0404429 -[ INFO ] -[ INFO ] Utterance 7: -[ INFO ] Total time in Infer (HW and SW): 2854.04ms -[ INFO ] Frames in utterance: 622 -[ INFO ] Average Infer time per frame: 4.59ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.7389560 -[ INFO ] avg error: 0.0465543 -[ INFO ] avg rms error: 0.0604941 -[ INFO ] stdev error: 0.0386294 -[ INFO ] -[ INFO ] Utterance 8: -[ INFO ] Total time in Infer (HW and SW): 2493.28ms -[ INFO ] Frames in utterance: 548 -[ INFO ] Average Infer time per frame: 4.55ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.6680136 -[ INFO ] avg error: 0.0439341 -[ INFO ] avg rms error: 0.0574614 -[ INFO ] stdev error: 0.0370353 -[ INFO ] -[ INFO ] Utterance 9: -[ INFO ] Total time in Infer (HW and SW): 1654.67ms -[ INFO ] Frames in utterance: 368 -[ INFO ] Average Infer time per frame: 4.50ms -[ INFO ] -[ INFO ] Output blob name: affinetransform14 -[ INFO ] Number scores per frame: 3425 -[ INFO ] -[ INFO ] max error: 0.6550579 -[ INFO ] avg error: 0.0467643 -[ INFO ] avg rms error: 0.0605045 -[ INFO ] stdev error: 0.0383914 -[ INFO ] -[ INFO ] Total sample time: 39722.60ms -[ INFO ] File result.npz was created! -[ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool -``` - -## See Also - -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) -- [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) -- [Model Downloader](@ref omz_tools_downloader) -- [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) - -[openvino.runtime.Core.import_model]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Core.html#openvino.runtime.Core.import_model -[openvino.runtime.CompiledModel.export_model]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.CompiledModel.html#openvino.runtime.CompiledModel.export_model -[openvino.runtime.Model.add_outputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.Model.html#openvino.runtime.Model.add_outputs -[openvino.runtime.set_batch]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.html#openvino.runtime.set_batch -[openvino.runtime.CompiledModel.inputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.CompiledModel.html#openvino.runtime.CompiledModel.inputs -[openvino.runtime.CompiledModel.outputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.CompiledModel.html#openvino.runtime.CompiledModel.outputs -[openvino.runtime.ConstOutput.any_name]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.ConstOutput.html#openvino.runtime.ConstOutput.any_name -[openvino.runtime.CompiledModel.create_infer_request]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.CompiledModel.html#openvino.runtime.CompiledModel.create_infer_request -[openvino.runtime.InferRequest.infer]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.infer -[openvino.runtime.InferRequest.model_outputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.model_outputs -[openvino.runtime.InferRequest.model_inputs]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.model_inputs -[openvino.runtime.InferRequest.query_state]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.query_state -[openvino.runtime.VariableState.reset]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.inference_engine.VariableState.html#openvino.inference_engine.VariableState.reset -[openvino.runtime.InferRequest.profiling_info]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.profiling_info -[openvino.runtime.ProfilingInfo.real_time]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.ProfilingInfo.html#openvino.runtime.ProfilingInfo.real_time -[openvino.runtime.InferRequest.get_input_tensor]:https://docs.openvino.ai/2022.3/api/ie_python_api/_autosummary/openvino.runtime.InferRequest.html#openvino.runtime.InferRequest.get_input_tensor +.. code-block:: sh + + [ INFO ] Creating OpenVINO Runtime Core + [ INFO ] Reading the model: /models/wsj_dnn5b_smbr_fp32.xml + [ INFO ] Using scale factor(s) calculated from first utterance + [ INFO ] For input 0 using scale factor of 2175.4322418 + [ INFO ] Loading the model to the plugin + [ INFO ] Starting inference in synchronous mode + [ INFO ] + [ INFO ] Utterance 0: + [ INFO ] Total time in Infer (HW and SW): 6326.06ms + [ INFO ] Frames in utterance: 1294 + [ INFO ] Average Infer time per frame: 4.89ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7051840 + [ INFO ] avg error: 0.0448388 + [ INFO ] avg rms error: 0.0582387 + [ INFO ] stdev error: 0.0371650 + [ INFO ] + [ INFO ] Utterance 1: + [ INFO ] Total time in Infer (HW and SW): 4526.57ms + [ INFO ] Frames in utterance: 1005 + [ INFO ] Average Infer time per frame: 4.50ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7575974 + [ INFO ] avg error: 0.0452166 + [ INFO ] avg rms error: 0.0586013 + [ INFO ] stdev error: 0.0372769 + [ INFO ] + [ INFO ] Utterance 2: + [ INFO ] Total time in Infer (HW and SW): 6636.56ms + [ INFO ] Frames in utterance: 1471 + [ INFO ] Average Infer time per frame: 4.51ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7191710 + [ INFO ] avg error: 0.0472226 + [ INFO ] avg rms error: 0.0612991 + [ INFO ] stdev error: 0.0390846 + [ INFO ] + [ INFO ] Utterance 3: + [ INFO ] Total time in Infer (HW and SW): 3927.01ms + [ INFO ] Frames in utterance: 845 + [ INFO ] Average Infer time per frame: 4.65ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7436461 + [ INFO ] avg error: 0.0477581 + [ INFO ] avg rms error: 0.0621334 + [ INFO ] stdev error: 0.0397457 + [ INFO ] + [ INFO ] Utterance 4: + [ INFO ] Total time in Infer (HW and SW): 3891.49ms + [ INFO ] Frames in utterance: 855 + [ INFO ] Average Infer time per frame: 4.55ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7071600 + [ INFO ] avg error: 0.0449147 + [ INFO ] avg rms error: 0.0585048 + [ INFO ] stdev error: 0.0374897 + [ INFO ] + [ INFO ] Utterance 5: + [ INFO ] Total time in Infer (HW and SW): 3378.61ms + [ INFO ] Frames in utterance: 699 + [ INFO ] Average Infer time per frame: 4.83ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.8870468 + [ INFO ] avg error: 0.0479243 + [ INFO ] avg rms error: 0.0625490 + [ INFO ] stdev error: 0.0401951 + [ INFO ] + [ INFO ] Utterance 6: + [ INFO ] Total time in Infer (HW and SW): 4034.31ms + [ INFO ] Frames in utterance: 790 + [ INFO ] Average Infer time per frame: 5.11ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7648273 + [ INFO ] avg error: 0.0482702 + [ INFO ] avg rms error: 0.0629734 + [ INFO ] stdev error: 0.0404429 + [ INFO ] + [ INFO ] Utterance 7: + [ INFO ] Total time in Infer (HW and SW): 2854.04ms + [ INFO ] Frames in utterance: 622 + [ INFO ] Average Infer time per frame: 4.59ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.7389560 + [ INFO ] avg error: 0.0465543 + [ INFO ] avg rms error: 0.0604941 + [ INFO ] stdev error: 0.0386294 + [ INFO ] + [ INFO ] Utterance 8: + [ INFO ] Total time in Infer (HW and SW): 2493.28ms + [ INFO ] Frames in utterance: 548 + [ INFO ] Average Infer time per frame: 4.55ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.6680136 + [ INFO ] avg error: 0.0439341 + [ INFO ] avg rms error: 0.0574614 + [ INFO ] stdev error: 0.0370353 + [ INFO ] + [ INFO ] Utterance 9: + [ INFO ] Total time in Infer (HW and SW): 1654.67ms + [ INFO ] Frames in utterance: 368 + [ INFO ] Average Infer time per frame: 4.50ms + [ INFO ] + [ INFO ] Output blob name: affinetransform14 + [ INFO ] Number scores per frame: 3425 + [ INFO ] + [ INFO ] max error: 0.6550579 + [ INFO ] avg error: 0.0467643 + [ INFO ] avg rms error: 0.0605045 + [ INFO ] stdev error: 0.0383914 + [ INFO ] + [ INFO ] Total sample time: 39722.60ms + [ INFO ] File result.npz was created! + [ INFO ] This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool + + +See Also +######## + +- :doc:`Integrate the OpenVINO™ Runtime with Your Application ` +- :doc:`Using OpenVINO™ Toolkit Samples ` +- :doc:`Model Downloader ` +- :doc:`Model Optimizer ` + +@endsphinxdirective + diff --git a/scripts/setupvars/setupvars.sh b/scripts/setupvars/setupvars.sh index 41789160e69a59..c9bcc61c643051 100755 --- a/scripts/setupvars/setupvars.sh +++ b/scripts/setupvars/setupvars.sh @@ -4,12 +4,12 @@ # SPDX-License-Identifier: Apache-2.0 abs_path () { - path=$(eval echo "$1") - directory=$(dirname "$path") - echo "$(cd "$directory" || exit; pwd -P)/$(basename "$path")"; + script_path=$(eval echo "$1") + directory=$(dirname "$script_path") + echo "$(cd "$directory" || exit; pwd -P)"; } -SCRIPT_DIR="$( cd "$( dirname "$(abs_path "${BASH_SOURCE[0]}")" )" >/dev/null 2>&1 && pwd )" +SCRIPT_DIR="$(abs_path "${BASH_SOURCE[0]}")" >/dev/null 2>&1 INSTALLDIR="${SCRIPT_DIR}" export INTEL_OPENVINO_DIR="$INSTALLDIR" @@ -36,15 +36,15 @@ if [ -e "$INSTALLDIR/runtime" ]; then export OpenVINO_DIR=$INSTALLDIR/runtime/cmake system_type=$(ls "$INSTALLDIR/runtime/lib/") - IE_PLUGINS_PATH=$INSTALLDIR/runtime/lib/$system_type + OV_PLUGINS_PATH=$INSTALLDIR/runtime/lib/$system_type if [[ "$OSTYPE" == "darwin"* ]]; then - export DYLD_LIBRARY_PATH=${IE_PLUGINS_PATH}/Release:${IE_PLUGINS_PATH}/Debug${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH} - export LD_LIBRARY_PATH=${IE_PLUGINS_PATH}/Release:${IE_PLUGINS_PATH}/Debug${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} - export PKG_CONFIG_PATH=${IE_PLUGINS_PATH}/Release/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH} + export DYLD_LIBRARY_PATH=${OV_PLUGINS_PATH}/Release:${OV_PLUGINS_PATH}/Debug${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH} + export LD_LIBRARY_PATH=${OV_PLUGINS_PATH}/Release:${OV_PLUGINS_PATH}/Debug${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} + export PKG_CONFIG_PATH=${OV_PLUGINS_PATH}/Release/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH} else - export LD_LIBRARY_PATH=${IE_PLUGINS_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} - export PKG_CONFIG_PATH=$IE_PLUGINS_PATH/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH} + export LD_LIBRARY_PATH=${OV_PLUGINS_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} + export PKG_CONFIG_PATH=$OV_PLUGINS_PATH/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH} fi if [ -e "$INSTALLDIR/runtime/3rdparty/tbb" ]; then @@ -121,7 +121,12 @@ check_python_version () { "${PYTHON_VERSION_MAJOR}.${MAX_SUPPORTED_PYTHON_VERSION_MINOR} (64-bit) from https://www.python.org/downloads/" return 0 fi - python_bitness=$(python"$python_version" -c 'import sys; print(64 if sys.maxsize > 2**32 else 32)') + if command -v python"$python_version" > /dev/null 2>&1; then + python_interp=python"$python_version" + else + python_interp=python"$python_version_major" + fi + python_bitness=$("$python_interp" -c 'import sys; print(64 if sys.maxsize > 2**32 else 32)') if [ "$python_bitness" != "" ] && [ "$python_bitness" != "64" ] && [ "$OS_NAME" != "Raspbian" ]; then echo "[setupvars.sh] WARNING: 64 bitness for Python $python_version is required" @@ -144,7 +149,7 @@ check_python_version () { echo "[setupvars.sh] WARNING: OpenVINO Python environment does not set properly" fi fi -} +} python_version_to_check="$python_version" if [ -z "$python_version" ]; then diff --git a/src/bindings/c/include/openvino/c/auto/properties.h b/src/bindings/c/include/openvino/c/auto/properties.h new file mode 100644 index 00000000000000..87d920cc131c90 --- /dev/null +++ b/src/bindings/c/include/openvino/c/auto/properties.h @@ -0,0 +1,34 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief This is a specified header file for auto plugin's properties + * + * @file properties.h + */ + +#pragma once +#include "openvino/c/ov_common.h" + +/** + * @brief Read-write property for setting that enables performance improvement by binding + * buffer to hw infer request + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_intel_auto_device_bind_buffer; + +/** + * @brief Read-write property to enable/disable CPU as accelerator (or helper device) at the beginning + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_intel_auto_enable_startup_fallback; + +/** + * @brief Read-write property to enable/disable runtime fallback to other devices when infer fails + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_intel_auto_enable_runtime_fallback; \ No newline at end of file diff --git a/src/bindings/c/include/openvino/c/gpu/gpu_plugin_properties.h b/src/bindings/c/include/openvino/c/gpu/gpu_plugin_properties.h index 9d3aa162ac12be..7ec3b46731b922 100644 --- a/src/bindings/c/include/openvino/c/gpu/gpu_plugin_properties.h +++ b/src/bindings/c/include/openvino/c/gpu/gpu_plugin_properties.h @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2022 Intel Corporation +// Copyright (C) 2018-2023 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // diff --git a/src/bindings/c/include/openvino/c/openvino.h b/src/bindings/c/include/openvino/c/openvino.h index 7ffcb79da6390e..f49a5049f7ed91 100644 --- a/src/bindings/c/include/openvino/c/openvino.h +++ b/src/bindings/c/include/openvino/c/openvino.h @@ -15,6 +15,7 @@ **/ #pragma once +#include "openvino/c/auto/properties.h" #include "openvino/c/ov_common.h" #include "openvino/c/ov_compiled_model.h" #include "openvino/c/ov_core.h" diff --git a/src/bindings/c/include/openvino/c/ov_property.h b/src/bindings/c/include/openvino/c/ov_property.h index b00f72aaedafc2..0f99d2870f620a 100644 --- a/src/bindings/c/include/openvino/c/ov_property.h +++ b/src/bindings/c/include/openvino/c/ov_property.h @@ -119,6 +119,21 @@ ov_property_key_affinity; OPENVINO_C_VAR(const char*) ov_property_key_inference_num_threads; +/** + * @brief Read-write property, it is high-level OpenVINO hint for using CPU pinning to bind CPU threads to processors + * during inference + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_hint_enable_cpu_pinning; + +/** + * @brief Read-write property, it is high-level OpenVINO hint for using hyper threading processors during CPU inference + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_hint_enable_hyper_threading; + /** * @brief Read-write property, it is high-level OpenVINO Performance Hints * @ingroup ov_property_c_api @@ -126,6 +141,13 @@ ov_property_key_inference_num_threads; OPENVINO_C_VAR(const char*) ov_property_key_hint_performance_mode; +/** + * @brief Read-write property, it is high-level OpenVINO Hints for the type of CPU core used during inference + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_hint_scheduling_core_type; + /** * @brief Read-write property to set the hint for device to use specified precision for inference. * @ingroup ov_property_c_api @@ -171,3 +193,30 @@ ov_property_key_enable_profiling; */ OPENVINO_C_VAR(const char*) ov_property_key_device_priorities; + +/** + * @brief Read-write property for high-level OpenVINO Execution hint + * unlike low-level properties that are individual (per-device), the hints are something that every device accepts + * and turns into device-specific settings + * Execution mode hint controls preferred optimization targets (performance or accuracy) for given model + * It can be set to be below value: + * "PERFORMANCE", //!< Optimize for max performance + * "ACCURACY", //!< Optimize for max accuracy + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_hint_execution_mode; + +/** + * @brief Read-write property to set whether force terminate tbb when ov core destruction + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_force_tbb_terminate; + +/** + * @brief Read-write property to configure `mmap()` use for model read + * @ingroup ov_property_c_api + */ +OPENVINO_C_VAR(const char*) +ov_property_key_enable_mmap; diff --git a/src/bindings/c/src/ov_auto_property.cpp b/src/bindings/c/src/ov_auto_property.cpp new file mode 100644 index 00000000000000..c8900e87c7a437 --- /dev/null +++ b/src/bindings/c/src/ov_auto_property.cpp @@ -0,0 +1,10 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/c/auto/properties.h" + +// Read-write property key +const char* ov_property_key_intel_auto_device_bind_buffer = "DEVICE_BIND_BUFFER"; +const char* ov_property_key_intel_auto_enable_startup_fallback = "ENABLE_STARTUP_FALLBACK"; +const char* ov_property_key_intel_auto_enable_runtime_fallback = "ENABLE_RUNTIME_FALLBACK"; diff --git a/src/bindings/c/src/ov_property.cpp b/src/bindings/c/src/ov_property.cpp index 613d52b376a228..78b3bfe4576ff9 100644 --- a/src/bindings/c/src/ov_property.cpp +++ b/src/bindings/c/src/ov_property.cpp @@ -23,9 +23,15 @@ const char* ov_property_key_num_streams = "NUM_STREAMS"; const char* ov_property_key_affinity = "AFFINITY"; const char* ov_property_key_inference_num_threads = "INFERENCE_NUM_THREADS"; const char* ov_property_key_hint_performance_mode = "PERFORMANCE_HINT"; +const char* ov_property_key_hint_enable_cpu_pinning = "ENABLE_CPU_PINNING"; +const char* ov_property_key_hint_scheduling_core_type = "SCHEDULING_CORE_TYPE"; +const char* ov_property_key_hint_enable_hyper_threading = "ENABLE_HYPER_THREADING"; const char* ov_property_key_hint_inference_precision = "INFERENCE_PRECISION_HINT"; const char* ov_property_key_hint_num_requests = "PERFORMANCE_HINT_NUM_REQUESTS"; const char* ov_property_key_hint_model_priority = "MODEL_PRIORITY"; const char* ov_property_key_log_level = "LOG_LEVEL"; const char* ov_property_key_enable_profiling = "PERF_COUNT"; const char* ov_property_key_device_priorities = "MULTI_DEVICE_PRIORITIES"; +const char* ov_property_key_hint_execution_mode = "EXECUTION_MODE_HINT"; +const char* ov_property_key_force_tbb_terminate = "FORCE_TBB_TERMINATE"; +const char* ov_property_key_enable_mmap = "ENABLE_MMAP"; diff --git a/src/bindings/c/tests/ie_c_api_test.cpp b/src/bindings/c/tests/ie_c_api_test.cpp index f4d527d6180b2f..013001b85c2aec 100644 --- a/src/bindings/c/tests/ie_c_api_test.cpp +++ b/src/bindings/c/tests/ie_c_api_test.cpp @@ -954,36 +954,6 @@ TEST_P(ie_c_api_test, ie_exec_network_get_config) { ie_core_free(&core); } -TEST_P(ie_c_api_test, ie_exec_network_set_config) { - ie_core_t *core = nullptr; - IE_ASSERT_OK(ie_core_create("", &core)); - ASSERT_NE(nullptr, core); - - ie_param_t param; - if (ie_core_get_metric(core, "GPU", "AVAILABLE_DEVICES", ¶m) != IEStatusCode::OK) { - ie_core_free(&core); - GTEST_SKIP(); - } - - ie_network_t *network = nullptr; - IE_EXPECT_OK(ie_core_read_network(core, xml_file_name.c_str(), bin_file_name.c_str(), &network)); - EXPECT_NE(nullptr, network); - - const char *device_name = "MULTI:GPU,CPU"; - ie_config_t config = {nullptr, nullptr, nullptr}; - ie_executable_network_t *exe_network = nullptr; - IE_EXPECT_OK(ie_core_load_network(core, network, device_name, &config, &exe_network)); - EXPECT_NE(nullptr, exe_network); - - ie_config_t config_param = {"MULTI_DEVICE_PRIORITIES", "GPU,CPU", nullptr}; - IE_EXPECT_OK(ie_exec_network_set_config(exe_network, &config_param)); - - ie_exec_network_free(&exe_network); - ie_network_free(&network); - ie_core_free(&core); - ie_param_free(¶m); -} - TEST_P(ie_c_api_test, ie_exec_network_get_metric) { ie_core_t *core = nullptr; IE_ASSERT_OK(ie_core_create("", &core)); diff --git a/src/bindings/c/tests/ov_auto_property_test.cpp b/src/bindings/c/tests/ov_auto_property_test.cpp new file mode 100644 index 00000000000000..6750092d845c93 --- /dev/null +++ b/src/bindings/c/tests/ov_auto_property_test.cpp @@ -0,0 +1,53 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ov_test.hpp" + +using test_params = std::tuple; + +class ov_auto_plugin_test : public ::testing::TestWithParam { +public: + std::string device_name; + const char* auto_property; + const char* property_value; + bool invalid_value; + +public: + void SetUp() override { + std::tie(device_name, auto_property, property_value, invalid_value) = GetParam(); + } +}; + +TEST_P(ov_auto_plugin_test, ov_core_auto_set_and_get_property_bool) { + ov_core_t* core = nullptr; + OV_EXPECT_OK(ov_core_create(&core)); + EXPECT_NE(nullptr, core); + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), auto_property, property_value)); + char* ret = nullptr; + if (invalid_value) { + OV_EXPECT_NOT_OK(ov_core_get_property(core, device_name.c_str(), auto_property, &ret)); + EXPECT_STRNE(property_value, ret); + } else { + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), auto_property, &ret)); + EXPECT_STREQ(property_value, ret); + } + ov_free(ret); + ov_core_free(core); +} + +const std::vector test_property_config = { + test_params{"AUTO", ov_property_key_intel_auto_device_bind_buffer, "YES", false}, + test_params{"AUTO", ov_property_key_intel_auto_device_bind_buffer, "NO", false}, + test_params{"AUTO", ov_property_key_intel_auto_device_bind_buffer, "TEST", true}, + test_params{"AUTO", ov_property_key_intel_auto_enable_startup_fallback, "YES", false}, + test_params{"AUTO", ov_property_key_intel_auto_enable_startup_fallback, "NO", false}, + test_params{"AUTO", ov_property_key_intel_auto_enable_startup_fallback, "TEST", true}, + test_params{"AUTO", ov_property_key_intel_auto_enable_runtime_fallback, "YES", false}, + test_params{"AUTO", ov_property_key_intel_auto_enable_runtime_fallback, "NO", false}, + test_params{"AUTO", ov_property_key_intel_auto_enable_runtime_fallback, "TEST", true}, +}; + +INSTANTIATE_TEST_SUITE_P(ov_auto_plugin_test_properties, + ov_auto_plugin_test, + ::testing::ValuesIn(test_property_config)); \ No newline at end of file diff --git a/src/bindings/c/tests/ov_compiled_model_test.cpp b/src/bindings/c/tests/ov_compiled_model_test.cpp index 0ff98f5ba2dff4..c1b6ed33cadac6 100644 --- a/src/bindings/c/tests/ov_compiled_model_test.cpp +++ b/src/bindings/c/tests/ov_compiled_model_test.cpp @@ -121,45 +121,6 @@ TEST_P(ov_compiled_model_test, ov_compiled_model_input_by_name) { ov_core_free(core); } -TEST_P(ov_compiled_model_test, set_and_get_property) { - // It seems that all set_property() for CPU plugin are not implement in compiled_model. - auto device_name = "MULTI:GPU,CPU"; - ov_core_t* core = nullptr; - OV_EXPECT_OK(ov_core_create(&core)); - EXPECT_NE(nullptr, core); - - char* info = nullptr; - const char* key_0 = ov_property_key_available_devices; - if (ov_core_get_property(core, "GPU", key_0, &info) != ov_status_e::OK) { - ov_core_free(core); - GTEST_SKIP(); - } - - ov_model_t* model = nullptr; - OV_EXPECT_OK(ov_core_read_model(core, xml_file_name.c_str(), bin_file_name.c_str(), &model)); - EXPECT_NE(nullptr, model); - - ov_compiled_model_t* compiled_model = nullptr; - OV_EXPECT_OK(ov_core_compile_model(core, model, device_name, 0, &compiled_model)); - EXPECT_NE(nullptr, compiled_model); - - const char* key_1 = ov_property_key_device_priorities; - const char* value_1 = "GPU,CPU"; - OV_EXPECT_OK(ov_compiled_model_set_property(compiled_model, key_1, value_1)); - char* result = nullptr; - OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_1, &result)); - EXPECT_STREQ(value_1, result); - ov_free(result); - - const char* key_2 = ov_property_key_supported_properties; - OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_2, &result)); - ov_free(result); - - ov_compiled_model_free(compiled_model); - ov_model_free(model); - ov_core_free(core); -} - TEST_P(ov_compiled_model_test, get_property) { auto device_name = GetParam(); ov_core_t* core = nullptr; diff --git a/src/bindings/c/tests/ov_core_test.cpp b/src/bindings/c/tests/ov_core_test.cpp index 6804504c94053f..621f35dcf969e1 100644 --- a/src/bindings/c/tests/ov_core_test.cpp +++ b/src/bindings/c/tests/ov_core_test.cpp @@ -133,13 +133,13 @@ TEST_P(ov_core_test, ov_core_compile_model_with_property) { ov_compiled_model_t* compiled_model = nullptr; const char* key = ov_property_key_num_streams; - const char* num = "11"; + const char* num = "2"; OV_EXPECT_OK(ov_core_compile_model(core, model, device_name.c_str(), 2, &compiled_model, key, num)); EXPECT_NE(nullptr, compiled_model); char* property_value = nullptr; OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key, &property_value)); - EXPECT_STREQ(property_value, "11"); + EXPECT_STREQ(property_value, "2"); ov_free(property_value); ov_compiled_model_free(compiled_model); @@ -147,6 +147,27 @@ TEST_P(ov_core_test, ov_core_compile_model_with_property) { ov_core_free(core); } +TEST_P(ov_core_test, ov_core_compile_model_with_excution_mode) { + std::string device_name = "AUTO"; + ov_core_t* core = nullptr; + OV_EXPECT_OK(ov_core_create(&core)); + EXPECT_NE(nullptr, core); + + ov_model_t* model = nullptr; + OV_EXPECT_OK(ov_core_read_model(core, xml_file_name.c_str(), nullptr, &model)); + EXPECT_NE(nullptr, model); + + ov_compiled_model_t* compiled_model = nullptr; + const char* key = ov_property_key_hint_execution_mode; + const char* value = "PERFORMANCE"; + OV_EXPECT_OK(ov_core_compile_model(core, model, device_name.c_str(), 2, &compiled_model, key, value)); + EXPECT_NE(nullptr, compiled_model); + + ov_compiled_model_free(compiled_model); + ov_model_free(model); + ov_core_free(core); +} + TEST_P(ov_core_test, ov_core_compile_model_with_property_invalid) { auto device_name = GetParam(); ov_core_t* core = nullptr; @@ -238,6 +259,50 @@ TEST_P(ov_core_test, ov_core_set_property_enum_invalid) { OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &ret)); EXPECT_STRNE(invalid_mode, ret); ov_free(ret); + + const char* key_pin = ov_property_key_hint_enable_cpu_pinning; + const char* val_pin = "YES"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key_pin, val_pin)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_pin, &ret)); + EXPECT_STREQ(val_pin, ret); + ov_free(ret); + + const char* invalid_val = "INVALID_VAL"; + OV_EXPECT_NOT_OK(ov_core_set_property(core, device_name.c_str(), key_pin, invalid_val)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_pin, &ret)); + EXPECT_STRNE(invalid_val, ret); + ov_free(ret); + + const char* key_type = ov_property_key_hint_scheduling_core_type; + const char* val_type = "PCORE_ONLY"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key_type, val_type)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_type, &ret)); + EXPECT_STREQ(val_type, ret); + ov_free(ret); + + OV_EXPECT_NOT_OK(ov_core_set_property(core, device_name.c_str(), key_type, invalid_val)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_type, &ret)); + EXPECT_STRNE(invalid_val, ret); + ov_free(ret); + + const char* key_ht = ov_property_key_hint_enable_hyper_threading; + const char* val_ht = "YES"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key_ht, val_ht)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_ht, &ret)); + EXPECT_STREQ(val_ht, ret); + ov_free(ret); + + OV_EXPECT_NOT_OK(ov_core_set_property(core, device_name.c_str(), key_ht, invalid_val)); + ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key_ht, &ret)); + EXPECT_STRNE(invalid_val, ret); + ov_free(ret); + ov_core_free(core); } @@ -247,31 +312,38 @@ TEST_P(ov_core_test, ov_core_set_and_get_property_enum) { OV_EXPECT_OK(ov_core_create(&core)); EXPECT_NE(nullptr, core); - const char* key = ov_property_key_hint_performance_mode; - const char* affinity = "LATENCY"; - OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key, affinity)); - char* ret = nullptr; - OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &ret)); - EXPECT_STREQ(affinity, ret); - ov_free(ret); + std::map properties = {{ov_property_key_hint_performance_mode, "LATENCY"}, + {ov_property_key_hint_scheduling_core_type, "PCORE_ONLY"}, + {ov_property_key_hint_enable_hyper_threading, "YES"}, + {ov_property_key_enable_profiling, "YES"}}; + + for (const auto& property : properties) { + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), property.first, property.second)); + char* ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), property.first, &ret)); + EXPECT_STREQ(property.second, ret); + ov_free(ret); + } ov_core_free(core); } -TEST_P(ov_core_test, ov_core_set_and_get_property_bool) { - auto device_name = GetParam(); +TEST_F(ov_core_test, ov_core_set_and_get_property_no_device) { ov_core_t* core = nullptr; OV_EXPECT_OK(ov_core_create(&core)); EXPECT_NE(nullptr, core); - const char* key = ov_property_key_enable_profiling; - const char* enable = "YES"; - OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key, enable)); + std::map properties = {{ov_property_key_force_tbb_terminate, "YES"}, + {ov_property_key_enable_mmap, "NO"}}; + + for (const auto& property : properties) { + OV_EXPECT_OK(ov_core_set_property(core, "", property.first, property.second)); + char* ret = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, "", property.first, &ret)); + EXPECT_STREQ(property.second, ret); + ov_free(ret); + } - char* ret = nullptr; - OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &ret)); - EXPECT_STREQ(enable, ret); - ov_free(ret); ov_core_free(core); } @@ -306,6 +378,30 @@ TEST_P(ov_core_test, ov_core_get_property) { ov_core_free(core); } +TEST_P(ov_core_test, ov_core_set_and_get_property_execution_mode) { + std::string device_name = "AUTO"; + ov_core_t* core = nullptr; + OV_EXPECT_OK(ov_core_create(&core)); + EXPECT_NE(nullptr, core); + + const char* key = ov_property_key_hint_execution_mode; + char* property_value = nullptr; + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &property_value)); + ov_free(property_value); + + const char* value1 = "ACCURACY"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key, value1)); + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &property_value)); + EXPECT_STREQ(value1, property_value); + + const char* value2 = "PERFORMANCE"; + OV_EXPECT_OK(ov_core_set_property(core, device_name.c_str(), key, value2)); + OV_EXPECT_OK(ov_core_get_property(core, device_name.c_str(), key, &property_value)); + EXPECT_STREQ(value2, property_value); + + ov_core_free(core); +} + TEST_P(ov_core_test, ov_core_set_get_property_str) { #ifdef __aarch64__ GTEST_SKIP() << "Skip this test for ARM CPU for now, cause no string property supported"; diff --git a/src/bindings/c/tests/ov_infer_request_test.cpp b/src/bindings/c/tests/ov_infer_request_test.cpp index 7c1ccd7e2688f7..c2ff469f34c2cc 100644 --- a/src/bindings/c/tests/ov_infer_request_test.cpp +++ b/src/bindings/c/tests/ov_infer_request_test.cpp @@ -164,6 +164,7 @@ class ov_infer_request_ppp : public ov_capi_test_base { OV_EXPECT_OK(ov_preprocess_input_model_info_set_layout(input_model, model_layout)); ov_layout_free(model_layout); + ov_model_free(model); // clean before assigning built model OV_EXPECT_OK(ov_preprocess_prepostprocessor_build(preprocess, &model)); EXPECT_NE(nullptr, model); diff --git a/src/bindings/python/constraints.txt b/src/bindings/python/constraints.txt new file mode 100644 index 00000000000000..8586435dc593e9 --- /dev/null +++ b/src/bindings/python/constraints.txt @@ -0,0 +1,22 @@ +# used in multiple components +onnx==1.13.1 # Python bindings, ONNX Frontend + +# pytest +pytest==7.2.0; python_version >= '3.10' +pytest>=5.0,<=7.0.1; python_version < '3.10' +pytest-dependency==0.5.1 +pytest-forked; platform_system != "Windows" +pytest-html==1.19.0 +pytest-timeout==2.0.1 + +# Python bindings +py>=1.9.0 +pygments>=2.8.1 +setuptools>=53.0.0 +wheel>=0.38.1 + +# Frontends +docopt~=0.6.2 +paddlepaddle==2.4.2 +tensorflow>=1.15.5,<2.13.0 +six~=1.16.0 \ No newline at end of file diff --git a/src/bindings/python/docs/code_examples.md b/src/bindings/python/docs/code_examples.md index 9194a8c5b38634..2c18ddf99cf3ac 100644 --- a/src/bindings/python/docs/code_examples.md +++ b/src/bindings/python/docs/code_examples.md @@ -9,7 +9,7 @@ Instructions can be found in ["Building the OpenVINO™ Python API"](./build.md) General guide: * Snake case (also known as the *lower_case_with_underscores* style) is used across the codebase. That includes modules (`runtime`, `offline_transformations`), function names, and arguments/variables (`async_infer`, `wait`, `path_to_xml`). * Naming of classes is an exception to the above rule. The *CamelCase* style is used in this case, for example: `Core`, `InferRequest` or `AsyncInferQueue`. -* If bindings (explained later in the [Pure pybind11 solution](#Pure pybind11 solution) section) are created to expose existing C++ code, make them similar to their C++ counterparts, regarding both names and placement, for example, C++'s `ov::InferRequest` and Python's `openvino.runtime.InferRequest`. If alignment is not possible, try to describe your class/function/module as well as possible, such as the pair of `openvino.runtime.ConstOutput/openvino.runtime.Output` which relates to `ov::Output/ov::Output`. This naming points out the functional difference between both classes - one is an immutable and the other a mutable version. +* If bindings (explained later in the [Pure pybind11 solution](#pure-pybind11-solution) section) are created to expose existing C++ code, make them similar to their C++ counterparts, regarding both names and placement, for example, C++'s `ov::InferRequest` and Python's `openvino.runtime.InferRequest`. If alignment is not possible, try to describe your class/function/module as well as possible, such as the pair of `openvino.runtime.ConstOutput/openvino.runtime.Output` which relates to `ov::Output/ov::Output`. This naming points out the functional difference between both classes - one is an immutable and the other a mutable version. #### Pure Python solution diff --git a/src/bindings/python/requirements.txt b/src/bindings/python/requirements.txt index e83f59eb8b3ae7..968d95b8760bed 100644 --- a/src/bindings/python/requirements.txt +++ b/src/bindings/python/requirements.txt @@ -1 +1,2 @@ numpy>=1.16.6 +singledispatchmethod; python_version<'3.8' diff --git a/src/bindings/python/requirements_test.txt b/src/bindings/python/requirements_test.txt index 530a28b3bf1e16..e056fa13852952 100644 --- a/src/bindings/python/requirements_test.txt +++ b/src/bindings/python/requirements_test.txt @@ -1,3 +1,7 @@ +-c ./constraints.txt +numpy>=1.16.6,<=1.23.4 +protobuf~=3.20.3 +onnx bandit black flake8 @@ -14,8 +18,7 @@ flake8-expression-complexity flake8-print flake8-pytest-style flake8-rst-docstrings -# Force version of pygments from flake8-rst-docstrings -pygments>=2.8.1 +pygments flake8-string-format flake8-variables-names flake8_builtins @@ -24,19 +27,16 @@ flake8_commas flake8_pep3101 flake8_quotes mypy -onnx==1.12.0 Pep8-naming pydocstyle -pytest-forked; platform_system != "Windows" +pytest-forked pytest-xdist -pytest>=5.0,<=7.0.1; python_version < '3.10' -pytest==7.2.1; python_version >= '3.10' -pytest-html==1.19.0 -py>=1.9.0 +pytest-html +pytest +py radon retrying tox types-pkg_resources -wheel>=0.38.1 -protobuf~=3.18.1 -numpy>=1.16.6,<=1.23.4 +wheel +singledispatchmethod diff --git a/src/bindings/python/setup.py b/src/bindings/python/setup.py index 021e37875e32da..6205f70aeb4e04 100644 --- a/src/bindings/python/setup.py +++ b/src/bindings/python/setup.py @@ -55,6 +55,7 @@ "openvino.runtime.opset8", "openvino.runtime.opset9", "openvino.runtime.opset10", + "openvino.runtime.opset11", "openvino.runtime.utils", "openvino.runtime.op", "openvino.runtime.op.util", diff --git a/src/bindings/python/src/compatibility/ngraph/__init__.py b/src/bindings/python/src/compatibility/ngraph/__init__.py index 3109d9e3d1d525..d80f2199dfc5d0 100644 --- a/src/bindings/python/src/compatibility/ngraph/__init__.py +++ b/src/bindings/python/src/compatibility/ngraph/__init__.py @@ -18,180 +18,180 @@ from ngraph.impl import PartialShape from ngraph.helpers import function_from_cnn from ngraph.helpers import function_to_cnn -from ngraph.opset10 import absolute -from ngraph.opset10 import absolute as abs -from ngraph.opset10 import acos -from ngraph.opset10 import acosh -from ngraph.opset10 import adaptive_avg_pool -from ngraph.opset10 import adaptive_max_pool -from ngraph.opset10 import add -from ngraph.opset10 import asin -from ngraph.opset10 import asinh -from ngraph.opset10 import assign -from ngraph.opset10 import atan -from ngraph.opset10 import atanh -from ngraph.opset10 import avg_pool -from ngraph.opset10 import batch_norm_inference -from ngraph.opset10 import batch_to_space -from ngraph.opset10 import binary_convolution -from ngraph.opset10 import broadcast -from ngraph.opset10 import bucketize -from ngraph.opset10 import ceiling -from ngraph.opset10 import ceiling as ceil -from ngraph.opset10 import clamp -from ngraph.opset10 import concat -from ngraph.opset10 import constant -from ngraph.opset10 import convert -from ngraph.opset10 import convert_like -from ngraph.opset10 import convolution -from ngraph.opset10 import convolution_backprop_data -from ngraph.opset10 import cos -from ngraph.opset10 import cosh -from ngraph.opset10 import ctc_greedy_decoder -from ngraph.opset10 import ctc_greedy_decoder_seq_len -from ngraph.opset10 import ctc_loss -from ngraph.opset10 import cum_sum -from ngraph.opset10 import cum_sum as cumsum -from ngraph.opset10 import deformable_convolution -from ngraph.opset10 import deformable_psroi_pooling -from ngraph.opset10 import depth_to_space -from ngraph.opset10 import detection_output -from ngraph.opset10 import dft -from ngraph.opset10 import divide -from ngraph.opset10 import einsum -from ngraph.opset10 import elu -from ngraph.opset10 import embedding_bag_offsets_sum -from ngraph.opset10 import embedding_bag_packed_sum -from ngraph.opset10 import embedding_segments_sum -from ngraph.opset10 import extract_image_patches -from ngraph.opset10 import equal -from ngraph.opset10 import erf -from ngraph.opset10 import exp -from ngraph.opset10 import eye -from ngraph.opset10 import fake_quantize -from ngraph.opset10 import floor -from ngraph.opset10 import floor_mod -from ngraph.opset10 import gather -from ngraph.opset10 import gather_elements -from ngraph.opset10 import gather_nd -from ngraph.opset10 import gather_tree -from ngraph.opset10 import gelu -from ngraph.opset10 import generate_proposals -from ngraph.opset10 import greater -from ngraph.opset10 import greater_equal -from ngraph.opset10 import grid_sample -from ngraph.opset10 import grn -from ngraph.opset10 import group_convolution -from ngraph.opset10 import group_convolution_backprop_data -from ngraph.opset10 import gru_cell -from ngraph.opset10 import gru_sequence -from ngraph.opset10 import hard_sigmoid -from ngraph.opset10 import hsigmoid -from ngraph.opset10 import hswish -from ngraph.opset10 import idft -from ngraph.opset10 import if_op -from ngraph.opset10 import interpolate -from ngraph.opset10 import irdft -from ngraph.opset10 import is_finite -from ngraph.opset10 import is_inf -from ngraph.opset10 import is_nan -from ngraph.opset10 import i420_to_bgr -from ngraph.opset10 import i420_to_rgb -from ngraph.opset10 import less -from ngraph.opset10 import less_equal -from ngraph.opset10 import log -from ngraph.opset10 import logical_and -from ngraph.opset10 import logical_not -from ngraph.opset10 import logical_or -from ngraph.opset10 import logical_xor -from ngraph.opset10 import log_softmax -from ngraph.opset10 import loop -from ngraph.opset10 import lrn -from ngraph.opset10 import lstm_cell -from ngraph.opset10 import lstm_sequence -from ngraph.opset10 import matmul -from ngraph.opset10 import matrix_nms -from ngraph.opset10 import max_pool -from ngraph.opset10 import maximum -from ngraph.opset10 import minimum -from ngraph.opset10 import mish -from ngraph.opset10 import mod -from ngraph.opset10 import multiclass_nms -from ngraph.opset10 import multiply -from ngraph.opset10 import mvn -from ngraph.opset10 import negative -from ngraph.opset10 import non_max_suppression -from ngraph.opset10 import non_zero -from ngraph.opset10 import normalize_l2 -from ngraph.opset10 import not_equal -from ngraph.opset10 import nv12_to_bgr -from ngraph.opset10 import nv12_to_rgb -from ngraph.opset10 import one_hot -from ngraph.opset10 import pad -from ngraph.opset10 import parameter -from ngraph.opset10 import power -from ngraph.opset10 import prelu -from ngraph.opset10 import prior_box -from ngraph.opset10 import prior_box_clustered -from ngraph.opset10 import psroi_pooling -from ngraph.opset10 import proposal -from ngraph.opset10 import random_uniform -from ngraph.opset10 import range -from ngraph.opset10 import rdft -from ngraph.opset10 import read_value -from ngraph.opset10 import reduce_l1 -from ngraph.opset10 import reduce_l2 -from ngraph.opset10 import reduce_logical_and -from ngraph.opset10 import reduce_logical_or -from ngraph.opset10 import reduce_max -from ngraph.opset10 import reduce_mean -from ngraph.opset10 import reduce_min -from ngraph.opset10 import reduce_prod -from ngraph.opset10 import reduce_sum -from ngraph.opset10 import region_yolo -from ngraph.opset10 import reorg_yolo -from ngraph.opset10 import relu -from ngraph.opset10 import reshape -from ngraph.opset10 import result -from ngraph.opset10 import reverse_sequence -from ngraph.opset10 import rnn_cell -from ngraph.opset10 import rnn_sequence -from ngraph.opset10 import roi_align -from ngraph.opset10 import roi_pooling -from ngraph.opset10 import roll -from ngraph.opset10 import round -from ngraph.opset10 import scatter_elements_update -from ngraph.opset10 import scatter_update -from ngraph.opset10 import select -from ngraph.opset10 import selu -from ngraph.opset10 import shape_of -from ngraph.opset10 import shuffle_channels -from ngraph.opset10 import sigmoid -from ngraph.opset10 import sign -from ngraph.opset10 import sin -from ngraph.opset10 import sinh -from ngraph.opset10 import slice -from ngraph.opset10 import softmax -from ngraph.opset10 import softplus -from ngraph.opset10 import softsign -from ngraph.opset10 import space_to_batch -from ngraph.opset10 import space_to_depth -from ngraph.opset10 import split -from ngraph.opset10 import sqrt -from ngraph.opset10 import squared_difference -from ngraph.opset10 import squeeze -from ngraph.opset10 import strided_slice -from ngraph.opset10 import subtract -from ngraph.opset10 import swish -from ngraph.opset10 import tan -from ngraph.opset10 import tanh -from ngraph.opset10 import tensor_iterator -from ngraph.opset10 import tile -from ngraph.opset10 import topk -from ngraph.opset10 import transpose -from ngraph.opset10 import unique -from ngraph.opset10 import unsqueeze -from ngraph.opset10 import variadic_split +from ngraph.opset11 import absolute +from ngraph.opset11 import absolute as abs +from ngraph.opset11 import acos +from ngraph.opset11 import acosh +from ngraph.opset11 import adaptive_avg_pool +from ngraph.opset11 import adaptive_max_pool +from ngraph.opset11 import add +from ngraph.opset11 import asin +from ngraph.opset11 import asinh +from ngraph.opset11 import assign +from ngraph.opset11 import atan +from ngraph.opset11 import atanh +from ngraph.opset11 import avg_pool +from ngraph.opset11 import batch_norm_inference +from ngraph.opset11 import batch_to_space +from ngraph.opset11 import binary_convolution +from ngraph.opset11 import broadcast +from ngraph.opset11 import bucketize +from ngraph.opset11 import ceiling +from ngraph.opset11 import ceiling as ceil +from ngraph.opset11 import clamp +from ngraph.opset11 import concat +from ngraph.opset11 import constant +from ngraph.opset11 import convert +from ngraph.opset11 import convert_like +from ngraph.opset11 import convolution +from ngraph.opset11 import convolution_backprop_data +from ngraph.opset11 import cos +from ngraph.opset11 import cosh +from ngraph.opset11 import ctc_greedy_decoder +from ngraph.opset11 import ctc_greedy_decoder_seq_len +from ngraph.opset11 import ctc_loss +from ngraph.opset11 import cum_sum +from ngraph.opset11 import cum_sum as cumsum +from ngraph.opset11 import deformable_convolution +from ngraph.opset11 import deformable_psroi_pooling +from ngraph.opset11 import depth_to_space +from ngraph.opset11 import detection_output +from ngraph.opset11 import dft +from ngraph.opset11 import divide +from ngraph.opset11 import einsum +from ngraph.opset11 import elu +from ngraph.opset11 import embedding_bag_offsets_sum +from ngraph.opset11 import embedding_bag_packed_sum +from ngraph.opset11 import embedding_segments_sum +from ngraph.opset11 import extract_image_patches +from ngraph.opset11 import equal +from ngraph.opset11 import erf +from ngraph.opset11 import exp +from ngraph.opset11 import eye +from ngraph.opset11 import fake_quantize +from ngraph.opset11 import floor +from ngraph.opset11 import floor_mod +from ngraph.opset11 import gather +from ngraph.opset11 import gather_elements +from ngraph.opset11 import gather_nd +from ngraph.opset11 import gather_tree +from ngraph.opset11 import gelu +from ngraph.opset11 import generate_proposals +from ngraph.opset11 import greater +from ngraph.opset11 import greater_equal +from ngraph.opset11 import grid_sample +from ngraph.opset11 import grn +from ngraph.opset11 import group_convolution +from ngraph.opset11 import group_convolution_backprop_data +from ngraph.opset11 import gru_cell +from ngraph.opset11 import gru_sequence +from ngraph.opset11 import hard_sigmoid +from ngraph.opset11 import hsigmoid +from ngraph.opset11 import hswish +from ngraph.opset11 import idft +from ngraph.opset11 import if_op +from ngraph.opset11 import interpolate +from ngraph.opset11 import irdft +from ngraph.opset11 import is_finite +from ngraph.opset11 import is_inf +from ngraph.opset11 import is_nan +from ngraph.opset11 import i420_to_bgr +from ngraph.opset11 import i420_to_rgb +from ngraph.opset11 import less +from ngraph.opset11 import less_equal +from ngraph.opset11 import log +from ngraph.opset11 import logical_and +from ngraph.opset11 import logical_not +from ngraph.opset11 import logical_or +from ngraph.opset11 import logical_xor +from ngraph.opset11 import log_softmax +from ngraph.opset11 import loop +from ngraph.opset11 import lrn +from ngraph.opset11 import lstm_cell +from ngraph.opset11 import lstm_sequence +from ngraph.opset11 import matmul +from ngraph.opset11 import matrix_nms +from ngraph.opset11 import max_pool +from ngraph.opset11 import maximum +from ngraph.opset11 import minimum +from ngraph.opset11 import mish +from ngraph.opset11 import mod +from ngraph.opset11 import multiclass_nms +from ngraph.opset11 import multiply +from ngraph.opset11 import mvn +from ngraph.opset11 import negative +from ngraph.opset11 import non_max_suppression +from ngraph.opset11 import non_zero +from ngraph.opset11 import normalize_l2 +from ngraph.opset11 import not_equal +from ngraph.opset11 import nv12_to_bgr +from ngraph.opset11 import nv12_to_rgb +from ngraph.opset11 import one_hot +from ngraph.opset11 import pad +from ngraph.opset11 import parameter +from ngraph.opset11 import power +from ngraph.opset11 import prelu +from ngraph.opset11 import prior_box +from ngraph.opset11 import prior_box_clustered +from ngraph.opset11 import psroi_pooling +from ngraph.opset11 import proposal +from ngraph.opset11 import random_uniform +from ngraph.opset11 import range +from ngraph.opset11 import rdft +from ngraph.opset11 import read_value +from ngraph.opset11 import reduce_l1 +from ngraph.opset11 import reduce_l2 +from ngraph.opset11 import reduce_logical_and +from ngraph.opset11 import reduce_logical_or +from ngraph.opset11 import reduce_max +from ngraph.opset11 import reduce_mean +from ngraph.opset11 import reduce_min +from ngraph.opset11 import reduce_prod +from ngraph.opset11 import reduce_sum +from ngraph.opset11 import region_yolo +from ngraph.opset11 import reorg_yolo +from ngraph.opset11 import relu +from ngraph.opset11 import reshape +from ngraph.opset11 import result +from ngraph.opset11 import reverse_sequence +from ngraph.opset11 import rnn_cell +from ngraph.opset11 import rnn_sequence +from ngraph.opset11 import roi_align +from ngraph.opset11 import roi_pooling +from ngraph.opset11 import roll +from ngraph.opset11 import round +from ngraph.opset11 import scatter_elements_update +from ngraph.opset11 import scatter_update +from ngraph.opset11 import select +from ngraph.opset11 import selu +from ngraph.opset11 import shape_of +from ngraph.opset11 import shuffle_channels +from ngraph.opset11 import sigmoid +from ngraph.opset11 import sign +from ngraph.opset11 import sin +from ngraph.opset11 import sinh +from ngraph.opset11 import slice +from ngraph.opset11 import softmax +from ngraph.opset11 import softplus +from ngraph.opset11 import softsign +from ngraph.opset11 import space_to_batch +from ngraph.opset11 import space_to_depth +from ngraph.opset11 import split +from ngraph.opset11 import sqrt +from ngraph.opset11 import squared_difference +from ngraph.opset11 import squeeze +from ngraph.opset11 import strided_slice +from ngraph.opset11 import subtract +from ngraph.opset11 import swish +from ngraph.opset11 import tan +from ngraph.opset11 import tanh +from ngraph.opset11 import tensor_iterator +from ngraph.opset11 import tile +from ngraph.opset11 import topk +from ngraph.opset11 import transpose +from ngraph.opset11 import unique +from ngraph.opset11 import unsqueeze +from ngraph.opset11 import variadic_split # Extend Node class to support binary operators diff --git a/src/bindings/python/src/compatibility/ngraph/opset11/__init__.py b/src/bindings/python/src/compatibility/ngraph/opset11/__init__.py new file mode 100644 index 00000000000000..047c93e4cc03d3 --- /dev/null +++ b/src/bindings/python/src/compatibility/ngraph/opset11/__init__.py @@ -0,0 +1,177 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from ngraph.opset1.ops import absolute +from ngraph.opset1.ops import absolute as abs +from ngraph.opset1.ops import acos +from ngraph.opset4.ops import acosh +from ngraph.opset8.ops import adaptive_avg_pool +from ngraph.opset8.ops import adaptive_max_pool +from ngraph.opset1.ops import add +from ngraph.opset1.ops import asin +from ngraph.opset4.ops import asinh +from ngraph.opset3.ops import assign +from ngraph.opset1.ops import atan +from ngraph.opset4.ops import atanh +from ngraph.opset1.ops import avg_pool +from ngraph.opset5.ops import batch_norm_inference +from ngraph.opset2.ops import batch_to_space +from ngraph.opset1.ops import binary_convolution +from ngraph.opset3.ops import broadcast +from ngraph.opset3.ops import bucketize +from ngraph.opset1.ops import ceiling +from ngraph.opset1.ops import ceiling as ceil +from ngraph.opset1.ops import clamp +from ngraph.opset1.ops import concat +from ngraph.opset1.ops import constant +from ngraph.opset1.ops import convert +from ngraph.opset1.ops import convert_like +from ngraph.opset1.ops import convolution +from ngraph.opset1.ops import convolution_backprop_data +from ngraph.opset1.ops import cos +from ngraph.opset1.ops import cosh +from ngraph.opset1.ops import ctc_greedy_decoder +from ngraph.opset6.ops import ctc_greedy_decoder_seq_len +from ngraph.opset4.ops import ctc_loss +from ngraph.opset3.ops import cum_sum +from ngraph.opset3.ops import cum_sum as cumsum +from ngraph.opset8.ops import deformable_convolution +from ngraph.opset1.ops import deformable_psroi_pooling +from ngraph.opset1.ops import depth_to_space +from ngraph.opset8.ops import detection_output +from ngraph.opset7.ops import dft +from ngraph.opset1.ops import divide +from ngraph.opset7.ops import einsum +from ngraph.opset1.ops import elu +from ngraph.opset3.ops import embedding_bag_offsets_sum +from ngraph.opset3.ops import embedding_bag_packed_sum +from ngraph.opset3.ops import embedding_segments_sum +from ngraph.opset3.ops import extract_image_patches +from ngraph.opset1.ops import equal +from ngraph.opset1.ops import erf +from ngraph.opset1.ops import exp +from ngraph.opset9.ops import eye +from ngraph.opset1.ops import fake_quantize +from ngraph.opset1.ops import floor +from ngraph.opset1.ops import floor_mod +from ngraph.opset8.ops import gather +from ngraph.opset6.ops import gather_elements +from ngraph.opset8.ops import gather_nd +from ngraph.opset1.ops import gather_tree +from ngraph.opset7.ops import gelu +from ngraph.opset9.ops import generate_proposals +from ngraph.opset1.ops import greater +from ngraph.opset1.ops import greater_equal +from ngraph.opset9.ops import grid_sample +from ngraph.opset1.ops import grn +from ngraph.opset1.ops import group_convolution +from ngraph.opset1.ops import group_convolution_backprop_data +from ngraph.opset3.ops import gru_cell +from ngraph.opset5.ops import gru_sequence +from ngraph.opset1.ops import hard_sigmoid +from ngraph.opset5.ops import hsigmoid +from ngraph.opset4.ops import hswish +from ngraph.opset7.ops import idft +from ngraph.opset8.ops import if_op +from ngraph.opset11.ops import interpolate +from ngraph.opset9.ops import irdft +from ngraph.opset10.ops import is_finite +from ngraph.opset10.ops import is_inf +from ngraph.opset10.ops import is_nan +from ngraph.opset8.ops import i420_to_bgr +from ngraph.opset8.ops import i420_to_rgb +from ngraph.opset1.ops import less +from ngraph.opset1.ops import less_equal +from ngraph.opset1.ops import log +from ngraph.opset1.ops import logical_and +from ngraph.opset1.ops import logical_not +from ngraph.opset1.ops import logical_or +from ngraph.opset1.ops import logical_xor +from ngraph.opset5.ops import log_softmax +from ngraph.opset5.ops import loop +from ngraph.opset1.ops import lrn +from ngraph.opset4.ops import lstm_cell +from ngraph.opset5.ops import lstm_sequence +from ngraph.opset1.ops import matmul +from ngraph.opset8.ops import matrix_nms +from ngraph.opset8.ops import max_pool +from ngraph.opset1.ops import maximum +from ngraph.opset1.ops import minimum +from ngraph.opset4.ops import mish +from ngraph.opset1.ops import mod +from ngraph.opset9.ops import multiclass_nms +from ngraph.opset1.ops import multiply +from ngraph.opset6.ops import mvn +from ngraph.opset1.ops import negative +from ngraph.opset9.ops import non_max_suppression +from ngraph.opset3.ops import non_zero +from ngraph.opset1.ops import normalize_l2 +from ngraph.opset1.ops import not_equal +from ngraph.opset8.ops import nv12_to_bgr +from ngraph.opset8.ops import nv12_to_rgb +from ngraph.opset1.ops import one_hot +from ngraph.opset1.ops import pad +from ngraph.opset1.ops import parameter +from ngraph.opset1.ops import power +from ngraph.opset1.ops import prelu +from ngraph.opset8.ops import prior_box +from ngraph.opset1.ops import prior_box_clustered +from ngraph.opset1.ops import psroi_pooling +from ngraph.opset4.ops import proposal +from ngraph.opset8.ops import random_uniform +from ngraph.opset1.ops import range +from ngraph.opset9.ops import rdft +from ngraph.opset3.ops import read_value +from ngraph.opset4.ops import reduce_l1 +from ngraph.opset4.ops import reduce_l2 +from ngraph.opset1.ops import reduce_logical_and +from ngraph.opset1.ops import reduce_logical_or +from ngraph.opset1.ops import reduce_max +from ngraph.opset1.ops import reduce_mean +from ngraph.opset1.ops import reduce_min +from ngraph.opset1.ops import reduce_prod +from ngraph.opset1.ops import reduce_sum +from ngraph.opset1.ops import region_yolo +from ngraph.opset2.ops import reorg_yolo +from ngraph.opset1.ops import relu +from ngraph.opset1.ops import reshape +from ngraph.opset1.ops import result +from ngraph.opset1.ops import reverse_sequence +from ngraph.opset3.ops import rnn_cell +from ngraph.opset5.ops import rnn_sequence +from ngraph.opset9.ops import roi_align +from ngraph.opset2.ops import roi_pooling +from ngraph.opset7.ops import roll +from ngraph.opset5.ops import round +from ngraph.opset3.ops import scatter_elements_update +from ngraph.opset3.ops import scatter_update +from ngraph.opset1.ops import select +from ngraph.opset1.ops import selu +from ngraph.opset3.ops import shape_of +from ngraph.opset3.ops import shuffle_channels +from ngraph.opset1.ops import sigmoid +from ngraph.opset1.ops import sign +from ngraph.opset1.ops import sin +from ngraph.opset1.ops import sinh +from ngraph.opset8.ops import slice +from ngraph.opset8.ops import softmax +from ngraph.opset4.ops import softplus +from ngraph.opset9.ops import softsign +from ngraph.opset2.ops import space_to_batch +from ngraph.opset1.ops import space_to_depth +from ngraph.opset1.ops import split +from ngraph.opset1.ops import sqrt +from ngraph.opset1.ops import squared_difference +from ngraph.opset1.ops import squeeze +from ngraph.opset1.ops import strided_slice +from ngraph.opset1.ops import subtract +from ngraph.opset4.ops import swish +from ngraph.opset1.ops import tan +from ngraph.opset1.ops import tanh +from ngraph.opset1.ops import tensor_iterator +from ngraph.opset1.ops import tile +from ngraph.opset11.ops import topk +from ngraph.opset1.ops import transpose +from ngraph.opset10.ops import unique +from ngraph.opset1.ops import unsqueeze +from ngraph.opset1.ops import variadic_split diff --git a/src/bindings/python/src/compatibility/ngraph/opset11/ops.py b/src/bindings/python/src/compatibility/ngraph/opset11/ops.py new file mode 100644 index 00000000000000..3a4b54059ca6fc --- /dev/null +++ b/src/bindings/python/src/compatibility/ngraph/opset11/ops.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Factory functions for all openvino ops.""" +from functools import partial +from typing import List, Optional + +from ngraph.impl import Node +from ngraph.opset_utils import _get_node_factory +from ngraph.utils.decorators import nameable_op +from ngraph.utils.types import ( + NodeInput, + as_nodes, +) + +_get_node_factory_opset11 = partial(_get_node_factory, "opset11") + +# -------------------------------------------- ops ------------------------------------------------ + + +@nameable_op +def interpolate( + image: NodeInput, + scales_or_sizes: NodeInput, + mode: str, + shape_calculation_mode: str, + pads_begin: Optional[List[int]] = None, + pads_end: Optional[List[int]] = None, + coordinate_transformation_mode: str = "half_pixel", + nearest_mode: str = "round_prefer_floor", + antialias: bool = False, + cube_coeff: float = -0.75, + axes: Optional[NodeInput] = None, + name: Optional[str] = None, +) -> Node: + """Performs the interpolation of the input tensor. + + :param image: The node providing input tensor with data for interpolation. + :param scales_or_sizes: + 1D tensor providing information used to calculate the output shape + of the operation. It might contain floats (scales) or integers(sizes). + :param mode: Specifies type of interpolation. Possible values are: nearest, linear, + linear_onnx, cubic, bilinear_pillow, bicubic_pillow. + :param shape_calculation_mode: + Specifies how the scales_or_sizes input should be interpreted. + :param pads_begin: Specifies the number of pixels to add to the beginning of the image + being interpolated. Default is None. + :param pads_end: Specifies the number of pixels to add to the end of the image being + interpolated. Default is None. + :param coordinate_transformation_mode: + Specifies how to transform the coordinate in the resized tensor to the + coordinate in the original tensor. Default is "half_pixel". + :param nearest_mode: Specifies round mode when mode == nearest and is used only when + mode == nearest. Default is "round_prefer_floor". + :param antialias: Specifies whether to perform anti-aliasing. Default is False. + :param cube_coeff: Specifies the parameter a for cubic interpolation. Default is -0.75. + :param axes: 1D tensor specifying dimension indices where interpolation is applied. + The default is None. + :param name: Optional name for the output node. The default is None. + :return: Node representing the interpolation operation. + """ + attrs = { + "mode": mode, + "shape_calculation_mode": shape_calculation_mode, + "coordinate_transformation_mode": coordinate_transformation_mode, + "nearest_mode": nearest_mode, + "antialias": antialias, + "cube_coeff": cube_coeff, + } + + attrs["pads_begin"] = [] if pads_begin is None else pads_begin + attrs["pads_end"] = [] if pads_end is None else pads_end + + inputs = as_nodes(image, scales_or_sizes) if axes is None else as_nodes(image, scales_or_sizes, axes) + + return _get_node_factory_opset11().create("Interpolate", inputs, attrs) + + +@nameable_op +def topk( + data: NodeInput, + k: NodeInput, + axis: int, + mode: str, + sort: str, + index_element_type: str = "i32", + stable: bool = False, + name: Optional[str] = None, +) -> Node: + """Return a node which performs TopK. + + :param data: Input data. + :param k: K. + :param axis: TopK Axis. + :param mode: Compute TopK largest ('max') or smallest ('min') + :param sort: Order of output elements (sort by: 'none', 'index' or 'value') + :param index_element_type: Type of output tensor with indices. + :param stable: Specifies whether the equivalent elements should maintain + their relative order from the input tensor during sorting. + :return: The new node which performs TopK + """ + return _get_node_factory_opset11().create( + "TopK", + as_nodes(data, k), + {"axis": axis, "mode": mode, "sort": sort, "index_element_type": index_element_type, "stable": stable}, + ) diff --git a/src/bindings/python/src/compatibility/ngraph/utils/node_factory.py b/src/bindings/python/src/compatibility/ngraph/utils/node_factory.py index 6aa951a65b66b7..0e3d2cc09cecc2 100644 --- a/src/bindings/python/src/compatibility/ngraph/utils/node_factory.py +++ b/src/bindings/python/src/compatibility/ngraph/utils/node_factory.py @@ -12,7 +12,7 @@ from ngraph.exceptions import UserInputError -DEFAULT_OPSET = "opset10" +DEFAULT_OPSET = "opset11" class NodeFactory(object): diff --git a/src/bindings/python/src/compatibility/openvino/cmake/CythonConfig.cmake b/src/bindings/python/src/compatibility/openvino/cmake/CythonConfig.cmake index 73a4f21f871c42..05f75ceeb51772 100644 --- a/src/bindings/python/src/compatibility/openvino/cmake/CythonConfig.cmake +++ b/src/bindings/python/src/compatibility/openvino/cmake/CythonConfig.cmake @@ -36,7 +36,8 @@ if( PYTHONINTERP_FOUND ) file(TO_CMAKE_PATH "$ENV{HOME}" ENV_HOME) find_host_program( CYTHON_EXECUTABLE NAMES cython cython.bat cython3 - HINTS ${_python_path} ${ENV_HOME}/.local/bin $ENV{HOMEBREW_OPT}/cython/bin + HINTS ${_python_path} ${ENV_HOME}/.local/bin $ENV{HOMEBREW_OPT}/cython/bin + ${ENV_HOME}/Library/Python/${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/bin ) else() find_host_program( CYTHON_EXECUTABLE diff --git a/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt b/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt index 7c503d6070f5cf..cfeb78aa1c3995 100644 --- a/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt +++ b/src/bindings/python/src/compatibility/openvino/inference_engine/CMakeLists.txt @@ -22,6 +22,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") ie_add_compiler_flags(/wd4267) ie_add_compiler_flags(/wd4244) endif() +if (OV_COMPILER_IS_CLANG OR CMAKE_COMPILER_IS_GNUCXX) + ie_add_compiler_flags(-Wno-undef) +endif() if(UNUSED_BUT_SET_VARIABLE_SUPPORTED) ie_add_compiler_flags(-Wno-unused-but-set-variable) endif() diff --git a/src/bindings/python/src/compatibility/pyngraph/CMakeLists.txt b/src/bindings/python/src/compatibility/pyngraph/CMakeLists.txt index c8639d40c491e4..9efba7a8369b87 100644 --- a/src/bindings/python/src/compatibility/pyngraph/CMakeLists.txt +++ b/src/bindings/python/src/compatibility/pyngraph/CMakeLists.txt @@ -34,7 +34,6 @@ endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # disable warning: This operator was deprecated and will be removed with v0 operation. add_compile_options(/wd4996) - add_compile_options(/wd4244) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") add_compile_options(-Wno-deprecated-register -Wno-range-loop-analysis) elseif(OV_COMPILER_IS_APPLECLANG) diff --git a/src/bindings/python/src/compatibility/pyngraph/axis_set.cpp b/src/bindings/python/src/compatibility/pyngraph/axis_set.cpp index de7ff91d336c54..0aa59b9a055e27 100644 --- a/src/bindings/python/src/compatibility/pyngraph/axis_set.cpp +++ b/src/bindings/python/src/compatibility/pyngraph/axis_set.cpp @@ -36,7 +36,7 @@ void regclass_pyngraph_AxisSet(py::module m) { axis_set.def("__repr__", [](const ngraph::AxisSet& self) -> std::string { std::stringstream data_ss; - std::copy(self.begin(), self.end(), std::ostream_iterator(data_ss, ", ")); + std::copy(self.begin(), self.end(), std::ostream_iterator(data_ss, ", ")); std::string data_str = data_ss.str(); return ""; }); diff --git a/src/bindings/python/src/compatibility/pyngraph/coordinate_diff.cpp b/src/bindings/python/src/compatibility/pyngraph/coordinate_diff.cpp index 0a62ba0a07e22b..6c91879ad3ca60 100644 --- a/src/bindings/python/src/compatibility/pyngraph/coordinate_diff.cpp +++ b/src/bindings/python/src/compatibility/pyngraph/coordinate_diff.cpp @@ -26,7 +26,7 @@ void regclass_pyngraph_CoordinateDiff(py::module m) { coordinate_diff.def("__str__", [](const ngraph::CoordinateDiff& self) -> std::string { std::stringstream stringstream; - std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); + std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); std::string string = stringstream.str(); return string.substr(0, string.size() - 2); }); diff --git a/src/bindings/python/src/compatibility/pyngraph/discrete_type_info.cpp b/src/bindings/python/src/compatibility/pyngraph/discrete_type_info.cpp index b7418def4d3acf..49f8bb97953ebf 100644 --- a/src/bindings/python/src/compatibility/pyngraph/discrete_type_info.cpp +++ b/src/bindings/python/src/compatibility/pyngraph/discrete_type_info.cpp @@ -28,14 +28,14 @@ void regclass_pyngraph_DiscreteTypeInfo(py::module m) { discrete_type_info.def(py::self != py::self); discrete_type_info.def_readonly("name", &ngraph::DiscreteTypeInfo::name); - discrete_type_info.def_readonly("version", &ngraph::DiscreteTypeInfo::version); + discrete_type_info.def_readonly("version_id", &ngraph::DiscreteTypeInfo::version_id); discrete_type_info.def_readonly("parent", &ngraph::DiscreteTypeInfo::parent); discrete_type_info.def("__repr__", [](const ngraph::DiscreteTypeInfo& self) { std::string name = std::string(self.name); - std::string version = std::to_string(self.version); + std::string version = std::string(self.version_id); if (self.parent != nullptr) { - std::string parent_version = std::to_string(self.parent->version); + std::string parent_version = std::string(self.parent->version_id); std::string parent_name = self.parent->name; return ""; diff --git a/src/bindings/python/src/compatibility/pyngraph/node.cpp b/src/bindings/python/src/compatibility/pyngraph/node.cpp index 02bfb3d1395630..f696a4297cad7f 100644 --- a/src/bindings/python/src/compatibility/pyngraph/node.cpp +++ b/src/bindings/python/src/compatibility/pyngraph/node.cpp @@ -277,16 +277,6 @@ void regclass_pyngraph_Node(py::module m) { get_rt_info : PyRTMap A dictionary of user defined data. )"); - node.def("get_version", - &ngraph::Node::get_version, - R"( - Returns operation's version of the node. - - Returns - ---------- - get_version : int - Operation version. - )"); node.def("set_argument", &ngraph::Node::set_argument); node.def("set_arguments", [](const std::shared_ptr& self, const ngraph::NodeVector& args) { @@ -301,7 +291,6 @@ void regclass_pyngraph_Node(py::module m) { node.def_property_readonly("rt_info", (PyRTMap & (ngraph::Node::*)()) & ngraph::Node::get_rt_info, py::return_value_policy::reference_internal); - node.def_property_readonly("version", &ngraph::Node::get_version); node.def_property_readonly("type_info", &ngraph::Node::get_type_info); node.def_property("friendly_name", &ngraph::Node::get_friendly_name, &ngraph::Node::set_friendly_name); diff --git a/src/bindings/python/src/compatibility/pyngraph/node_factory.cpp b/src/bindings/python/src/compatibility/pyngraph/node_factory.cpp index 281525cca95832..2108a7a057bb3c 100644 --- a/src/bindings/python/src/compatibility/pyngraph/node_factory.cpp +++ b/src/bindings/python/src/compatibility/pyngraph/node_factory.cpp @@ -82,7 +82,7 @@ class NodeFactory { return it->second(); } - const ngraph::OpSet& m_opset = ngraph::get_opset10(); + const ngraph::OpSet& m_opset = ngraph::get_opset11(); std::unordered_map> m_variables; }; } // namespace diff --git a/src/bindings/python/src/compatibility/pyngraph/strides.cpp b/src/bindings/python/src/compatibility/pyngraph/strides.cpp index 88ff61655509ed..b740336e2288f5 100644 --- a/src/bindings/python/src/compatibility/pyngraph/strides.cpp +++ b/src/bindings/python/src/compatibility/pyngraph/strides.cpp @@ -24,7 +24,7 @@ void regclass_pyngraph_Strides(py::module m) { strides.def("__str__", [](const ngraph::Strides& self) -> std::string { std::stringstream stringstream; - std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); + std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); std::string string = stringstream.str(); return string.substr(0, string.size() - 2); }); diff --git a/src/bindings/python/src/openvino/frontend/paddle/__init__.py b/src/bindings/python/src/openvino/frontend/paddle/__init__.py index 240c9a7ac395c5..c90eb4079b0d8c 100644 --- a/src/bindings/python/src/openvino/frontend/paddle/__init__.py +++ b/src/bindings/python/src/openvino/frontend/paddle/__init__.py @@ -15,5 +15,6 @@ try: from openvino.frontend.paddle.py_paddle_frontend import ConversionExtensionPaddle as ConversionExtension + from openvino.frontend.paddle.py_paddle_frontend import OpExtensionPaddle as OpExtension except ImportError as err: raise ImportError("OpenVINO Paddle frontend is not available, please make sure the frontend is built." "{}".format(err)) diff --git a/src/bindings/python/src/openvino/frontend/pytorch/__init__.py b/src/bindings/python/src/openvino/frontend/pytorch/__init__.py index 78cfb7a018220f..c84f4df24425e7 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/__init__.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/__init__.py @@ -16,6 +16,8 @@ try: from openvino.frontend.pytorch.py_pytorch_frontend import _FrontEndPytorchDecoder as Decoder from openvino.frontend.pytorch.py_pytorch_frontend import _Type as DecoderType + from openvino.frontend.pytorch.py_pytorch_frontend import ConversionExtensionPytorch as ConversionExtension + from openvino.frontend.pytorch.py_pytorch_frontend import OpExtensionPytorch as OpExtension except ImportError as err: raise ImportError("OpenVINO PyTorch frontend is not available, please make sure the frontend is built." "{}".format(err)) diff --git a/src/bindings/python/src/openvino/frontend/pytorch/decoder.py b/src/bindings/python/src/openvino/frontend/pytorch/decoder.py index e28f9f6171e793..e1af35db27da82 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/decoder.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/decoder.py @@ -9,7 +9,6 @@ from openvino.runtime import op, PartialShape, Type as OVType, OVAny, Shape import typing -import warnings import torch import numpy as np import inspect @@ -98,18 +97,75 @@ def get_value_from_getattr(getattr_node, self_module): class TorchScriptPythonDecoder (Decoder): - def __init__(self, pt_module, graph_element=None): + def __init__(self, pt_module, graph_element=None, example_input=None, freeze=True): Decoder.__init__(self) # We store every decoder created by this decoder so that all them are not deleted until the first decoder is deleted self.m_decoders = [] + self._input_signature = None + converted_model = False if graph_element is None: - assert hasattr(pt_module, "inlined_graph"), "graph_element must have inlined_graph" + converted_model = True + pt_module = self._get_scripted_model(pt_module, example_input, freeze) self.graph_element = pt_module.inlined_graph else: self.graph_element = graph_element self.pt_module = pt_module self.raw_inputs = list(self.graph_element.inputs()) self.raw_outputs = list(self.graph_element.outputs()) + if self._input_signature is not None and self.raw_inputs[0].debugName() == "self": + self._input_signature.insert(0, "self") + + def _get_scripted_model(self, pt_module, example_inputs=None, freeze=True): + import torch + import inspect + + def prepare_example_inputs(inputs, input_signature): + if inputs is not None: + if isinstance(inputs, dict): + if input_signature is not None: + ordered_inputs = [] + used_sign = [] + for key in input_signature: + if key not in inputs: + continue + ordered_inputs.append(inputs[key]) + used_sign.append(key) + inputs = ordered_inputs + input_signature = used_sign + else: + inputs = list(inputs.values()) + input_signature = input_signature[:len(inputs)] + if isinstance(inputs, torch.Tensor): + inputs = [inputs] + return inputs, input_signature + + pt_module.eval() + input_signature = None + if isinstance(pt_module, torch.nn.Module) and not isinstance(pt_module, (torch.jit._trace.TopLevelTracedModule, torch.jit._script.RecursiveScriptModule)): + input_signature = list(inspect.signature(pt_module.forward).parameters.keys()) + try: + scripted = torch.jit.script(pt_module) + except Exception as scripting_err: + if example_inputs is not None: + inputs, input_signature = prepare_example_inputs(example_inputs, input_signature) + try: + scripted = torch.jit.trace(pt_module, inputs) + except Exception as tracing_e: + raise tracing_e + else: + raise scripting_err + else: + scripted = pt_module + if freeze: + try: + f_model = torch.jit.freeze(scripted) + except Exception: + # usually freezing failed when model already frozen for inference + f_model = scripted + else: + f_model = scripted + self._input_signature = input_signature + return f_model def inputs(self) -> list: return [x.unique() for x in self.raw_inputs] @@ -120,6 +176,11 @@ def get_input(self, index: int): def get_input_debug_name(self, index: int) -> str: return self._raw_input(index).debugName() + def get_input_signature_name(self, index: int) -> str: + if self._input_signature is not None and index < len(self._input_signature): + return self._input_signature[index] + return self.get_input_debug_name(index) + def get_input_shape(self, index: int): raw_input = self._raw_input(index) return self.get_shape_for_value(raw_input) @@ -410,6 +471,9 @@ def input(self, index): # TODO: remove def get_input_debug_name(self, index): return "input"+str(index) + def get_input_signature_name(self, index: int) -> str: + return self.get_input_debug_name(index) + def get_input_shape(self, index): print(f'[ FX DECODER DEBUG ] Decoder method called: {inspect.currentframe().f_code.co_name}') if index < len(self.input_shapes): diff --git a/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py b/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py index 7f84127a374b0c..b0cbee0605afff 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py @@ -2,6 +2,7 @@ from functools import partial import torch +import os from torch._dynamo.backends.common import aot_autograd, mem_efficient_fusion_kwargs from torch._dynamo.backends.common import fake_tensor_unsupported from torch._dynamo.backends.registry import register_backend @@ -18,7 +19,9 @@ @register_backend @fake_tensor_unsupported def openvino(subgraph, example_inputs): - return fx_openvino(subgraph, example_inputs) + if(os.getenv("PYTORCH_TRACING_MODE")=="TORCHFX"): + return fx_openvino(subgraph, example_inputs) + return ts_openvino(subgraph, example_inputs) def ts_openvino(subgraph, example_inputs): try: @@ -48,7 +51,12 @@ def ts_openvino(subgraph, example_inputs): om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) om.validate_nodes_and_infer_types() - compiled_model = core.compile_model(om, 'CPU') + device = 'CPU' + if os.getenv("OPENVINO_DEVICE") is not None: + device = os.getenv("OPENVINO_DEVICE") + assert device in core.available_devices, "Specified device " + device + " is not in the list of OpenVINO Available Devices" + + compiled_model = core.compile_model(om, device) def _call(*args): ov_inputs = [a.detach().cpu().numpy() for a in args] @@ -60,6 +68,7 @@ def _call(*args): return result return _call except Exception as e: + print(str(e)) return subgraph @@ -76,4 +85,5 @@ def _call(*args): return res return _call except Exception as e: + print(str(e)) return subgraph diff --git a/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/compile.py b/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/compile.py index 34147670f445bb..c29030bb450b43 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/compile.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/compile.py @@ -4,6 +4,7 @@ from types import MappingProxyType from warnings import warn +import os import torch import torch.overrides @@ -48,12 +49,19 @@ def openvino_compile(gm: GraphModule, *args): torch.bool: Type.boolean } - for idx, input_data in enumerate(args): #subgraph.example_inputs): + for idx, input_data in enumerate(args): om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) om.validate_nodes_and_infer_types() core = Core() - compiled = core.compile_model(om, "CPU") + + device = 'CPU' + + if os.getenv("OPENVINO_DEVICE") is not None: + device = os.getenv("OPENVINO_DEVICE") + assert device in core.available_devices, "Specified device " + device + " is not in the list of OpenVINO Available Devices" + + compiled = core.compile_model(om, device) print("!!Returning compiled model!!") return compiled diff --git a/src/bindings/python/src/openvino/runtime/__init__.py b/src/bindings/python/src/openvino/runtime/__init__.py index 3c2937c214ed70..5a6fe3efcc925c 100644 --- a/src/bindings/python/src/openvino/runtime/__init__.py +++ b/src/bindings/python/src/openvino/runtime/__init__.py @@ -44,6 +44,7 @@ from openvino._pyopenvino import get_batch from openvino._pyopenvino import set_batch from openvino._pyopenvino import serialize +from openvino._pyopenvino import shutdown # Import opsets from openvino.runtime import opset1 @@ -56,6 +57,7 @@ from openvino.runtime import opset8 from openvino.runtime import opset9 from openvino.runtime import opset10 +from openvino.runtime import opset11 # Import properties API from openvino._pyopenvino import properties @@ -66,19 +68,19 @@ # Extend Node class to support binary operators -Node.__add__ = opset10.add -Node.__sub__ = opset10.subtract -Node.__mul__ = opset10.multiply -Node.__div__ = opset10.divide -Node.__truediv__ = opset10.divide -Node.__radd__ = lambda left, right: opset10.add(right, left) -Node.__rsub__ = lambda left, right: opset10.subtract(right, left) -Node.__rmul__ = lambda left, right: opset10.multiply(right, left) -Node.__rdiv__ = lambda left, right: opset10.divide(right, left) -Node.__rtruediv__ = lambda left, right: opset10.divide(right, left) -Node.__eq__ = opset10.equal -Node.__ne__ = opset10.not_equal -Node.__lt__ = opset10.less -Node.__le__ = opset10.less_equal -Node.__gt__ = opset10.greater -Node.__ge__ = opset10.greater_equal +Node.__add__ = opset11.add +Node.__sub__ = opset11.subtract +Node.__mul__ = opset11.multiply +Node.__div__ = opset11.divide +Node.__truediv__ = opset11.divide +Node.__radd__ = lambda left, right: opset11.add(right, left) +Node.__rsub__ = lambda left, right: opset11.subtract(right, left) +Node.__rmul__ = lambda left, right: opset11.multiply(right, left) +Node.__rdiv__ = lambda left, right: opset11.divide(right, left) +Node.__rtruediv__ = lambda left, right: opset11.divide(right, left) +Node.__eq__ = opset11.equal +Node.__ne__ = opset11.not_equal +Node.__lt__ = opset11.less +Node.__le__ = opset11.less_equal +Node.__gt__ = opset11.greater +Node.__ge__ = opset11.greater_equal diff --git a/src/bindings/python/src/openvino/runtime/ie_api.py b/src/bindings/python/src/openvino/runtime/ie_api.py index 7bab65a0382113..90099609a1a313 100644 --- a/src/bindings/python/src/openvino/runtime/ie_api.py +++ b/src/bindings/python/src/openvino/runtime/ie_api.py @@ -2,7 +2,6 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from functools import singledispatch from typing import Any, Iterable, Union, Dict, Optional from pathlib import Path @@ -16,6 +15,7 @@ from openvino._pyopenvino import Tensor from openvino.runtime.utils.data_helpers import ( + OVDict, _InferRequestWrapper, _data_dispatch, tensor_from_file, @@ -25,7 +25,7 @@ class InferRequest(_InferRequestWrapper): """InferRequest class represents infer request which can be run in asynchronous or synchronous manners.""" - def infer(self, inputs: Any = None, shared_memory: bool = False) -> dict: + def infer(self, inputs: Any = None, shared_memory: bool = False) -> OVDict: """Infers specified input(s) in synchronous mode. Blocks all methods of InferRequest while request is running. @@ -68,14 +68,14 @@ def infer(self, inputs: Any = None, shared_memory: bool = False) -> dict: Default value: False :type shared_memory: bool, optional - :return: Dictionary of results from output tensors with ports as keys. - :rtype: Dict[openvino.runtime.ConstOutput, numpy.ndarray] + :return: Dictionary of results from output tensors with port/int/str keys. + :rtype: OVDict """ - return super().infer(_data_dispatch( + return OVDict(super().infer(_data_dispatch( self, inputs, is_shared=shared_memory, - )) + ))) def start_async( self, @@ -138,6 +138,15 @@ def start_async( userdata, ) + @property + def results(self) -> OVDict: + """Gets all outputs tensors of this InferRequest. + + :return: Dictionary of results from output tensors with ports as keys. + :rtype: Dict[openvino.runtime.ConstOutput, numpy.array] + """ + return OVDict(super().results) + class CompiledModel(CompiledModelBase): """CompiledModel class. @@ -161,7 +170,7 @@ def create_infer_request(self) -> InferRequest: """ return InferRequest(super().create_infer_request()) - def infer_new_request(self, inputs: Union[dict, list, tuple, Tensor, np.ndarray] = None) -> dict: + def infer_new_request(self, inputs: Union[dict, list, tuple, Tensor, np.ndarray] = None) -> OVDict: """Infers specified input(s) in synchronous mode. Blocks all methods of CompiledModel while request is running. @@ -187,8 +196,8 @@ def infer_new_request(self, inputs: Union[dict, list, tuple, Tensor, np.ndarray] :param inputs: Data to be set on input tensors. :type inputs: Union[Dict[keys, values], List[values], Tuple[values], Tensor, numpy.ndarray], optional - :return: Dictionary of results from output tensors with ports as keys. - :rtype: Dict[openvino.runtime.ConstOutput, numpy.array] + :return: Dictionary of results from output tensors with port/int/str keys. + :rtype: OVDict """ # It returns wrapped python InferReqeust and then call upon # overloaded functions of InferRequest class @@ -196,7 +205,7 @@ def infer_new_request(self, inputs: Union[dict, list, tuple, Tensor, np.ndarray] def __call__(self, inputs: Union[dict, list, tuple, Tensor, np.ndarray] = None, - shared_memory: bool = True) -> dict: + shared_memory: bool = True) -> OVDict: """Callable infer wrapper for CompiledModel. Infers specified input(s) in synchronous mode. @@ -248,8 +257,8 @@ def __call__(self, Default value: True :type shared_memory: bool, optional - :return: Dictionary of results from output tensors with ports as keys. - :rtype: Dict[openvino.runtime.ConstOutput, numpy.ndarray] + :return: Dictionary of results from output tensors with port/int/str as keys. + :rtype: OVDict """ if self._infer_request is None: self._infer_request = self.create_infer_request() diff --git a/src/bindings/python/src/openvino/runtime/opset11/__init__.py b/src/bindings/python/src/openvino/runtime/opset11/__init__.py new file mode 100644 index 00000000000000..3e867f548699b0 --- /dev/null +++ b/src/bindings/python/src/openvino/runtime/opset11/__init__.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from openvino.runtime.opset1.ops import absolute +from openvino.runtime.opset1.ops import absolute as abs +from openvino.runtime.opset1.ops import acos +from openvino.runtime.opset4.ops import acosh +from openvino.runtime.opset8.ops import adaptive_avg_pool +from openvino.runtime.opset8.ops import adaptive_max_pool +from openvino.runtime.opset1.ops import add +from openvino.runtime.opset1.ops import asin +from openvino.runtime.opset4.ops import asinh +from openvino.runtime.opset3.ops import assign +from openvino.runtime.opset1.ops import atan +from openvino.runtime.opset4.ops import atanh +from openvino.runtime.opset1.ops import avg_pool +from openvino.runtime.opset5.ops import batch_norm_inference +from openvino.runtime.opset2.ops import batch_to_space +from openvino.runtime.opset1.ops import binary_convolution +from openvino.runtime.opset3.ops import broadcast +from openvino.runtime.opset3.ops import bucketize +from openvino.runtime.opset1.ops import ceiling +from openvino.runtime.opset1.ops import ceiling as ceil +from openvino.runtime.opset1.ops import clamp +from openvino.runtime.opset1.ops import concat +from openvino.runtime.opset1.ops import constant +from openvino.runtime.opset1.ops import convert +from openvino.runtime.opset1.ops import convert_like +from openvino.runtime.opset1.ops import convolution +from openvino.runtime.opset1.ops import convolution_backprop_data +from openvino.runtime.opset1.ops import cos +from openvino.runtime.opset1.ops import cosh +from openvino.runtime.opset1.ops import ctc_greedy_decoder +from openvino.runtime.opset6.ops import ctc_greedy_decoder_seq_len +from openvino.runtime.opset4.ops import ctc_loss +from openvino.runtime.opset3.ops import cum_sum +from openvino.runtime.opset3.ops import cum_sum as cumsum +from openvino.runtime.opset8.ops import deformable_convolution +from openvino.runtime.opset1.ops import deformable_psroi_pooling +from openvino.runtime.opset1.ops import depth_to_space +from openvino.runtime.opset8.ops import detection_output +from openvino.runtime.opset7.ops import dft +from openvino.runtime.opset1.ops import divide +from openvino.runtime.opset7.ops import einsum +from openvino.runtime.opset1.ops import elu +from openvino.runtime.opset3.ops import embedding_bag_offsets_sum +from openvino.runtime.opset3.ops import embedding_bag_packed_sum +from openvino.runtime.opset3.ops import embedding_segments_sum +from openvino.runtime.opset3.ops import extract_image_patches +from openvino.runtime.opset1.ops import equal +from openvino.runtime.opset1.ops import erf +from openvino.runtime.opset1.ops import exp +from openvino.runtime.opset9.ops import eye +from openvino.runtime.opset1.ops import fake_quantize +from openvino.runtime.opset1.ops import floor +from openvino.runtime.opset1.ops import floor_mod +from openvino.runtime.opset8.ops import gather +from openvino.runtime.opset6.ops import gather_elements +from openvino.runtime.opset8.ops import gather_nd +from openvino.runtime.opset1.ops import gather_tree +from openvino.runtime.opset7.ops import gelu +from openvino.runtime.opset9.ops import generate_proposals +from openvino.runtime.opset1.ops import greater +from openvino.runtime.opset1.ops import greater_equal +from openvino.runtime.opset9.ops import grid_sample +from openvino.runtime.opset1.ops import grn +from openvino.runtime.opset1.ops import group_convolution +from openvino.runtime.opset1.ops import group_convolution_backprop_data +from openvino.runtime.opset3.ops import gru_cell +from openvino.runtime.opset5.ops import gru_sequence +from openvino.runtime.opset1.ops import hard_sigmoid +from openvino.runtime.opset5.ops import hsigmoid +from openvino.runtime.opset4.ops import hswish +from openvino.runtime.opset7.ops import idft +from openvino.runtime.opset8.ops import if_op +from openvino.runtime.opset11.ops import interpolate +from openvino.runtime.opset9.ops import irdft +from openvino.runtime.opset10.ops import is_finite +from openvino.runtime.opset10.ops import is_inf +from openvino.runtime.opset10.ops import is_nan +from openvino.runtime.opset8.ops import i420_to_bgr +from openvino.runtime.opset8.ops import i420_to_rgb +from openvino.runtime.opset1.ops import less +from openvino.runtime.opset1.ops import less_equal +from openvino.runtime.opset1.ops import log +from openvino.runtime.opset1.ops import logical_and +from openvino.runtime.opset1.ops import logical_not +from openvino.runtime.opset1.ops import logical_or +from openvino.runtime.opset1.ops import logical_xor +from openvino.runtime.opset5.ops import log_softmax +from openvino.runtime.opset5.ops import loop +from openvino.runtime.opset1.ops import lrn +from openvino.runtime.opset4.ops import lstm_cell +from openvino.runtime.opset5.ops import lstm_sequence +from openvino.runtime.opset1.ops import matmul +from openvino.runtime.opset8.ops import matrix_nms +from openvino.runtime.opset8.ops import max_pool +from openvino.runtime.opset1.ops import maximum +from openvino.runtime.opset1.ops import minimum +from openvino.runtime.opset4.ops import mish +from openvino.runtime.opset1.ops import mod +from openvino.runtime.opset9.ops import multiclass_nms +from openvino.runtime.opset1.ops import multiply +from openvino.runtime.opset6.ops import mvn +from openvino.runtime.opset1.ops import negative +from openvino.runtime.opset9.ops import non_max_suppression +from openvino.runtime.opset3.ops import non_zero +from openvino.runtime.opset1.ops import normalize_l2 +from openvino.runtime.opset1.ops import not_equal +from openvino.runtime.opset8.ops import nv12_to_bgr +from openvino.runtime.opset8.ops import nv12_to_rgb +from openvino.runtime.opset1.ops import one_hot +from openvino.runtime.opset1.ops import pad +from openvino.runtime.opset1.ops import parameter +from openvino.runtime.opset1.ops import power +from openvino.runtime.opset1.ops import prelu +from openvino.runtime.opset8.ops import prior_box +from openvino.runtime.opset1.ops import prior_box_clustered +from openvino.runtime.opset1.ops import psroi_pooling +from openvino.runtime.opset4.ops import proposal +from openvino.runtime.opset1.ops import range +from openvino.runtime.opset8.ops import random_uniform +from openvino.runtime.opset9.ops import rdft +from openvino.runtime.opset3.ops import read_value +from openvino.runtime.opset4.ops import reduce_l1 +from openvino.runtime.opset4.ops import reduce_l2 +from openvino.runtime.opset1.ops import reduce_logical_and +from openvino.runtime.opset1.ops import reduce_logical_or +from openvino.runtime.opset1.ops import reduce_max +from openvino.runtime.opset1.ops import reduce_mean +from openvino.runtime.opset1.ops import reduce_min +from openvino.runtime.opset1.ops import reduce_prod +from openvino.runtime.opset1.ops import reduce_sum +from openvino.runtime.opset1.ops import region_yolo +from openvino.runtime.opset2.ops import reorg_yolo +from openvino.runtime.opset1.ops import relu +from openvino.runtime.opset1.ops import reshape +from openvino.runtime.opset1.ops import result +from openvino.runtime.opset1.ops import reverse_sequence +from openvino.runtime.opset3.ops import rnn_cell +from openvino.runtime.opset5.ops import rnn_sequence +from openvino.runtime.opset9.ops import roi_align +from openvino.runtime.opset2.ops import roi_pooling +from openvino.runtime.opset7.ops import roll +from openvino.runtime.opset5.ops import round +from openvino.runtime.opset3.ops import scatter_elements_update +from openvino.runtime.opset3.ops import scatter_update +from openvino.runtime.opset1.ops import select +from openvino.runtime.opset1.ops import selu +from openvino.runtime.opset3.ops import shape_of +from openvino.runtime.opset3.ops import shuffle_channels +from openvino.runtime.opset1.ops import sigmoid +from openvino.runtime.opset1.ops import sign +from openvino.runtime.opset1.ops import sin +from openvino.runtime.opset1.ops import sinh +from openvino.runtime.opset8.ops import slice +from openvino.runtime.opset8.ops import softmax +from openvino.runtime.opset4.ops import softplus +from openvino.runtime.opset9.ops import softsign +from openvino.runtime.opset2.ops import space_to_batch +from openvino.runtime.opset1.ops import space_to_depth +from openvino.runtime.opset1.ops import split +from openvino.runtime.opset1.ops import sqrt +from openvino.runtime.opset1.ops import squared_difference +from openvino.runtime.opset1.ops import squeeze +from openvino.runtime.opset1.ops import strided_slice +from openvino.runtime.opset1.ops import subtract +from openvino.runtime.opset4.ops import swish +from openvino.runtime.opset1.ops import tan +from openvino.runtime.opset1.ops import tanh +from openvino.runtime.opset1.ops import tensor_iterator +from openvino.runtime.opset1.ops import tile +from openvino.runtime.opset11.ops import topk +from openvino.runtime.opset1.ops import transpose +from openvino.runtime.opset10.ops import unique +from openvino.runtime.opset1.ops import unsqueeze +from openvino.runtime.opset1.ops import variadic_split diff --git a/src/bindings/python/src/openvino/runtime/opset11/ops.py b/src/bindings/python/src/openvino/runtime/opset11/ops.py new file mode 100644 index 00000000000000..235b0e0ef37af5 --- /dev/null +++ b/src/bindings/python/src/openvino/runtime/opset11/ops.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Factory functions for all openvino ops.""" +from functools import partial +from typing import List, Optional + +from openvino.runtime import Node +from openvino.runtime.opset_utils import _get_node_factory +from openvino.runtime.utils.decorators import nameable_op +from openvino.runtime.utils.types import ( + NodeInput, + as_nodes, +) + +_get_node_factory_opset11 = partial(_get_node_factory, "opset11") + +# -------------------------------------------- ops ------------------------------------------------ + + +@nameable_op +def interpolate( + image: NodeInput, + scales_or_sizes: NodeInput, + mode: str, + shape_calculation_mode: str, + pads_begin: Optional[List[int]] = None, + pads_end: Optional[List[int]] = None, + coordinate_transformation_mode: str = "half_pixel", + nearest_mode: str = "round_prefer_floor", + antialias: bool = False, + cube_coeff: float = -0.75, + axes: Optional[NodeInput] = None, + name: Optional[str] = None, +) -> Node: + """Perfors the interpolation of the input tensor. + + :param image: The node providing input tensor with data for interpolation. + :param scales_or_sizes: + 1D tensor providing information used to calculate the output shape + of the operation. It might contain floats (scales) or integers(sizes). + :param mode: Specifies type of interpolation. Possible values are: nearest, linear, + linear_onnx, cubic, bilinear_pillow, bicubic_pillow. + :param shape_calculation_mode: + Specifies how the scales_or_sizes input should be interpreted. + :param pads_begin: Specifies the number of pixels to add to the beginning of the image + being interpolated. Default is None. + :param pads_end: Specifies the number of pixels to add to the end of the image being + interpolated. Default is None. + :param coordinate_transformation_mode: + Specifies how to transform the coordinate in the resized tensor to the + coordinate in the original tensor. Default is "half_pixel". + :param nearest_mode: Specifies round mode when mode == nearest and is used only when + mode == nearest. Default is "round_prefer_floor". + :param antialias: Specifies whether to perform anti-aliasing. Default is False. + :param cube_coeff: Specifies the parameter a for cubic interpolation. Default is -0.75. + :param axes: 1D tensor specifying dimension indices where interpolation is applied. + The default is None. + :param name: Optional name for the output node. The default is None. + :return: Node representing the interpolation operation. + """ + attrs = { + "mode": mode, + "shape_calculation_mode": shape_calculation_mode, + "coordinate_transformation_mode": coordinate_transformation_mode, + "nearest_mode": nearest_mode, + "antialias": antialias, + "cube_coeff": cube_coeff, + } + + attrs["pads_begin"] = [] if pads_begin is None else pads_begin + attrs["pads_end"] = [] if pads_end is None else pads_end + + inputs = as_nodes(image, scales_or_sizes) if axes is None else as_nodes(image, scales_or_sizes, axes) + + return _get_node_factory_opset11().create("Interpolate", inputs, attrs) + + +@nameable_op +def topk( + data: NodeInput, + k: NodeInput, + axis: int, + mode: str, + sort: str, + index_element_type: str = "i32", + stable: bool = False, + name: Optional[str] = None, +) -> Node: + """Return a node which performs TopK. + + :param data: Input data. + :param k: K. + :param axis: TopK Axis. + :param mode: Compute TopK largest ('max') or smallest ('min') + :param sort: Order of output elements (sort by: 'none', 'index' or 'value') + :param index_element_type: Type of output tensor with indices. + :param stable: Specifies whether the equivalent elements should maintain + their relative order from the input tensor during sorting. + :return: The new node which performs TopK + """ + return _get_node_factory_opset11().create( + "TopK", + as_nodes(data, k), + {"axis": axis, "mode": mode, "sort": sort, "index_element_type": index_element_type, "stable": stable}, + ) diff --git a/src/bindings/python/src/openvino/runtime/utils/data_helpers/__init__.py b/src/bindings/python/src/openvino/runtime/utils/data_helpers/__init__.py index e49265ccca987f..829a77af96a04c 100644 --- a/src/bindings/python/src/openvino/runtime/utils/data_helpers/__init__.py +++ b/src/bindings/python/src/openvino/runtime/utils/data_helpers/__init__.py @@ -5,3 +5,4 @@ from openvino.runtime.utils.data_helpers.data_dispatcher import _data_dispatch from openvino.runtime.utils.data_helpers.wrappers import tensor_from_file from openvino.runtime.utils.data_helpers.wrappers import _InferRequestWrapper +from openvino.runtime.utils.data_helpers.wrappers import OVDict diff --git a/src/bindings/python/src/openvino/runtime/utils/data_helpers/wrappers.py b/src/bindings/python/src/openvino/runtime/utils/data_helpers/wrappers.py index 24b09d40de9555..1bf23a7cad4f30 100644 --- a/src/bindings/python/src/openvino/runtime/utils/data_helpers/wrappers.py +++ b/src/bindings/python/src/openvino/runtime/utils/data_helpers/wrappers.py @@ -4,7 +4,18 @@ import numpy as np -from openvino._pyopenvino import Tensor +# TODO: remove this WA and refactor OVDict when Python3.8 +# becomes minimal supported version. +try: + from functools import singledispatchmethod +except ImportError: + from singledispatchmethod import singledispatchmethod # type: ignore[no-redef] + +from collections.abc import Mapping +from typing import Dict, Set, Tuple, Union, Iterator, Optional +from typing import KeysView, ItemsView, ValuesView + +from openvino._pyopenvino import Tensor, ConstOutput from openvino._pyopenvino import InferRequest as InferRequestBase @@ -20,3 +31,121 @@ def __init__(self, other: InferRequestBase) -> None: # Private memeber to store newly created shared memory data self._inputs_data = None super().__init__(other) + + +class OVDict(Mapping): + """Custom OpenVINO dictionary with inference results. + + This class is a dict-like object. It provides possibility to + address data tensors with three key types: + + * `openvino.runtime.ConstOutput` - port of the output + * `int` - index of the output + * `str` - names of the output + + This class follows `frozenset`/`tuple` concept of immutability. + It is prohibited to assign new items or edit them. + + To revert to the previous behavior use `to_dict` method which + return shallow copy of underlaying dictionary. + Note: It removes addressing feature! New dictionary keeps + only `ConstOutput` keys. + + If a tuple returns value is needed, use `to_tuple` method which + converts values to the tuple. + + :Example: + + .. code-block:: python + + # Reverts to the previous behavior of the native dict + result = request.infer(inputs).to_dict() + # or alternatively: + result = dict(request.infer(inputs)) + + .. code-block:: python + + # To dispatch outputs of multi-ouput inference: + out1, out2, out3, _ = request.infer(inputs).values() + # or alternatively: + out1, out2, out3, _ = request.infer(inputs).to_tuple() + """ + def __init__(self, _dict: Dict[ConstOutput, np.ndarray]) -> None: + self._dict = _dict + self._names: Optional[Dict[ConstOutput, Set[str]]] = None + + def __iter__(self) -> Iterator: + return self._dict.__iter__() + + def __len__(self) -> int: + return len(self._dict) + + def __repr__(self) -> str: + return self._dict.__repr__() + + def __get_names(self) -> Dict[ConstOutput, Set[str]]: + """Return names of every output key. + + Insert empty set if key has no name. + """ + return {key: key.get_names() for key in self._dict.keys()} + + def __get_key(self, index: int) -> ConstOutput: + return list(self._dict.keys())[index] + + @singledispatchmethod + def __getitem_impl(self, key: Union[ConstOutput, int, str]) -> np.ndarray: + raise TypeError(f"Unknown key type: {type(key)}") + + @__getitem_impl.register + def _(self, key: ConstOutput) -> np.ndarray: + return self._dict[key] + + @__getitem_impl.register + def _(self, key: int) -> np.ndarray: + try: + return self._dict[self.__get_key(key)] + except IndexError: + raise KeyError(key) + + @__getitem_impl.register + def _(self, key: str) -> np.ndarray: + if self._names is None: + self._names = self.__get_names() + for port, port_names in self._names.items(): + if key in port_names: + return self._dict[port] + raise KeyError(key) + + def __getitem__(self, key: Union[ConstOutput, int, str]) -> np.ndarray: + return self.__getitem_impl(key) + + def keys(self) -> KeysView[ConstOutput]: + return self._dict.keys() + + def values(self) -> ValuesView[np.ndarray]: + return self._dict.values() + + def items(self) -> ItemsView[ConstOutput, np.ndarray]: + return self._dict.items() + + def names(self) -> Tuple[Set[str], ...]: + """Return names of every output key. + + Insert empty set if key has no name. + """ + if self._names is None: + self._names = self.__get_names() + return tuple(self._names.values()) + + def to_dict(self) -> Dict[ConstOutput, np.ndarray]: + """Return underlaying native dictionary. + + Function performs shallow copy, thus any modifications to + returned values may affect this class as well. + """ + return self._dict + + def to_tuple(self) -> tuple: + """Convert values of this dictionary to a tuple.""" + return tuple(self._dict.values()) diff --git a/src/bindings/python/src/openvino/runtime/utils/node_factory.py b/src/bindings/python/src/openvino/runtime/utils/node_factory.py index a89c05ab0cffb6..f952bcf90fb4dc 100644 --- a/src/bindings/python/src/openvino/runtime/utils/node_factory.py +++ b/src/bindings/python/src/openvino/runtime/utils/node_factory.py @@ -13,7 +13,7 @@ from openvino.runtime.exceptions import UserInputError -DEFAULT_OPSET = "opset10" +DEFAULT_OPSET = "opset11" class NodeFactory(object): diff --git a/src/bindings/python/src/pyopenvino/CMakeLists.txt b/src/bindings/python/src/pyopenvino/CMakeLists.txt index c23c84edcf0115..6dc2226e450499 100644 --- a/src/bindings/python/src/pyopenvino/CMakeLists.txt +++ b/src/bindings/python/src/pyopenvino/CMakeLists.txt @@ -26,15 +26,8 @@ endif() # compile options -if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - # disable warning: This operator was deprecated and will be removed with v0 operation. - add_compile_options(/wd4996) - add_compile_options(/wd4244) -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - add_compile_options(-Wno-deprecated-register -Wno-range-loop-analysis) -elseif(OV_COMPILER_IS_APPLECLANG) +if(OV_COMPILER_IS_APPLECLANG) add_link_options(-stdlib=libc++) - add_compile_options(-Wno-unused-value -Wno-range-loop-analysis) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # WA for GCC 7.5 "PYBIND11_NOINLINE inline" warning add_compile_options(-Wno-error=attributes) @@ -85,7 +78,7 @@ set_target_properties(${PROJECT_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO} OUTPUT_NAME "_pyopenvino") -addVersionDefines(pyopenvino.cpp CI_BUILD_NUMBER) +ov_add_version_defines(pyopenvino.cpp ${PROJECT_NAME}) if(OV_GENERATOR_MULTI_CONFIG) string(APPEND _cmd_echo diff --git a/src/bindings/python/src/pyopenvino/core/common.cpp b/src/bindings/python/src/pyopenvino/core/common.cpp index 2ad7e395a92895..d899cb5f121ec2 100644 --- a/src/bindings/python/src/pyopenvino/core/common.cpp +++ b/src/bindings/python/src/pyopenvino/core/common.cpp @@ -53,6 +53,27 @@ const std::map& dtype_to_ov_type() { return dtype_to_ov_type_mapping; } +namespace containers { +const TensorIndexMap cast_to_tensor_index_map(const py::dict& inputs) { + TensorIndexMap result_map; + for (auto&& input : inputs) { + int idx; + if (py::isinstance(input.first)) { + idx = input.first.cast(); + } else { + throw py::type_error("incompatible function arguments!"); + } + if (py::isinstance(input.second)) { + auto tensor = Common::cast_to_tensor(input.second); + result_map[idx] = tensor; + } else { + throw ov::Exception("Unable to cast tensor " + std::to_string(idx) + "!"); + } + } + return result_map; +} +}; // namespace containers + namespace array_helpers { bool is_contiguous(const py::array& array) { @@ -110,6 +131,67 @@ py::array as_contiguous(py::array& array, ov::element::Type type) { } } +py::array array_from_tensor(ov::Tensor&& t) { + switch (t.get_element_type()) { + case ov::element::Type_t::f32: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::f64: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::bf16: { + return py::array(py::dtype("float16"), t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::f16: { + return py::array(py::dtype("float16"), t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::i8: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::i16: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::i32: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::i64: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::u8: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::u16: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::u32: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::u64: { + return py::array_t(t.get_shape(), t.data()); + break; + } + case ov::element::Type_t::boolean: { + return py::array_t(t.get_shape(), t.data()); + break; + } + default: { + throw ov::Exception("Numpy array cannot be created from given OV Tensor!"); + break; + } + } +} + }; // namespace array_helpers template <> @@ -206,14 +288,14 @@ ov::PartialShape partial_shape_from_list(const py::list& shape) { std::to_string(bounded_dim.size()) + " elements were given."); } if (!(py::isinstance(bounded_dim[0]) && py::isinstance(bounded_dim[1]))) { - throw py::type_error("Incorrect pair of types (" + std::string(bounded_dim[0].get_type().str()) + ", " + - std::string(bounded_dim[1].get_type().str()) + + throw py::type_error("Incorrect pair of types (" + std::string(py::str(bounded_dim[0].get_type())) + + ", " + std::string(py::str(bounded_dim[1].get_type())) + ") for dynamic dimension, ints are expected."); } pshape.insert(pshape.end(), ov::Dimension(bounded_dim[0].cast(), bounded_dim[1].cast())); } else { - throw py::type_error("Incorrect type " + std::string(dim.get_type().str()) + + throw py::type_error("Incorrect type " + std::string(py::str(dim.get_type())) + " for dimension. Expected types are: " "int, str, openvino.runtime.Dimension, list/tuple with lower and upper values for " "dynamic dimension."); @@ -226,38 +308,6 @@ const ov::Tensor& cast_to_tensor(const py::handle& tensor) { return tensor.cast(); } -const Containers::TensorNameMap cast_to_tensor_name_map(const py::dict& inputs) { - Containers::TensorNameMap result_map; - for (auto&& input : inputs) { - std::string name; - if (py::isinstance(input.first)) { - name = input.first.cast(); - } else { - throw py::type_error("incompatible function arguments!"); - } - OPENVINO_ASSERT(py::isinstance(input.second), "Unable to cast tensor ", name, "!"); - auto tensor = Common::cast_to_tensor(input.second); - result_map[name] = tensor; - } - return result_map; -} - -const Containers::TensorIndexMap cast_to_tensor_index_map(const py::dict& inputs) { - Containers::TensorIndexMap result_map; - for (auto&& input : inputs) { - int idx; - if (py::isinstance(input.first)) { - idx = input.first.cast(); - } else { - throw py::type_error("incompatible function arguments!"); - } - OPENVINO_ASSERT(py::isinstance(input.second), "Unable to cast tensor ", idx, "!"); - auto tensor = Common::cast_to_tensor(input.second); - result_map[idx] = tensor; - } - return result_map; -} - void set_request_tensors(ov::InferRequest& request, const py::dict& inputs) { if (!inputs.empty()) { for (auto&& input : inputs) { @@ -293,67 +343,10 @@ uint32_t get_optimal_number_of_requests(const ov::CompiledModel& actual) { } } -py::dict outputs_to_dict(const std::vector>& outputs, ov::InferRequest& request) { +py::dict outputs_to_dict(InferRequestWrapper& request) { py::dict res; - for (const auto& out : outputs) { - ov::Tensor t{request.get_tensor(out)}; - switch (t.get_element_type()) { - case ov::element::Type_t::i8: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::i16: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::i32: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::i64: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::u8: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::u16: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::u32: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::u64: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::bf16: { - res[py::cast(out)] = py::array(py::dtype("float16"), t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::f16: { - res[py::cast(out)] = py::array(py::dtype("float16"), t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::f32: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::f64: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - case ov::element::Type_t::boolean: { - res[py::cast(out)] = py::array_t(t.get_shape(), t.data()); - break; - } - default: { - break; - } - } + for (const auto& out : request.m_outputs) { + res[py::cast(out)] = array_helpers::array_from_tensor(request.m_request.get_tensor(out)); } return res; } diff --git a/src/bindings/python/src/pyopenvino/core/common.hpp b/src/bindings/python/src/pyopenvino/core/common.hpp index 910d9e55e966ed..de033c3ddf383c 100644 --- a/src/bindings/python/src/pyopenvino/core/common.hpp +++ b/src/bindings/python/src/pyopenvino/core/common.hpp @@ -20,14 +20,20 @@ #include "openvino/runtime/infer_request.hpp" #include "openvino/runtime/tensor.hpp" #include "openvino/pass/serialize.hpp" -#include "pyopenvino/core/containers.hpp" #include "pyopenvino/graph/any.hpp" #include "pyopenvino/graph/ops/constant.hpp" +#include "pyopenvino/core/infer_request.hpp" namespace py = pybind11; namespace Common { +namespace containers { + using TensorIndexMap = std::map; + + const TensorIndexMap cast_to_tensor_index_map(const py::dict& inputs); +}; // namespace containers + namespace values { // Minimum amount of bits for common numpy types. Used to perform checks against OV types. @@ -52,6 +58,8 @@ std::vector get_strides(const py::array& array); py::array as_contiguous(py::array& array, ov::element::Type type); +py::array array_from_tensor(ov::Tensor&& t); + }; // namespace array_helpers template @@ -80,15 +88,11 @@ ov::PartialShape partial_shape_from_list(const py::list& shape); const ov::Tensor& cast_to_tensor(const py::handle& tensor); -const Containers::TensorNameMap cast_to_tensor_name_map(const py::dict& inputs); - -const Containers::TensorIndexMap cast_to_tensor_index_map(const py::dict& inputs); - void set_request_tensors(ov::InferRequest& request, const py::dict& inputs); uint32_t get_optimal_number_of_requests(const ov::CompiledModel& actual); -py::dict outputs_to_dict(const std::vector>& outputs, ov::InferRequest& request); +py::dict outputs_to_dict(InferRequestWrapper& request); ov::pass::Serialize::Version convert_to_version(const std::string& version); diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 9cd0202f32f415..7cca9af077e15a 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -9,13 +9,9 @@ #include "common.hpp" #include "pyopenvino/core/compiled_model.hpp" -#include "pyopenvino/core/containers.hpp" #include "pyopenvino/core/infer_request.hpp" #include "pyopenvino/utils/utils.hpp" -PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap); -PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap); - namespace py = pybind11; void regclass_CompiledModel(py::module m) { diff --git a/src/bindings/python/src/pyopenvino/core/containers.cpp b/src/bindings/python/src/pyopenvino/core/containers.cpp deleted file mode 100644 index 8ee414e007a14f..00000000000000 --- a/src/bindings/python/src/pyopenvino/core/containers.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "pyopenvino/core/containers.hpp" - -#include - -PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap); -PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap); - -namespace py = pybind11; - -namespace Containers { - -void regclass_TensorIndexMap(py::module m) { - py::bind_map(m, "TensorIndexMap"); -} - -void regclass_TensorNameMap(py::module m) { - py::bind_map(m, "TensorNameMap"); -} -} // namespace Containers diff --git a/src/bindings/python/src/pyopenvino/core/containers.hpp b/src/bindings/python/src/pyopenvino/core/containers.hpp deleted file mode 100644 index becf2f717847de..00000000000000 --- a/src/bindings/python/src/pyopenvino/core/containers.hpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -#include - -#include - -namespace py = pybind11; - -namespace Containers { - using TensorIndexMap = std::map; - using TensorNameMap = std::map; - - void regclass_TensorIndexMap(py::module m); - void regclass_TensorNameMap(py::module m); -} diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index ef94b298c9480f..da4bdfa6b076f7 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -360,7 +360,7 @@ void regclass_Core(py::module m) { } std::stringstream str; - str << "Provided python object type " << model_path.get_type().str() + str << "Provided python object type " << py::str(model_path.get_type()) << " isn't supported as 'model' argument."; OPENVINO_THROW(str.str()); }, @@ -583,6 +583,21 @@ void regclass_Core(py::module m) { :type extensions: list[openvino.runtime.Extension] )"); + cls.def("get_available_devices", + &ov::Core::get_available_devices, + py::call_guard(), + R"( + Returns devices available for inference Core objects goes over all registered plugins. + + GIL is released while running this function. + + :returns: A list of devices. The devices are returned as: CPU, GPU.0, GPU.1, GNA... + If there more than one device of specific type, they are enumerated with .# suffix. + Such enumerated device can later be used as a device name in all Core methods like: + compile_model, query_model, set_property and so on. + :rtype: list + )"); + cls.def_property_readonly("available_devices", &ov::Core::get_available_devices, py::call_guard(), diff --git a/src/bindings/python/src/pyopenvino/core/infer_request.cpp b/src/bindings/python/src/pyopenvino/core/infer_request.cpp index eb71fd7f953bcf..8be02e8adb86bb 100644 --- a/src/bindings/python/src/pyopenvino/core/infer_request.cpp +++ b/src/bindings/python/src/pyopenvino/core/infer_request.cpp @@ -11,10 +11,7 @@ #include #include "pyopenvino/core/common.hpp" -#include "pyopenvino/core/containers.hpp" - -PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap); -PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap); +#include "pyopenvino/utils/utils.hpp" namespace py = pybind11; @@ -25,7 +22,7 @@ inline py::dict run_sync_infer(InferRequestWrapper& self) { self.m_request.infer(); *self.m_end_time = Time::now(); } - return Common::outputs_to_dict(self.m_outputs, self.m_request); + return Common::outputs_to_dict(self); } void regclass_InferRequest(py::module m) { @@ -102,7 +99,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_output_tensors", [](InferRequestWrapper& self, const py::dict& outputs) { - auto outputs_map = Common::cast_to_tensor_index_map(outputs); + auto outputs_map = Common::containers::cast_to_tensor_index_map(outputs); for (auto&& output : outputs_map) { self.m_request.set_output_tensor(output.first, output.second); } @@ -119,7 +116,7 @@ void regclass_InferRequest(py::module m) { cls.def( "set_input_tensors", [](InferRequestWrapper& self, const py::dict& inputs) { - auto inputs_map = Common::cast_to_tensor_index_map(inputs); + auto inputs_map = Common::containers::cast_to_tensor_index_map(inputs); for (auto&& input : inputs_map) { self.m_request.set_input_tensor(input.first, input.second); } @@ -647,21 +644,29 @@ void regclass_InferRequest(py::module m) { :rtype: List[openvino.runtime.ConstOutput] )"); - cls.def_property_readonly("inputs", - &InferRequestWrapper::get_input_tensors, - R"( - Gets all input tensors of this InferRequest. - - :rtype: List[openvino.runtime.Tensor] - )"); + cls.def_property_readonly( + "inputs", + [](InferRequestWrapper& self) { + Common::utils::deprecation_warning("inputs", "2024.0", "Please use 'input_tensors' property instead."); + return self.get_input_tensors(); + }, + R"( + Gets all input tensors of this InferRequest. + + :rtype: List[openvino.runtime.Tensor] + )"); - cls.def_property_readonly("outputs", - &InferRequestWrapper::get_output_tensors, - R"( - Gets all output tensors of this InferRequest. - - :rtype: List[openvino.runtime.Tensor] - )"); + cls.def_property_readonly( + "outputs", + [](InferRequestWrapper& self) { + Common::utils::deprecation_warning("outputs", "2024.0", "Please use 'output_tensors' property instead."); + return self.get_output_tensors(); + }, + R"( + Gets all output tensors of this InferRequest. + + :rtype: List[openvino.runtime.Tensor] + )"); cls.def_property_readonly("input_tensors", &InferRequestWrapper::get_input_tensors, @@ -710,7 +715,7 @@ void regclass_InferRequest(py::module m) { cls.def_property_readonly( "results", [](InferRequestWrapper& self) { - return Common::outputs_to_dict(self.m_outputs, self.m_request); + return Common::outputs_to_dict(self); }, R"( Gets all outputs tensors of this InferRequest. diff --git a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp index 7c45761700fca9..96149c7df6d5e3 100644 --- a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp +++ b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp @@ -30,6 +30,7 @@ void regmodule_properties(py::module m) { wrap_property_RW(m_properties, ov::compilation_num_threads, "compilation_num_threads"); wrap_property_RW(m_properties, ov::affinity, "affinity"); wrap_property_RW(m_properties, ov::force_tbb_terminate, "force_tbb_terminate"); + wrap_property_RW(m_properties, ov::enable_mmap, "enable_mmap"); wrap_property_RO(m_properties, ov::supported_properties, "supported_properties"); wrap_property_RO(m_properties, ov::available_devices, "available_devices"); @@ -39,7 +40,6 @@ void regmodule_properties(py::module m) { wrap_property_RO(m_properties, ov::optimal_batch_size, "optimal_batch_size"); wrap_property_RO(m_properties, ov::max_batch_size, "max_batch_size"); wrap_property_RO(m_properties, ov::range_for_async_infer_requests, "range_for_async_infer_requests"); - wrap_property_RW(m_properties, ov::inference_precision, "inference_precision"); // Submodule hint py::module m_hint = @@ -52,14 +52,20 @@ void regmodule_properties(py::module m) { .value("HIGH", ov::hint::Priority::HIGH) .value("DEFAULT", ov::hint::Priority::DEFAULT); + OPENVINO_SUPPRESS_DEPRECATED_START py::enum_(m_hint, "PerformanceMode", py::arithmetic()) .value("UNDEFINED", ov::hint::PerformanceMode::UNDEFINED) .value("LATENCY", ov::hint::PerformanceMode::LATENCY) .value("THROUGHPUT", ov::hint::PerformanceMode::THROUGHPUT) .value("CUMULATIVE_THROUGHPUT", ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT); + OPENVINO_SUPPRESS_DEPRECATED_END + + py::enum_(m_hint, "SchedulingCoreType", py::arithmetic()) + .value("ANY_CORE", ov::hint::SchedulingCoreType::ANY_CORE) + .value("PCORE_ONLY", ov::hint::SchedulingCoreType::PCORE_ONLY) + .value("ECORE_ONLY", ov::hint::SchedulingCoreType::ECORE_ONLY); py::enum_(m_hint, "ExecutionMode", py::arithmetic()) - .value("UNDEFINED", ov::hint::ExecutionMode::UNDEFINED) .value("PERFORMANCE", ov::hint::ExecutionMode::PERFORMANCE) .value("ACCURACY", ov::hint::ExecutionMode::ACCURACY); @@ -67,6 +73,9 @@ void regmodule_properties(py::module m) { wrap_property_RW(m_hint, ov::hint::inference_precision, "inference_precision"); wrap_property_RW(m_hint, ov::hint::model_priority, "model_priority"); wrap_property_RW(m_hint, ov::hint::performance_mode, "performance_mode"); + wrap_property_RW(m_hint, ov::hint::enable_cpu_pinning, "enable_cpu_pinning"); + wrap_property_RW(m_hint, ov::hint::scheduling_core_type, "scheduling_core_type"); + wrap_property_RW(m_hint, ov::hint::enable_hyper_threading, "enable_hyper_threading"); wrap_property_RW(m_hint, ov::hint::execution_mode, "execution_mode"); wrap_property_RW(m_hint, ov::hint::num_requests, "num_requests"); wrap_property_RW(m_hint, ov::hint::model, "model"); @@ -264,4 +273,13 @@ void regmodule_properties(py::module m) { m_streams.def("num", [](const int32_t value) { return ov::streams::num(ov::streams::Num(value)); }); + + // Submodule auto + py::module m_intel_auto = + m_properties.def_submodule("intel_auto", + "openvino.runtime.properties.intel_auto submodule that simulates ov::intel_auto"); + + wrap_property_RW(m_intel_auto, ov::intel_auto::device_bind_buffer, "device_bind_buffer"); + wrap_property_RW(m_intel_auto, ov::intel_auto::enable_startup_fallback, "enable_startup_fallback"); + wrap_property_RW(m_intel_auto, ov::intel_auto::enable_runtime_fallback, "enable_runtime_fallback"); } diff --git a/src/bindings/python/src/pyopenvino/core/properties/properties.hpp b/src/bindings/python/src/pyopenvino/core/properties/properties.hpp index 7625e58e90e209..7437ba8073f019 100644 --- a/src/bindings/python/src/pyopenvino/core/properties/properties.hpp +++ b/src/bindings/python/src/pyopenvino/core/properties/properties.hpp @@ -10,6 +10,7 @@ #include "openvino/runtime/properties.hpp" #include "openvino/runtime/intel_cpu/properties.hpp" #include "openvino/runtime/intel_gpu/properties.hpp" +#include "openvino/runtime/auto/properties.hpp" #include "pyopenvino/core/properties/properties.hpp" namespace py = pybind11; diff --git a/src/bindings/python/src/pyopenvino/frontend/extension.cpp b/src/bindings/python/src/pyopenvino/frontend/extension.cpp index 72b18d58f74d68..57c65af64a4536 100644 --- a/src/bindings/python/src/pyopenvino/frontend/extension.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/extension.cpp @@ -140,4 +140,50 @@ void regclass_frontend_OpExtension(py::module m) { py::arg("fw_type_name"), py::arg("attr_names_map") = std::map(), py::arg("attr_values_map") = std::map()); + + ext.def(py::init([](const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map, + const std::map& attr_values_map) { + std::map any_map; + for (const auto& it : attr_values_map) { + any_map[it.first] = Common::utils::py_object_to_any(it.second); + } + return std::make_shared>(fw_type_name, + in_names_vec, + out_names_vec, + attr_names_map, + any_map); + }), + py::arg("fw_type_name"), + py::arg("in_names_vec"), + py::arg("out_names_vec"), + py::arg("attr_names_map") = std::map(), + py::arg("attr_values_map") = std::map()); + + ext.def(py::init([](const std::string& ov_type_name, + const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map, + const std::map& attr_values_map) { + std::map any_map; + for (const auto& it : attr_values_map) { + any_map[it.first] = Common::utils::py_object_to_any(it.second); + } + + return std::make_shared>(ov_type_name, + fw_type_name, + in_names_vec, + out_names_vec, + attr_names_map, + any_map); + }), + py::arg("ov_type_name"), + py::arg("fw_type_name"), + py::arg("in_names_vec"), + py::arg("out_names_vec"), + py::arg("attr_names_map") = std::map(), + py::arg("attr_values_map") = std::map()); } diff --git a/src/bindings/python/src/pyopenvino/frontend/frontend_module.cmake b/src/bindings/python/src/pyopenvino/frontend/frontend_module.cmake index 33aaa10a6b3b55..d056bbfc1e93b0 100644 --- a/src/bindings/python/src/pyopenvino/frontend/frontend_module.cmake +++ b/src/bindings/python/src/pyopenvino/frontend/frontend_module.cmake @@ -25,7 +25,7 @@ function(frontend_module TARGET FRAMEWORK INSTALL_COMPONENT) target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}" "${OpenVINOPython_SOURCE_DIR}/src/pyopenvino/utils/") - target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime openvino::frontend::${FRAMEWORK}) + target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime openvino::runtime::dev openvino::frontend::${FRAMEWORK}) set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) diff --git a/src/bindings/python/src/pyopenvino/frontend/node_context.cpp b/src/bindings/python/src/pyopenvino/frontend/node_context.cpp index 58a662779f7431..1f0904070ea60d 100644 --- a/src/bindings/python/src/pyopenvino/frontend/node_context.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/node_context.cpp @@ -53,6 +53,31 @@ void regclass_frontend_NodeContext(py::module m) { "NodeContext", py::dynamic_attr()); + auto cast_attribute = [](const ov::Any& any, const py::object& dtype) -> py::object { + CAST_TO_PY(any, dtype, int32_t); + CAST_TO_PY(any, dtype, int64_t); + CAST_TO_PY(any, dtype, bool); + CAST_TO_PY(any, dtype, std::string); + CAST_TO_PY(any, dtype, float); + CAST_TO_PY(any, dtype, double); + CAST_TO_PY(any, dtype, ov::element::Type); + CAST_TO_PY(any, dtype, ov::PartialShape); + + CAST_VEC_TO_PY(any, dtype, std::vector); + CAST_VEC_TO_PY(any, dtype, std::vector); +#ifndef __APPLE__ + // TODO: investigate the issue in pybind11 on MacOS + CAST_VEC_TO_PY(any, dtype, std::vector); +#endif + CAST_VEC_TO_PY(any, dtype, std::vector); + CAST_VEC_TO_PY(any, dtype, std::vector); + CAST_VEC_TO_PY(any, dtype, std::vector); + CAST_VEC_TO_PY(any, dtype, std::vector); + CAST_VEC_TO_PY(any, dtype, std::vector); + + return py::none(); + }; + ext.def( "get_attribute", [=](NodeContext& self, const std::string& name, const py::object& default_value, const py::object& dtype) @@ -60,7 +85,7 @@ void regclass_frontend_NodeContext(py::module m) { auto any = self.get_attribute_as_any(name); auto type = m.attr("Type"); - if (dtype == type) { + if (dtype.is(type)) { if (any.is() || any.is()) { return py::cast(self.get_attribute(name)); } else if (any.is>() || any.is>()) { @@ -68,24 +93,9 @@ void regclass_frontend_NodeContext(py::module m) { } } - CAST_TO_PY(any, dtype, int32_t); - CAST_TO_PY(any, dtype, int64_t); - CAST_TO_PY(any, dtype, bool); - CAST_TO_PY(any, dtype, std::string); - CAST_TO_PY(any, dtype, double); - CAST_TO_PY(any, dtype, ov::element::Type); - CAST_TO_PY(any, dtype, ov::PartialShape); - - CAST_VEC_TO_PY(any, dtype, std::vector); - CAST_VEC_TO_PY(any, dtype, std::vector); -#ifndef __APPLE__ - // TODO: investigate the issue in pybind11 on MacOS - CAST_VEC_TO_PY(any, dtype, std::vector); -#endif - CAST_VEC_TO_PY(any, dtype, std::vector); - CAST_VEC_TO_PY(any, dtype, std::vector); - CAST_VEC_TO_PY(any, dtype, std::vector); - CAST_VEC_TO_PY(any, dtype, std::vector); + auto casted = cast_attribute(any, dtype); + if (!casted.is_none()) + return casted; if (default_value.is_none()) FRONT_END_GENERAL_CHECK(false, "Attribute ", name, " can't be converted to defined types."); @@ -108,6 +118,26 @@ void regclass_frontend_NodeContext(py::module m) { return self.get_input(name, idx); }); + ext.def( + "get_values_from_const_input", + [=](NodeContext& self, int idx, const py::object& default_value, const py::object& dtype) -> py::object { + auto any = self.get_values_from_const_input(idx); + if (any.empty()) + return py::none(); + + auto casted = cast_attribute(any, dtype); + if (!casted.is_none()) + return casted; + + if (default_value.is_none()) + FRONT_END_GENERAL_CHECK(false, "Const input with index ", idx, " can't be converted to defined types."); + else + return default_value; + }, + py::arg("idx"), + py::arg("default_value") = py::none(), + py::arg("dtype") = py::none()); + ext.def("get_input_size", [](NodeContext& self) { return self.get_input_size(); }); diff --git a/src/bindings/python/src/pyopenvino/frontend/paddle/extension.cpp b/src/bindings/python/src/pyopenvino/frontend/paddle/extension.cpp index 75ce233c8aaf34..9140aacd1e1845 100644 --- a/src/bindings/python/src/pyopenvino/frontend/paddle/extension.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/paddle/extension.cpp @@ -3,6 +3,7 @@ // #include "extension.hpp" +#include "utils.hpp" #include #include @@ -10,6 +11,9 @@ #include #include "openvino/frontend/paddle/extension/conversion.hpp" +#include "openvino/frontend/paddle/extension/op.hpp" +#include "openvino/frontend/paddle/frontend.hpp" +#include "openvino/frontend/paddle/node_context.hpp" namespace py = pybind11; @@ -41,3 +45,47 @@ void regclass_frontend_paddle_ConversionExtension(py::module m) { return std::make_shared(op_type, f); })); } + +void regclass_frontend_paddle_OpExtension(py::module m) { + py::class_, std::shared_ptr>, ConversionExtension> ext( + m, + "OpExtensionPaddle", + py::dynamic_attr()); + + ext.def(py::init([](const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map, + const std::map& attr_values_map) { + + std::map any_map; + for (const auto& it : attr_values_map) { + any_map[it.first] = Common::utils::py_object_to_any(it.second); + } + return std::make_shared>(fw_type_name, in_names_vec, out_names_vec, attr_names_map, any_map); + }), py::arg("fw_type_name"), + py::arg("in_names_vec"), + py::arg("out_names_vec"), + py::arg("attr_names_map") = std::map(), + py::arg("attr_values_map") = std::map()); + + ext.def(py::init([](const std::string& ov_type_name, + const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map, + const std::map& attr_values_map) { + + std::map any_map; + for (const auto& it : attr_values_map) { + any_map[it.first] = Common::utils::py_object_to_any(it.second); + } + return std::make_shared>(ov_type_name, fw_type_name, in_names_vec, out_names_vec, attr_names_map, any_map); + }), + py::arg("ov_type_name"), + py::arg("fw_type_name"), + py::arg("in_names_vec"), + py::arg("out_names_vec"), + py::arg("attr_names_map") = std::map(), + py::arg("attr_values_map") = std::map()); +} diff --git a/src/bindings/python/src/pyopenvino/frontend/paddle/extension.hpp b/src/bindings/python/src/pyopenvino/frontend/paddle/extension.hpp index 533081e0d15291..4c322ec0358f00 100644 --- a/src/bindings/python/src/pyopenvino/frontend/paddle/extension.hpp +++ b/src/bindings/python/src/pyopenvino/frontend/paddle/extension.hpp @@ -9,3 +9,4 @@ namespace py = pybind11; void regclass_frontend_paddle_ConversionExtension(py::module m); +void regclass_frontend_paddle_OpExtension(py::module m); diff --git a/src/bindings/python/src/pyopenvino/frontend/paddle/py_module.cpp b/src/bindings/python/src/pyopenvino/frontend/paddle/py_module.cpp index 880bd36aeec414..f63cf7b33b13cd 100644 --- a/src/bindings/python/src/pyopenvino/frontend/paddle/py_module.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/paddle/py_module.cpp @@ -12,4 +12,5 @@ namespace py = pybind11; PYBIND11_MODULE(py_paddle_frontend, m) { regclass_frontend_paddle_ConversionExtension(m); + regclass_frontend_paddle_OpExtension(m); } diff --git a/src/bindings/python/src/pyopenvino/frontend/place.cpp b/src/bindings/python/src/pyopenvino/frontend/place.cpp index 32dc0ff07ba35b..57c51969a3c40c 100644 --- a/src/bindings/python/src/pyopenvino/frontend/place.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/place.cpp @@ -75,14 +75,14 @@ void regclass_frontend_Place(py::module m) { place.def( "get_consuming_operations", [](const ov::frontend::Place& self, py::object outputName, py::object outputPortIndex) { - if (outputName == py::none()) { - if (outputPortIndex == py::none()) { + if (outputName.is(py::none())) { + if (outputPortIndex.is(py::none())) { return self.get_consuming_operations(); } else { return self.get_consuming_operations(py::cast(outputPortIndex)); } } else { - if (outputPortIndex == py::none()) { + if (outputPortIndex.is(py::none())) { return self.get_consuming_operations(py::cast(outputName)); } else { return self.get_consuming_operations(py::cast(outputName), @@ -108,14 +108,14 @@ void regclass_frontend_Place(py::module m) { place.def( "get_target_tensor", [](const ov::frontend::Place& self, py::object outputName, py::object outputPortIndex) { - if (outputName == py::none()) { - if (outputPortIndex == py::none()) { + if (outputName.is(py::none())) { + if (outputPortIndex.is(py::none())) { return self.get_target_tensor(); } else { return self.get_target_tensor(py::cast(outputPortIndex)); } } else { - if (outputPortIndex == py::none()) { + if (outputPortIndex.is(py::none())) { return self.get_target_tensor(py::cast(outputName)); } else { return self.get_target_tensor(py::cast(outputName), py::cast(outputPortIndex)); @@ -140,14 +140,14 @@ void regclass_frontend_Place(py::module m) { place.def( "get_producing_operation", [](const ov::frontend::Place& self, py::object inputName, py::object inputPortIndex) { - if (inputName == py::none()) { - if (inputPortIndex == py::none()) { + if (inputName.is(py::none())) { + if (inputPortIndex.is(py::none())) { return self.get_producing_operation(); } else { return self.get_producing_operation(py::cast(inputPortIndex)); } } else { - if (inputPortIndex == py::none()) { + if (inputPortIndex.is(py::none())) { return self.get_producing_operation(py::cast(inputName)); } else { return self.get_producing_operation(py::cast(inputName), @@ -181,14 +181,14 @@ void regclass_frontend_Place(py::module m) { place.def( "get_input_port", [](const ov::frontend::Place& self, py::object inputName, py::object inputPortIndex) { - if (inputName == py::none()) { - if (inputPortIndex == py::none()) { + if (inputName.is(py::none())) { + if (inputPortIndex.is(py::none())) { return self.get_input_port(); } else { return self.get_input_port(py::cast(inputPortIndex)); } } else { - if (inputPortIndex == py::none()) { + if (inputPortIndex.is(py::none())) { return self.get_input_port(py::cast(inputName)); } else { return self.get_input_port(py::cast(inputName), py::cast(inputPortIndex)); @@ -211,14 +211,14 @@ void regclass_frontend_Place(py::module m) { place.def( "get_output_port", [](const ov::frontend::Place& self, py::object outputName, py::object outputPortIndex) { - if (outputName == py::none()) { - if (outputPortIndex == py::none()) { + if (outputName.is(py::none())) { + if (outputPortIndex.is(py::none())) { return self.get_output_port(); } else { return self.get_output_port(py::cast(outputPortIndex)); } } else { - if (outputPortIndex == py::none()) { + if (outputPortIndex.is(py::none())) { return self.get_output_port(py::cast(outputName)); } else { return self.get_output_port(py::cast(outputName), py::cast(outputPortIndex)); @@ -250,14 +250,14 @@ void regclass_frontend_Place(py::module m) { place.def( "get_source_tensor", [](const ov::frontend::Place& self, py::object inputName, py::object inputPortIndex) { - if (inputName == py::none()) { - if (inputPortIndex == py::none()) { + if (inputName.is(py::none())) { + if (inputPortIndex.is(py::none())) { return self.get_source_tensor(); } else { return self.get_source_tensor(py::cast(inputPortIndex)); } } else { - if (inputPortIndex == py::none()) { + if (inputPortIndex.is(py::none())) { return self.get_source_tensor(py::cast(inputName)); } else { return self.get_source_tensor(py::cast(inputName), py::cast(inputPortIndex)); diff --git a/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp b/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp index 583b42f4950578..b762c464eb747a 100644 --- a/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp +++ b/src/bindings/python/src/pyopenvino/frontend/pytorch/decoder.hpp @@ -26,6 +26,10 @@ class PyDecoder : public ov::frontend::pytorch::TorchDecoder { PYBIND11_OVERRIDE_PURE(const std::string&, TorchDecoder, get_input_debug_name, index); } + const std::string& get_input_signature_name(size_t index) const override { + PYBIND11_OVERRIDE_PURE(const std::string&, TorchDecoder, get_input_signature_name, index); + } + ov::PartialShape get_input_shape(size_t index) const override { PYBIND11_OVERRIDE_PURE(ov::PartialShape, TorchDecoder, get_input_shape, index); } diff --git a/src/bindings/python/src/pyopenvino/frontend/pytorch/extension.cpp b/src/bindings/python/src/pyopenvino/frontend/pytorch/extension.cpp new file mode 100644 index 00000000000000..6f64da9b1b6dc5 --- /dev/null +++ b/src/bindings/python/src/pyopenvino/frontend/pytorch/extension.cpp @@ -0,0 +1,78 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "extension.hpp" +#include "utils.hpp" + +#include +#include +#include +#include + +#include "openvino/frontend/extension/conversion.hpp" +#include "openvino/frontend/pytorch/extension/conversion.hpp" +#include "openvino/frontend/pytorch/extension/op.hpp" + +namespace py = pybind11; + +using namespace ov::frontend::pytorch; + + +void regclass_frontend_pytorch_ConversionExtension(py::module m) { + py::class_ _ext( + m, + "_ConversionExtensionPytorch", + py::dynamic_attr()); + class PyConversionExtension : public ConversionExtension { + public: + using Ptr = std::shared_ptr; + using PyCreatorFunction = std::function; + PyConversionExtension(const std::string& op_type, const PyCreatorFunction& f) + : ConversionExtension(op_type, [f](const ov::frontend::NodeContext& node) -> ov::OutputVector { + return f(static_cast(&node)); + }) {} + }; + py::class_ ext( + m, + "ConversionExtensionPytorch", + py::dynamic_attr()); + + ext.def(py::init([](const std::string& op_type, const PyConversionExtension::PyCreatorFunction& f) { + return std::make_shared(op_type, f); + })); +} + +void regclass_frontend_pytorch_OpExtension(py::module m) { + py::class_, std::shared_ptr>, ConversionExtension> ext( + m, + "OpExtensionPytorch", + py::dynamic_attr()); + + ext.def(py::init([](const std::string& fw_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map) { + std::map any_map; + for (const auto& it : attr_values_map) { + any_map[it.first] = Common::utils::py_object_to_any(it.second); + } + return std::make_shared>(fw_type_name, attr_names_map, any_map); + }), py::arg("fw_type_name"), + py::arg("attr_names_map") = std::map(), + py::arg("attr_values_map") = std::map()); + + ext.def(py::init([](const std::string& ov_type_name, + const std::string& fw_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map) { + std::map any_map; + for (const auto& it : attr_values_map) { + any_map[it.first] = Common::utils::py_object_to_any(it.second); + } + return std::make_shared>(ov_type_name, fw_type_name, attr_names_map, any_map); + }), + py::arg("ov_type_name"), + py::arg("fw_type_name"), + py::arg("attr_names_map") = std::map(), + py::arg("attr_values_map") = std::map()); +} diff --git a/src/bindings/python/src/pyopenvino/frontend/pytorch/extension.hpp b/src/bindings/python/src/pyopenvino/frontend/pytorch/extension.hpp new file mode 100644 index 00000000000000..797d6d81793440 --- /dev/null +++ b/src/bindings/python/src/pyopenvino/frontend/pytorch/extension.hpp @@ -0,0 +1,12 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +namespace py = pybind11; + +void regclass_frontend_pytorch_ConversionExtension(py::module m); +void regclass_frontend_pytorch_OpExtension(py::module m); diff --git a/src/bindings/python/src/pyopenvino/frontend/pytorch/py_module.cpp b/src/bindings/python/src/pyopenvino/frontend/pytorch/py_module.cpp index 185f91ef59f542..a2e3a2c7639342 100644 --- a/src/bindings/python/src/pyopenvino/frontend/pytorch/py_module.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/pytorch/py_module.cpp @@ -5,9 +5,12 @@ #include #include "decoder.hpp" +#include "extension.hpp" namespace py = pybind11; PYBIND11_MODULE(py_pytorch_frontend, m) { regclass_frontend_pytorch_decoder(m); + regclass_frontend_pytorch_ConversionExtension(m); + regclass_frontend_pytorch_OpExtension(m); } diff --git a/src/bindings/python/src/pyopenvino/graph/any.cpp b/src/bindings/python/src/pyopenvino/graph/any.cpp index 19c9e3c52cb111..a53fffac102119 100644 --- a/src/bindings/python/src/pyopenvino/graph/any.cpp +++ b/src/bindings/python/src/pyopenvino/graph/any.cpp @@ -5,12 +5,19 @@ #include "openvino/core/any.hpp" #include +#include #include "pyopenvino/graph/any.hpp" #include "pyopenvino/utils/utils.hpp" namespace py = pybind11; +namespace { +bool check_key(py::object key, py::object obj) { + return key.is(py::type::of(obj)); +} +}; // namespace + void regclass_graph_Any(py::module m) { py::class_> ov_any(m, "OVAny"); @@ -23,9 +30,7 @@ void regclass_graph_Any(py::module m) { })); ov_any.def("__repr__", [](const ov::Any& self) { - std::stringstream ret; - self.print(ret); - return ret.str(); + return ""; }); ov_any.def("__hash__", [](ov::Any& self) { @@ -62,6 +67,72 @@ void regclass_graph_Any(py::module m) { ov_any.def("__eq__", [](const ov::Any& a, py::object& b) -> bool { return a == ov::Any(Common::utils::py_object_to_any(b)); }); + ov_any.def( + "astype", + [](ov::Any& self, py::object dtype) { + if (check_key(dtype, py::bool_())) { + return py::cast(self.as()); + } else if (check_key(dtype, py::str())) { + return py::cast(self.as()); + } else if (check_key(dtype, py::int_())) { + return py::cast(self.as()); + } else if (check_key(dtype, py::float_())) { + return py::cast(self.as()); + } else if (check_key(dtype, py::dict())) { + return Common::utils::from_ov_any_map_no_leaves(self); + } + std::stringstream str; + str << "Unsupported data type : '" << dtype << "' is passed as an argument."; + OPENVINO_THROW(str.str()); + }, + R"( + Returns runtime attribute casted to defined data type. + + :param dtype: Data type in which runtime attribute will be casted. + :type dtype: Union[bool, int, str, float, dict] + + :return: A runtime attribute. + :rtype: Any + )"); + ov_any.def( + "aslist", + [](ov::Any& self, py::object dtype) { + // before serialization + if (self.is() || dtype.is_none()) { + return py::cast(py::list()); + } else if (self.is>()) { + return py::cast(self.as>()); + } else if (self.is>()) { + return py::cast(self.as>()); + } else if (self.is>()) { + return py::cast(self.as>()); + } else if (self.is>()) { + return py::cast(self.as>()); + } + // after serialization + if (check_key(dtype, py::str())) { + return py::cast(self.as>()); + } else if (check_key(dtype, py::int_())) { + return py::cast(self.as>()); + } else if (check_key(dtype, py::float_())) { + return py::cast(self.as>()); + } else if (check_key(dtype, py::bool_())) { + return py::cast(self.as>()); + } + std::stringstream str; + str << "Unsupported data type : '" << dtype << "' is passed as an argument."; + OPENVINO_THROW(str.str()); + }, + py::arg("dtype") = py::none(), + R"( + Returns runtime attribute as a list with specified data type. + + :param dtype: Data type of a list in which runtime attribute will be casted. + :type dtype: Union[bool, int, str, float] + + :return: A runtime attribute as a list. + :rtype: Union[List[float], List[int], List[str], List[bool]] + )"); ov_any.def( "get", [](const ov::Any& self) -> py::object { diff --git a/src/bindings/python/src/pyopenvino/graph/axis_set.cpp b/src/bindings/python/src/pyopenvino/graph/axis_set.cpp index 6799c2f368f1b2..9ff932b0ee54b9 100644 --- a/src/bindings/python/src/pyopenvino/graph/axis_set.cpp +++ b/src/bindings/python/src/pyopenvino/graph/axis_set.cpp @@ -36,7 +36,7 @@ void regclass_graph_AxisSet(py::module m) { axis_set.def("__repr__", [](const ov::AxisSet& self) -> std::string { std::stringstream data_ss; - std::copy(self.begin(), self.end(), std::ostream_iterator(data_ss, ", ")); + std::copy(self.begin(), self.end(), std::ostream_iterator(data_ss, ", ")); std::string data_str = data_ss.str(); return ""; }); diff --git a/src/bindings/python/src/pyopenvino/graph/coordinate_diff.cpp b/src/bindings/python/src/pyopenvino/graph/coordinate_diff.cpp index dae14a7100afbc..88e71bac5f01d1 100644 --- a/src/bindings/python/src/pyopenvino/graph/coordinate_diff.cpp +++ b/src/bindings/python/src/pyopenvino/graph/coordinate_diff.cpp @@ -24,7 +24,7 @@ void regclass_graph_CoordinateDiff(py::module m) { coordinate_diff.def("__str__", [](const ov::CoordinateDiff& self) -> std::string { std::stringstream stringstream; - std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); + std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); std::string string = stringstream.str(); return string.substr(0, string.size() - 2); }); diff --git a/src/bindings/python/src/pyopenvino/graph/dimension.cpp b/src/bindings/python/src/pyopenvino/graph/dimension.cpp index fc98155c95ae36..b8e6241bad4a07 100644 --- a/src/bindings/python/src/pyopenvino/graph/dimension.cpp +++ b/src/bindings/python/src/pyopenvino/graph/dimension.cpp @@ -91,6 +91,15 @@ void regclass_graph_Dimension(py::module m) { Return this dimension's min_dimension as integer. This dimension must be dynamic and non-negative. + :return: Value of the dimension. + :rtype: int + )"); + dim.def_property_readonly("min_length", + &ov::Dimension::get_min_length, + R"( + Return this dimension's min_dimension as integer. + This dimension must be dynamic and non-negative. + :return: Value of the dimension. :rtype: int )"); @@ -103,7 +112,15 @@ void regclass_graph_Dimension(py::module m) { :return: Value of the dimension. :rtype: int )"); + dim.def_property_readonly("max_length", + &ov::Dimension::get_max_length, + R"( + Return this dimension's max_dimension as integer. + This dimension must be dynamic and non-negative. + :return: Value of the dimension. + :rtype: int + )"); dim.def("same_scheme", &ov::Dimension::same_scheme, py::arg("dim"), diff --git a/src/bindings/python/src/pyopenvino/graph/discrete_type_info.cpp b/src/bindings/python/src/pyopenvino/graph/discrete_type_info.cpp index 74b81eb62a45cf..cd19fa2a121cbc 100644 --- a/src/bindings/python/src/pyopenvino/graph/discrete_type_info.cpp +++ b/src/bindings/python/src/pyopenvino/graph/discrete_type_info.cpp @@ -9,6 +9,7 @@ #include #include "openvino/core/type.hpp" +#include "pyopenvino/utils/utils.hpp" namespace py = pybind11; @@ -25,20 +26,18 @@ void regclass_graph_DiscreteTypeInfo(py::module m) { discrete_type_info.def(py::self != py::self); discrete_type_info.def_readonly("name", &ov::DiscreteTypeInfo::name); - discrete_type_info.def_readonly("version", &ov::DiscreteTypeInfo::version); discrete_type_info.def_readonly("version_id", &ov::DiscreteTypeInfo::version_id); discrete_type_info.def_readonly("parent", &ov::DiscreteTypeInfo::parent); - discrete_type_info.def("get_version", &ov::DiscreteTypeInfo::get_version); discrete_type_info.def("hash", [](const ov::DiscreteTypeInfo& self) { return self.hash(); }); discrete_type_info.def("__repr__", [](const ov::DiscreteTypeInfo& self) { std::string name = std::string(self.name); - std::string version = std::to_string(self.version); + std::string version = std::string(self.version_id); if (self.parent != nullptr) { - std::string parent_version = std::to_string(self.parent->version); + std::string parent_version = std::string(self.parent->version_id); std::string parent_name = self.parent->name; return ""; diff --git a/src/bindings/python/src/pyopenvino/graph/model.cpp b/src/bindings/python/src/pyopenvino/graph/model.cpp index 6c898713f0647e..40ed6876739bc8 100644 --- a/src/bindings/python/src/pyopenvino/graph/model.cpp +++ b/src/bindings/python/src/pyopenvino/graph/model.cpp @@ -347,7 +347,7 @@ void regclass_graph_Model(py::module m) { } else if (py::isinstance>(item.first)) { new_shape.first = item.first.cast>(); } else { - throw py::type_error("Incorrect key type " + std::string(item.first.get_type().str()) + + throw py::type_error("Incorrect key type " + std::string(py::str(item.first.get_type())) + " to reshape a model, expected keys as openvino.runtime.Output, int or str."); } // check values @@ -359,7 +359,7 @@ void regclass_graph_Model(py::module m) { new_shape.second = ov::PartialShape(item.second.cast()); } else { throw py::type_error( - "Incorrect value type " + std::string(item.second.get_type().str()) + + "Incorrect value type " + std::string(py::str(item.second.get_type())) + " to reshape a model, expected values as openvino.runtime.PartialShape, str, list or tuple."); } new_shapes.insert(new_shape); @@ -480,6 +480,14 @@ void regclass_graph_Model(py::module m) { :return: ParameterVector containing model parameters. :rtype: ParameterVector )"); + model.def_property_readonly("parameters", + &ov::Model::get_parameters, + R"( + Return the model parameters. + + :return: ParameterVector containing model parameters. + :rtype: ParameterVector + )"); model.def("get_results", &ov::Model::get_results, R"( @@ -488,6 +496,14 @@ void regclass_graph_Model(py::module m) { :return: ResultVector containing model parameters. :rtype: ResultVector )"); + model.def_property_readonly("results", + &ov::Model::get_results, + R"( + Return a list of model outputs. + + :return: ResultVector containing model parameters. + :rtype: ResultVector + )"); model.def("get_result", &ov::Model::get_result, R"( @@ -496,6 +512,14 @@ void regclass_graph_Model(py::module m) { :return: Node object representing result. :rtype: openvino.runtime.Node )"); + model.def_property_readonly("result", + &ov::Model::get_result, + R"( + Return single result. + + :return: Node object representing result. + :rtype: openvino.runtime.Node + )"); model.def("get_result_index", (int64_t(ov::Model::*)(const ov::Output&) const) & ov::Model::get_result_index, py::arg("value"), @@ -561,6 +585,14 @@ void regclass_graph_Model(py::module m) { :rtype: bool )"); + model.def_property_readonly("dynamic", + &ov::Model::is_dynamic, + R"( + Returns true if any of the op's defined in the model + contains partial shape. + + :rtype: bool + )"); model.def("input", (ov::Output(ov::Model::*)()) & ov::Model::input); model.def("input", (ov::Output(ov::Model::*)(size_t)) & ov::Model::input, py::arg("index")); @@ -742,47 +774,41 @@ void regclass_graph_Model(py::module m) { for (size_t i = 0; i < path.size(); i++) { cpp_args[i] = path[i].cast(); } - return Common::utils::from_ov_any(self.get_rt_info(cpp_args)); + return py::cast(self.get_rt_info(cpp_args)); }, py::arg("path"), R"( - Returns runtime attribute. + Returns runtime attribute as a OVAny object. :param path: List of strings which defines a path to runtime info. :type path: List[str] :return: A runtime attribute. - :rtype: Any + :rtype: openvino.runtime.OVAny )"); model.def( "get_rt_info", [](const ov::Model& self, const py::str& path) -> py::object { - return Common::utils::from_ov_any(self.get_rt_info(path.cast())); + return py::cast(self.get_rt_info(path.cast())); }, py::arg("path"), R"( - Returns runtime attribute. + Returns runtime attribute as a OVAny object. :param path: List of strings which defines a path to runtime info. :type path: str :return: A runtime attribute. - :rtype: Any + :rtype: openvino.runtime.OVAny )"); model.def( "has_rt_info", [](const ov::Model& self, const py::list& path) -> bool { - // FIXME: understand why has_rt_info causes Python crash - try { - std::vector cpp_args(path.size()); - for (size_t i = 0; i < path.size(); i++) { - cpp_args[i] = path[i].cast(); - } - self.get_rt_info(cpp_args); - return true; - } catch (ov::Exception&) { - return false; + std::vector cpp_args(path.size()); + for (size_t i = 0; i < path.size(); i++) { + cpp_args[i] = path[i].cast(); } + return self.has_rt_info(cpp_args); }, py::arg("path"), R"( diff --git a/src/bindings/python/src/pyopenvino/graph/node.cpp b/src/bindings/python/src/pyopenvino/graph/node.cpp index 599194ab9b96f0..1e99bff44210e0 100644 --- a/src/bindings/python/src/pyopenvino/graph/node.cpp +++ b/src/bindings/python/src/pyopenvino/graph/node.cpp @@ -304,14 +304,6 @@ void regclass_graph_Node(py::module m) { :return: A dictionary of user defined data. :rtype: openvino.runtime.RTMap )"); - node.def("get_version", - &ov::Node::get_version, - R"( - Returns operation's version of the node. - - :return: Operation version. - :rtype: int - )"); node.def("set_argument", &ov::Node::set_argument); node.def("set_arguments", [](const std::shared_ptr& self, const ov::NodeVector& args) { @@ -326,7 +318,6 @@ void regclass_graph_Node(py::module m) { node.def_property_readonly("rt_info", (PyRTMap & (ov::Node::*)()) & ov::Node::get_rt_info, py::return_value_policy::reference_internal); - node.def_property_readonly("version", &ov::Node::get_version); node.def_property_readonly("type_info", &ov::Node::get_type_info); node.def_property("friendly_name", &ov::Node::get_friendly_name, &ov::Node::set_friendly_name); diff --git a/src/bindings/python/src/pyopenvino/graph/node_factory.cpp b/src/bindings/python/src/pyopenvino/graph/node_factory.cpp index bdf7c982b3e8fd..9aed62c2e00a17 100644 --- a/src/bindings/python/src/pyopenvino/graph/node_factory.cpp +++ b/src/bindings/python/src/pyopenvino/graph/node_factory.cpp @@ -79,7 +79,7 @@ class NodeFactory { return it->second(); } - const ov::OpSet& m_opset = ov::get_opset10(); + const ov::OpSet& m_opset = ov::get_opset11(); std::unordered_map> m_variables; }; } // namespace diff --git a/src/bindings/python/src/pyopenvino/graph/rt_map.cpp b/src/bindings/python/src/pyopenvino/graph/rt_map.cpp index 5985c87f06136f..e666b3972e605c 100644 --- a/src/bindings/python/src/pyopenvino/graph/rt_map.cpp +++ b/src/bindings/python/src/pyopenvino/graph/rt_map.cpp @@ -10,6 +10,7 @@ #include #include "dict_attribute_visitor.hpp" +#include "meta_data.hpp" #include "openvino/core/node.hpp" #include "openvino/core/runtime_attribute.hpp" #include "openvino/op/add.hpp" @@ -27,11 +28,48 @@ using PyRTMap = ov::RTMap; PYBIND11_MAKE_OPAQUE(PyRTMap); +// Create our custom iterator to return python object not OVAny itself. +class PyRTMapIterator { +public: + PyRTMapIterator(const PyRTMap& py_rt_map, py::object ref, bool is_value) + : py_rt_map(py_rt_map), + is_value(is_value), + ref(ref), + it(py_rt_map.cbegin()) {} + + py::object next() { + if (it == py_rt_map.end()) { + throw py::stop_iteration(); + } + const auto result = *it; + it++; + if (is_value) { + return Common::utils::from_ov_any_no_leaves(result.second); + } else { + std::pair res = {result.first, + Common::utils::from_ov_any_no_leaves(result.second)}; + return py::cast(res); + } + } + + const PyRTMap& py_rt_map; + bool is_value = false; + py::object ref; // keep a reference + std::map::const_iterator it; +}; + void regclass_graph_PyRTMap(py::module m) { auto py_map = py::class_(m, "RTMap"); py_map.doc() = "openvino.runtime.RTMap makes bindings for std::map, which can later be used as ov::Node::RTMap"; + py::class_(m, "Iterator") + .def("__iter__", + [](PyRTMapIterator& it) -> PyRTMapIterator& { + return it; + }) + .def("__next__", &PyRTMapIterator::next); + py_map.def("__setitem__", [](PyRTMap& m, const std::string& k, const std::string v) { m[k] = v; }); @@ -39,7 +77,7 @@ void regclass_graph_PyRTMap(py::module m) { m[k] = v; }); py_map.def("__getitem__", [](PyRTMap& m, const std::string& k) -> py::object { - return Common::utils::from_ov_any(m[k]); + return Common::utils::from_ov_any_no_leaves(m[k]); }); py_map.def( "__bool__", @@ -50,20 +88,28 @@ void regclass_graph_PyRTMap(py::module m) { py_map.def( "__iter__", - [](PyRTMap& m) { - return py::make_key_iterator(m.begin(), m.end()); + [](PyRTMap& rt_map) { + return py::make_key_iterator(rt_map.begin(), rt_map.end()); }, py::keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ ); py_map.def( - "items", - [](PyRTMap& m) { - return py::make_iterator(m.begin(), m.end()); + "keys", + [](PyRTMap& rt_map) { + return py::make_key_iterator(rt_map.begin(), rt_map.end()); }, py::keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ ); + py_map.def("items", [](py::object rt_map) { + return PyRTMapIterator(rt_map.cast(), rt_map, false); + }); + + py_map.def("values", [](py::object rt_map) { + return PyRTMapIterator(rt_map.cast(), rt_map, true); + }); + py_map.def("__contains__", [](PyRTMap& m, const std::string& k) -> bool { auto it = m.find(k); if (it == m.end()) diff --git a/src/bindings/python/src/pyopenvino/graph/strides.cpp b/src/bindings/python/src/pyopenvino/graph/strides.cpp index 772ca6047e6429..202dbef3fd5bd1 100644 --- a/src/bindings/python/src/pyopenvino/graph/strides.cpp +++ b/src/bindings/python/src/pyopenvino/graph/strides.cpp @@ -24,7 +24,7 @@ void regclass_graph_Strides(py::module m) { strides.def("__str__", [](const ov::Strides& self) -> std::string { std::stringstream stringstream; - std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); + std::copy(self.begin(), self.end(), std::ostream_iterator(stringstream, ", ")); std::string string = stringstream.str(); return string.substr(0, string.size() - 2); }); diff --git a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp index 0123dd780e2c1c..41524dfd1b6bad 100644 --- a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp +++ b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp @@ -68,11 +68,17 @@ void regclass_graph_Type(py::module m) { type.def("is_static", &ov::element::Type::is_static); type.def("is_dynamic", &ov::element::Type::is_dynamic); type.def("is_real", &ov::element::Type::is_real); + type.def_property_readonly("real", &ov::element::Type::is_real); type.def("is_integral", &ov::element::Type::is_integral); + type.def_property_readonly("integral", &ov::element::Type::is_integral); type.def("is_integral_number", &ov::element::Type::is_integral_number); + type.def_property_readonly("integral_number", &ov::element::Type::is_integral_number); type.def("is_signed", &ov::element::Type::is_signed); + type.def_property_readonly("signed", &ov::element::Type::is_signed); type.def("is_quantized", &ov::element::Type::is_quantized); + type.def_property_readonly("quantized", &ov::element::Type::is_quantized); type.def("get_type_name", &ov::element::Type::get_type_name); + type.def_property_readonly("type_name", &ov::element::Type::get_type_name); type.def("compatible", &ov::element::Type::compatible, py::arg("other"), @@ -121,5 +127,7 @@ void regclass_graph_Type(py::module m) { )"); type.def_property_readonly("size", &ov::element::Type::size); + type.def("get_size", &ov::element::Type::size); type.def_property_readonly("bitwidth", &ov::element::Type::bitwidth); + type.def("get_bitwidth", &ov::element::Type::bitwidth); } diff --git a/src/bindings/python/src/pyopenvino/graph/util.cpp b/src/bindings/python/src/pyopenvino/graph/util.cpp index 35ea9003c70eb1..affc8ff6b47e91 100644 --- a/src/bindings/python/src/pyopenvino/graph/util.cpp +++ b/src/bindings/python/src/pyopenvino/graph/util.cpp @@ -27,6 +27,7 @@ inline void* numpy_to_c(py::array a) { void regmodule_graph_util(py::module m) { py::module mod = m.def_submodule("util", "openvino.runtime.utils"); mod.def("numpy_to_c", &numpy_to_c); + OPENVINO_SUPPRESS_DEPRECATED_START mod.def("get_constant_from_source", &ov::get_constant_from_source, py::arg("output"), @@ -40,6 +41,7 @@ void regmodule_graph_util(py::module m) { from the resulting bound, otherwise Null. :rtype: openvino.runtime.op.Constant or openvino.runtime.Node )"); + OPENVINO_SUPPRESS_DEPRECATED_END mod.def( "clone_model", [](ov::Model& model) { diff --git a/src/bindings/python/src/pyopenvino/pyopenvino.cpp b/src/bindings/python/src/pyopenvino/pyopenvino.cpp index a229f9eaa7d72e..0f2cdf38278010 100644 --- a/src/bindings/python/src/pyopenvino/pyopenvino.cpp +++ b/src/bindings/python/src/pyopenvino/pyopenvino.cpp @@ -24,7 +24,6 @@ #endif #include "pyopenvino/core/async_infer_queue.hpp" #include "pyopenvino/core/compiled_model.hpp" -#include "pyopenvino/core/containers.hpp" #include "pyopenvino/core/core.hpp" #include "pyopenvino/core/extension.hpp" #include "pyopenvino/core/infer_request.hpp" @@ -210,9 +209,6 @@ PYBIND11_MODULE(_pyopenvino, m) { regclass_Core(m); regclass_Tensor(m); - // Registering specific types of containers - Containers::regclass_TensorIndexMap(m); - Containers::regclass_TensorNameMap(m); regclass_CompiledModel(m); regclass_InferRequest(m); diff --git a/src/bindings/python/src/pyopenvino/utils/utils.cpp b/src/bindings/python/src/pyopenvino/utils/utils.cpp index f029323e35ab09..e8c55d5eb7b75c 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.cpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.cpp @@ -12,6 +12,7 @@ #include #include "Python.h" +#include "meta_data.hpp" #include "openvino/core/except.hpp" #include "openvino/frontend/decoder.hpp" @@ -20,12 +21,57 @@ using Version = ov::pass::Serialize::Version; namespace Common { namespace utils { +// For complex structure if an element isn't map, then just cast it to OVAny +py::object from_ov_any_no_leaves(const ov::Any& any) { + if (any.is>() || any.is()) { + return Common::utils::from_ov_any_map_no_leaves(any); + } else { + return py::cast(any); + } +} + +// Recursively go through dict to unwrap nested dicts and keep leaves as OVAny. +py::object from_ov_any_map_no_leaves(const ov::Any& any) { + const auto traverse_map = [](const ov::AnyMap& map) { + const auto unwrap_only_maps = [](const ov::Any& any) { + if (any.is>()) { + const ov::AnyMap& as_map = *any.as>(); + return from_ov_any_map_no_leaves(as_map); + } else if (any.is()) { + return from_ov_any_map_no_leaves(any.as()); + } + return py::cast(any); + }; + + std::map result; + for (const auto& entry : map) { + result[entry.first] = unwrap_only_maps(entry.second); + } + return py::cast(result); + }; + + if (any.is>()) { + const ov::AnyMap& as_map = *any.as>(); + return traverse_map(as_map); + } else if (any.is()) { + return traverse_map(any.as()); + } + OPENVINO_THROW("Only ov::AnyMap or ov::Meta are expected here."); +} + +py::object from_ov_any_map(const ov::AnyMap& map) { + std::map result; + for (const auto& entry : map) { + result[entry.first] = from_ov_any(entry.second); + } + return py::cast(result); +} + py::object from_ov_any(const ov::Any& any) { // Check for py::object if (any.is()) { return any.as(); - } - // Check for std::string + } // Check for std::string else if (any.is()) { return py::cast(any.as().c_str()); } @@ -98,6 +144,9 @@ py::object from_ov_any(const ov::Any& any) { // Check for std::map else if (any.is>()) { return py::cast(any.as>()); + } // Check for ov::AnyMap (std::map) + else if (any.is()) { + return from_ov_any_map(any.as()); } // Check for std::map { else if (any.is>()) { @@ -113,12 +162,17 @@ py::object from_ov_any(const ov::Any& any) { PyDict_SetItemString(dict, property_name.c_str(), PyUnicode_FromString(mutability.c_str())); } return py::cast(dict); + } else if (any.is>()) { + const ov::AnyMap& as_map = *any.as>(); + return from_ov_any_map(as_map); } else if (any.is()) { return py::cast(any.as()); } else if (any.is()) { return py::cast(any.as()); } else if (any.is()) { return py::cast(any.as()); + } else if (any.is()) { + return py::cast(any.as()); } else if (any.is()) { return py::cast(any.as()); } else if (any.is()) { @@ -161,7 +215,7 @@ std::string convert_path_to_string(const py::object& path) { py::object Path = py::module_::import("pathlib").attr("Path"); // check if model path is either a string or pathlib.Path if (py::isinstance(path, Path) || py::isinstance(path)) { - return path.str(); + return py::str(path); } // Convert bytes to string if (py::isinstance(path)) { @@ -258,9 +312,8 @@ ov::Any py_object_to_any(const py::object& py_obj) { } } - // In case of empty vector works like with vector of strings if (_list.empty()) - return _list.cast>(); + return ov::Any(EmptyList()); switch (detected_type) { case PY_TYPE::STR: @@ -285,6 +338,8 @@ ov::Any py_object_to_any(const py::object& py_obj) { return py::cast(py_obj); } else if (py::isinstance(py_obj)) { return py::cast(py_obj); + } else if (py::isinstance(py_obj)) { + return py::cast(py_obj); } else if (py::isinstance(py_obj)) { return py::cast(py_obj); } else if (py::isinstance(py_obj)) { diff --git a/src/bindings/python/src/pyopenvino/utils/utils.hpp b/src/bindings/python/src/pyopenvino/utils/utils.hpp index 5b0d00165b968a..3b635c2e9bb0b1 100644 --- a/src/bindings/python/src/pyopenvino/utils/utils.hpp +++ b/src/bindings/python/src/pyopenvino/utils/utils.hpp @@ -6,7 +6,18 @@ #include +#ifdef _MSC_VER + // Warning occurred at the junction of pybind11 + // and the templates inside ov::Any. + // Generated by operator `==` inside pybind::handle. + #pragma warning( push ) + #pragma warning( disable: 4996 ) +#endif #include "openvino/core/any.hpp" +#ifdef _MSC_VER + #pragma warning( pop ) +#endif + #include "openvino/core/type/element_type.hpp" #include "openvino/runtime/properties.hpp" #include "openvino/pass/serialize.hpp" @@ -15,6 +26,14 @@ namespace py = pybind11; namespace Common { namespace utils { + struct EmptyList {}; + + py::object from_ov_any_no_leaves(const ov::Any& any); + + py::object from_ov_any_map_no_leaves(const ov::Any& almost_map); + + py::object from_ov_any_map(const ov::AnyMap& map); + py::object from_ov_any(const ov::Any& any); std::map properties_to_any_map(const std::map& properties); diff --git a/src/bindings/python/tests/__init__.py b/src/bindings/python/tests/__init__.py index 06d8dfb043480f..88eb7a97b6bb17 100644 --- a/src/bindings/python/tests/__init__.py +++ b/src/bindings/python/tests/__init__.py @@ -58,6 +58,28 @@ def xfail_test(reason="Mark the test as expected to fail", strict=True): xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed") xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '': " "Axes input must be constant") +xfail_issue_99949 = xfail_test(reason="Bitwise operators are not supported") +xfail_issue_99950 = xfail_test(reason="CenterCropPad func is not supported") +xfail_issue_99952 = xfail_test(reason="Col2Im operator is not supported") +xfail_issue_99954 = xfail_test(reason="Constant Pad - RuntimeError: Shape inference of Reference node with name y failed") +xfail_issue_99955 = xfail_test(reason="GroupNorm is not supported") +xfail_issue_99957 = xfail_test(reason="LayerNorm - RuntimeError: While validating node ''") +xfail_issue_99958 = xfail_test(reason="LogSoftmax - Results mismatch") +xfail_issue_99959 = xfail_test(reason="Mish function is not supported") +xfail_issue_99960 = xfail_test(reason="MVN - Results mismatch") +xfail_issue_99961 = xfail_test(reason="Optional has/get element operators are not supported)'") +xfail_issue_99962 = pytest.mark.skip(reason="ReduceL1/L2 - Unrecognized attribute: axes for operator ReduceL1/L2") +xfail_issue_99968 = xfail_test(reason="ReduceL1/L2 - Results mismatch or unsupported ReduceSum with " + "dynamic rank by CPU plugin") +xfail_issue_99969 = xfail_test(reason="Resize - Results mismatch / " + "RuntimeError: While validating ONNX node '' / " + "RuntimeError: Check '(false)' failed at onnx/frontend/src/op/resize.cpp") +xfail_issue_99970 = xfail_test(reason="Scatter and ScatterND - RuntimeError: Check '(reduction == none)' failed at " + "src/frontends/onnx/frontend/src/op/scatter_elements.cpp OR at " + "src/frontends/onnx/frontend/src/op/scatter_nd") +xfail_issue_99972 = xfail_test(reason="Softmax - Results mismatch") +xfail_issue_99973 = xfail_test(reason="Split - RuntimeError: While validating ONNX node " + "''") xfail_issue_38710 = xfail_test(reason="RuntimeError: data has zero dimension which is not allowed") xfail_issue_38713 = xfail_test(reason="RuntimeError: OV does not support the following ONNX operations: " "ai.onnx.preview.training.Momentum") @@ -117,7 +139,6 @@ def xfail_test(reason="Mark the test as expected to fail", strict=True): xfail_issue_63033 = xfail_test(reason="BatchNormalization: Training mode is not supported") xfail_issue_63036 = xfail_test(reason="Changes in ConvTranspose padding") -xfail_issue_63039 = xfail_test(reason="Result mismatches with UINT8 operations") xfail_issue_63043 = xfail_test(reason="Recurrent node expects constants as W, R, B inputs.") skip_rng_tests = pytest.mark.skip(reason="Tests use random number generator with no seed.") diff --git a/src/bindings/python/tests/test_graph/test_any.py b/src/bindings/python/tests/test_graph/test_any.py index e4817e52f7e296..4a8643a7586189 100644 --- a/src/bindings/python/tests/test_graph/test_any.py +++ b/src/bindings/python/tests/test_graph/test_any.py @@ -34,7 +34,7 @@ def test_any_list(values, data_type): @pytest.mark.parametrize(("value_dict", "value_type", "data_type"), [ - ({"key": "value"}, OVAny, str), + ({"key": "value"}, str, str), ({21: 37}, int, int), ({21.0: 37.0}, float, float), ]) @@ -65,3 +65,26 @@ def __init__(self): value = OVAny(TestClass()) assert isinstance(value.value, TestClass) assert value.value.text == "test" + + +@pytest.mark.parametrize(("value", "dtype"), [ + ("some_value", str), + (31.23456, float), + (True, bool), + (42, int), +]) +def test_astype(value, dtype): + ovany = OVAny(value) + assert ovany.astype(dtype) == value + + +@pytest.mark.parametrize(("value", "dtype"), [ + (["some_value", "another value"], str), + ([31.23456, -31.3453], float), + ([True, False], bool), + ([42, 21], int), + ([], None), +]) +def test_aslist(value, dtype): + ovany = OVAny(value) + assert ovany.aslist(dtype) == value diff --git a/src/bindings/python/tests/test_graph/test_basic.py b/src/bindings/python/tests/test_graph/test_basic.py index 67bb1f1afad7c5..b4cc21edb27150 100644 --- a/src/bindings/python/tests/test_graph/test_basic.py +++ b/src/bindings/python/tests/test_graph/test_basic.py @@ -537,13 +537,6 @@ def test_sink_function_ctor(): assert function.get_friendly_name() == "TestModel" -def test_node_version(): - node = ops.add([1], [2]) - - assert node.get_version() == 1 - assert node.version == 1 - - def test_strides_iteration_methods(): data = np.array([1, 2, 3]) strides = Strides(data) diff --git a/src/bindings/python/tests/test_graph/test_core.py b/src/bindings/python/tests/test_graph/test_core.py index bd02af0fe69089..57e0d26252eec4 100644 --- a/src/bindings/python/tests/test_graph/test_core.py +++ b/src/bindings/python/tests/test_graph/test_core.py @@ -369,10 +369,10 @@ def test_discrete_type_info(): assert n1.get_type_info().name == "TopK" assert n3.get_type_info().name == "Sin" assert n1.type_info.name == n2.type_info.name - assert n1.type_info.version == n2.type_info.version + assert n1.type_info.version_id == n2.type_info.version_id assert n1.type_info.parent == n2.type_info.parent assert n1.get_type_info().name == n2.get_type_info().name - assert n1.get_type_info().version == n2.get_type_info().version + assert n1.get_type_info().version_id == n2.get_type_info().version_id assert n1.get_type_info().parent == n2.get_type_info().parent assert n1.get_type_info().name != n3.get_type_info().name assert n1.get_type_info().name > n3.get_type_info().name diff --git a/src/bindings/python/tests/test_graph/test_create_op.py b/src/bindings/python/tests/test_graph/test_create_op.py index 120f07562d4457..da8cfca15c1c51 100644 --- a/src/bindings/python/tests/test_graph/test_create_op.py +++ b/src/bindings/python/tests/test_graph/test_create_op.py @@ -11,7 +11,8 @@ import openvino.runtime.opset1 as ov_opset1 import openvino.runtime.opset5 as ov_opset5 -import openvino.runtime.opset10 as ov +import openvino.runtime.opset10 as ov_opset10 +import openvino.runtime.opset11 as ov from openvino.runtime import Type np_types = [np.float32, np.int32] @@ -2145,8 +2146,29 @@ def test_interpolate_opset10(dtype, expected_shape, shape_calculation_mode): axes = [2, 3] mode = "cubic" - node = ov.interpolate(image=image_node, output_shape=output_shape, scales=scales, - axes=axes, mode=mode, + node = ov_opset10.interpolate(image=image_node, output_shape=output_shape, scales=scales, + axes=axes, mode=mode, shape_calculation_mode=shape_calculation_mode) + assert node.get_type_name() == "Interpolate" + assert node.get_output_size() == 1 + assert list(node.get_output_shape(0)) == expected_shape + + +@pytest.mark.parametrize( + ("expected_shape", "shape_calculation_mode", "input_value"), + [ + ([1, 3, 64, 64], "scales", np.array([1 / 16, 1 / 16], dtype=np.float32)), + ([1, 3, 256, 256], "sizes", np.array([256, 256], dtype=np.int32)), + ], +) +@pytest.mark.parametrize("dtype", np_types) +def test_interpolate_opset11(dtype, expected_shape, shape_calculation_mode, input_value): + + image_shape = [1, 3, 1024, 1024] + image_node = ov.parameter(image_shape, dtype, name="Image") + axes = [2, 3] + mode = "bilinear_pillow" + + node = ov.interpolate(image=image_node, scales_or_sizes=input_value, axes=axes, mode=mode, shape_calculation_mode=shape_calculation_mode) assert node.get_type_name() == "Interpolate" assert node.get_output_size() == 1 @@ -2278,3 +2300,16 @@ def test_unique_opset10(): assert node.get_output_element_type(1) == Type.i64 assert node.get_output_element_type(2) == Type.i64 assert node.get_output_element_type(3) == Type.i64 + + +def test_topk_opset11(): + data_shape = [1, 3, 256] + data = ov.parameter(data_shape, dtype=np.int32, name="Data") + k_val = np.int32(3) + axis = np.int32(-1) + node = ov.topk(data, k_val, axis, "min", "value", stable=True) + + assert node.get_type_name() == "TopK" + assert node.get_output_size() == 2 + assert list(node.get_output_shape(0)) == [1, 3, 3] + assert list(node.get_output_shape(1)) == [1, 3, 3] diff --git a/src/bindings/python/tests/test_graph/test_manager.py b/src/bindings/python/tests/test_graph/test_manager.py index dad03fecaeaa2c..bdc47e8fee0f8d 100644 --- a/src/bindings/python/tests/test_graph/test_manager.py +++ b/src/bindings/python/tests/test_graph/test_manager.py @@ -48,9 +48,22 @@ def test_constant_folding(): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_serialize_seperate_paths_kwargs(request, tmp_path): - core = Core() +@pytest.fixture +def prepare_ir_paths(request, tmp_path): xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + + yield xml_path, bin_path + + # IR Files deletion should be done after `Model` is destructed. + # It may be achieved by splitting scopes (`Model` will be destructed + # just after test scope finished), or by calling `del Model` + os.remove(xml_path) + os.remove(bin_path) + + +def test_serialize_separate_paths_kwargs(prepare_ir_paths): + core = Core() + shape = [2, 2] parameter_a = ops.parameter(shape, dtype=np.float32, name="A") parameter_b = ops.parameter(shape, dtype=np.float32, name="B") @@ -58,6 +71,7 @@ def test_serialize_seperate_paths_kwargs(request, tmp_path): model = (parameter_a + parameter_b) * parameter_c func = Model(model, [parameter_a, parameter_b, parameter_c], "Model") + xml_path, bin_path = prepare_ir_paths pass_manager = Manager() pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path)) pass_manager.run_passes(func) @@ -67,14 +81,10 @@ def test_serialize_seperate_paths_kwargs(request, tmp_path): assert func.get_parameters() == res_model.get_parameters() assert func.get_ordered_ops() == res_model.get_ordered_ops() - os.remove(xml_path) - os.remove(bin_path) - -# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_serialize_seperate_paths_args(request, tmp_path): +def test_serialize_separate_paths_args(prepare_ir_paths): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + shape = [2, 2] parameter_a = ops.parameter(shape, dtype=np.float32, name="A") parameter_b = ops.parameter(shape, dtype=np.float32, name="B") @@ -83,6 +93,7 @@ def test_serialize_seperate_paths_args(request, tmp_path): model = ((parameter_a + parameter_b) * parameter_c) / parameter_d func = Model(model, [parameter_a, parameter_b, parameter_c, parameter_d], "Model") + xml_path, bin_path = prepare_ir_paths pass_manager = Manager() pass_manager.register_pass(Serialize(xml_path, bin_path)) pass_manager.run_passes(func) @@ -92,20 +103,17 @@ def test_serialize_seperate_paths_args(request, tmp_path): assert func.get_parameters() == res_model.get_parameters() assert func.get_ordered_ops() == res_model.get_ordered_ops() - os.remove(xml_path) - os.remove(bin_path) - -# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_serialize_pass_mixed_args_kwargs(request, tmp_path): +def test_serialize_pass_mixed_args_kwargs(prepare_ir_paths): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + shape = [3, 2] parameter_a = ops.parameter(shape, dtype=np.float32, name="A") parameter_b = ops.parameter(shape, dtype=np.float32, name="B") model = parameter_a - parameter_b func = Model(model, [parameter_a, parameter_b], "Model") + xml_path, bin_path = prepare_ir_paths pass_manager = Manager() pass_manager.register_pass(Serialize(xml_path, path_to_bin=bin_path)) pass_manager.run_passes(func) @@ -115,14 +123,11 @@ def test_serialize_pass_mixed_args_kwargs(request, tmp_path): assert func.get_parameters() == res_model.get_parameters() assert func.get_ordered_ops() == res_model.get_ordered_ops() - os.remove(xml_path) - os.remove(bin_path) - -# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_serialize_pass_mixed_args_kwargs_v2(request, tmp_path): +def test_serialize_pass_mixed_args_kwargs_v2(prepare_ir_paths): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + + xml_path, bin_path = prepare_ir_paths model = create_model() pass_manager = Manager() pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path)) @@ -133,9 +138,6 @@ def test_serialize_pass_mixed_args_kwargs_v2(request, tmp_path): assert model.get_parameters() == res_model.get_parameters() assert model.get_ordered_ops() == res_model.get_ordered_ops() - os.remove(xml_path) - os.remove(bin_path) - # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_serialize_pass_wrong_num_of_args(request, tmp_path): @@ -147,14 +149,13 @@ def test_serialize_pass_wrong_num_of_args(request, tmp_path): assert "Invoked with:" in str(e.value) -# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_serialize_results(request, tmp_path): +def test_serialize_results(prepare_ir_paths): core = Core() node_constant = ops.constant(np.array([[0.0, 0.1, -0.1], [-2.5, 2.5, 3.0]], dtype=np.float32)) node_ceil = ops.ceiling(node_constant) func = Model(node_ceil, [], "Model") - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + xml_path, bin_path = prepare_ir_paths pass_manager = Manager() pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path)) pass_manager.run_passes(func) @@ -165,14 +166,32 @@ def test_serialize_results(request, tmp_path): assert const == new_const - os.remove(xml_path) - os.remove(bin_path) + import gc + gc.collect() # CVS-106805 -# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_default_version(request, tmp_path): +@pytest.mark.xfail(reason="CVS-106805. Delete test after fix") +def test_serialize_results_fails(prepare_ir_paths): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + node_constant = ops.constant(np.array([[0.0, 0.1, -0.1], [-2.5, 2.5, 3.0]], dtype=np.float32)) + func = Model(node_constant, [], "Model") + + xml_path, bin_path = prepare_ir_paths + pass_manager = Manager() + pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path)) + pass_manager.run_passes(func) + + res_model = core.read_model(model=xml_path, weights=bin_path) + const = func.get_results() + new_const = res_model.get_results() + + assert const == new_const + + +def test_default_version(prepare_ir_paths): + core = Core() + + xml_path, bin_path = prepare_ir_paths model = create_model() pass_manager = Manager() pass_manager.register_pass(Serialize(xml_path, bin_path)) @@ -183,14 +202,11 @@ def test_default_version(request, tmp_path): assert model.get_parameters() == res_model.get_parameters() assert model.get_ordered_ops() == res_model.get_ordered_ops() - os.remove(xml_path) - os.remove(bin_path) - -# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request -def test_default_version_IR_V11_seperate_paths(request, tmp_path): +def test_default_version_IR_V11_seperate_paths(prepare_ir_paths): core = Core() - xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) + + xml_path, bin_path = prepare_ir_paths model = create_model() pass_manager = Manager() pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path, version=Version.IR_V11)) @@ -200,6 +216,3 @@ def test_default_version_IR_V11_seperate_paths(request, tmp_path): assert model.get_parameters() == res_model.get_parameters() assert model.get_ordered_ops() == res_model.get_ordered_ops() - - os.remove(xml_path) - os.remove(bin_path) diff --git a/src/bindings/python/tests/test_onnx/test_backend.py b/src/bindings/python/tests/test_onnx/test_backend.py index c681f376348142..e97a5adcbb6535 100644 --- a/src/bindings/python/tests/test_onnx/test_backend.py +++ b/src/bindings/python/tests/test_onnx/test_backend.py @@ -37,7 +37,6 @@ xfail_issue_58033, xfail_issue_63033, xfail_issue_63036, - xfail_issue_63039, xfail_issue_63043, xfail_issue_63137, xfail_issue_63138, @@ -49,6 +48,21 @@ xfail_issue_90649, xfail_issue_91151, xfail_issue_91490, + xfail_issue_99949, + xfail_issue_99950, + xfail_issue_99952, + xfail_issue_99954, + xfail_issue_99955, + xfail_issue_99957, + xfail_issue_99958, + xfail_issue_99959, + xfail_issue_99960, + xfail_issue_99961, + xfail_issue_99968, + xfail_issue_99969, + xfail_issue_99970, + xfail_issue_99972, + xfail_issue_99973, xfail_issue_101965, ) from tests.test_onnx.utils.onnx_backend import OpenVinoTestBackend @@ -278,10 +292,6 @@ def expect_fail(test_case_path, xfail): # type: (str) -> None "OnnxBackendNodeModelTest.test_batchnorm_example_training_mode_cpu", ), (xfail_issue_63036, "OnnxBackendNodeModelTest.test_convtranspose_autopad_same_cpu"), - ( - xfail_issue_63039, - "OnnxBackendNodeModelTest.test_div_uint8_cpu", - ), ( xfail_issue_63043, "OnnxBackendNodeModelTest.test_gru_batchwise_cpu", @@ -394,6 +404,193 @@ def expect_fail(test_case_path, xfail): # type: (str) -> None "OnnxBackendNodeModelTest.test_castlike_BFLOAT16_to_FLOAT_cpu", "OnnxBackendNodeModelTest.test_castlike_FLOAT_to_BFLOAT16_cpu", ), + ( + xfail_issue_99949, + "OnnxBackendNodeModelTest.test_bitwise_and_i16_3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_and_i32_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_and_ui64_bcast_3v1d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_not_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_not_3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_not_4d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_i16_4d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_i32_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_ui64_bcast_3v1d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_ui8_bcast_4v3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_i16_3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_i32_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_and_ui8_bcast_4v3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_ui8_bcast_4v3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_ui64_bcast_3v1d_cpu", + ), + ( + xfail_issue_99950, + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_and_pad_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_chw_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_chw_expanded_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_hwc_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_hwc_expanded_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_pad_cpu", + ), + ( + xfail_issue_99952, + "OnnxBackendNodeModelTest.test_col2im_5d_cpu", + "OnnxBackendNodeModelTest.test_col2im_cpu", + "OnnxBackendNodeModelTest.test_col2im_dilations_cpu", + "OnnxBackendNodeModelTest.test_col2im_pads_cpu", + "OnnxBackendNodeModelTest.test_col2im_strides_cpu", + ), + ( + xfail_issue_99954, + "OnnxBackendNodeModelTest.test_constant_pad_axes_cpu", + ), + ( + xfail_issue_99955, + "OnnxBackendNodeModelTest.test_group_normalization_epsilon_cpu", + "OnnxBackendNodeModelTest.test_group_normalization_epsilon_expanded_cpu", + "OnnxBackendNodeModelTest.test_group_normalization_example_cpu", + "OnnxBackendNodeModelTest.test_group_normalization_example_expanded_cpu", + ), + ( + xfail_issue_99957, + "OnnxBackendNodeModelTest.test_layer_normalization_2d_axis1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_2d_axis_negative_1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis1_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis2_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis2_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis3_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis_negative_1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis_negative_2_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis_negative_3_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_default_axis_expanded_ver18_cpu", + ), + ( + xfail_issue_99958, + "OnnxBackendNodeModelTest.test_logsoftmax_large_number_expanded_ver18_cpu", + ), + ( + xfail_issue_99959, + "OnnxBackendNodeModelTest.test_mish_cpu", + ), + ( + xfail_issue_99960, + "OnnxBackendNodeModelTest.test_mvn_expanded_ver18_cpu", + ), + ( + xfail_issue_99961, + "OnnxBackendNodeModelTest.test_optional_get_element_optional_sequence_cpu", + "OnnxBackendNodeModelTest.test_optional_get_element_optional_tensor_cpu", + "OnnxBackendNodeModelTest.test_optional_get_element_tensor_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_name_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_name_tensor_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_tensor_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_tensor_input_cpu", + ), + ( + xfail_issue_99968, + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_negative_axes_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_negative_axes_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_negative_axes_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_negative_axes_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_asc_axes_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_asc_axes_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_desc_axes_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_desc_axes_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_negative_axes_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_negative_axes_keepdims_random_cpu", + ), + ( + xfail_issue_99969, + "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_cubic_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_linear_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_not_smaller_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_not_larger_cpu", + "OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_axes_2_3_cpu", + "OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_axes_3_2_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_nearest_axes_2_3_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_nearest_axes_3_2_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_axes_2_3_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_axes_3_2_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_not_larger_cpu", + ), + ( + xfail_issue_99970, + "OnnxBackendNodeModelTest.test_scatter_elements_with_reduction_max_cpu", + "OnnxBackendNodeModelTest.test_scatter_elements_with_reduction_min_cpu", + "OnnxBackendNodeModelTest.test_scatternd_max_cpu", + "OnnxBackendNodeModelTest.test_scatternd_min_cpu", + ), + ( + xfail_issue_99972, + "OnnxBackendNodeModelTest.test_softmax_large_number_expanded_ver18_cpu", + ), + ( + xfail_issue_99973, + "OnnxBackendNodeModelTest.test_split_1d_uneven_split_opset18_cpu", + "OnnxBackendNodeModelTest.test_split_2d_uneven_split_opset18_cpu", + "OnnxBackendNodeModelTest.test_split_zero_size_splits_opset13_cpu", + "OnnxBackendNodeModelTest.test_split_zero_size_splits_opset18_cpu", + ), ( xfail_issue_101965, "OnnxBackendNodeModelTest.test_dft_axis_cpu", diff --git a/src/bindings/python/tests/test_onnx/test_ops_reduction.py b/src/bindings/python/tests/test_onnx/test_ops_reduction.py index f9fe00ddb43fcb..9d2a06e9042b6b 100644 --- a/src/bindings/python/tests/test_onnx/test_ops_reduction.py +++ b/src/bindings/python/tests/test_onnx/test_ops_reduction.py @@ -6,6 +6,7 @@ import onnx import pytest +from tests import xfail_issue_99962 from tests.runtime import get_runtime from tests.test_onnx.utils import ( run_node, @@ -80,6 +81,7 @@ def test_reduce_operation_keepdims_none_axes(operation, ref_operation): @pytest.mark.parametrize(("operation", "ref_operation"), reduce_operation_parameters_as_attr) @pytest.mark.parametrize("axes", reduce_axis_parameters) +@xfail_issue_99962 def test_reduce_operation_keepdims_with_axes_as_attr(operation, ref_operation, axes): assert np.array_equal(import_and_compute(operation, reduce_data, axes=axes, keepdims=True), ref_operation(reduce_data, keepdims=True, axis=axes)) @@ -102,6 +104,7 @@ def test_reduce_operation_keepdims_with_axes_as_const(operation, ref_operation, (1, 2), (0, 1, 2)]) @pytest.mark.parametrize(("operation", "ref_operation"), reduce_operation_parameters_as_attr) +@xfail_issue_99962 def test_reduce_operation_no_keepdims_axes_as_attr(operation, ref_operation, axes): if axes: assert np.array_equal(import_and_compute(operation, reduce_data, axes=axes, keepdims=False), @@ -133,6 +136,7 @@ def test_reduce_operation_no_keepdims_axes_as_const(operation, ref_operation, ax ref_operation(reduce_data, keepdims=False)) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_l1(reduction_axes): shape = [2, 4, 3, 2] @@ -152,6 +156,7 @@ def test_reduce_l1(reduction_axes): assert np.allclose(expected, graph_result) +@xfail_issue_99962 def test_reduce_l1_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -170,6 +175,7 @@ def test_reduce_l1_default_axes(): assert np.allclose(expected, graph_result) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_l2(reduction_axes): shape = [2, 4, 3, 2] @@ -190,6 +196,7 @@ def test_reduce_l2(reduction_axes): assert np.allclose(expected, graph_result) +@xfail_issue_99962 def test_reduce_l2_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -208,6 +215,7 @@ def test_reduce_l2_default_axes(): assert np.allclose(expected, graph_result) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_log_sum(reduction_axes): shape = [2, 4, 3, 2] @@ -227,6 +235,7 @@ def test_reduce_log_sum(reduction_axes): assert np.allclose(expected, graph_result) +@xfail_issue_99962 def test_reduce_log_sum_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -245,6 +254,7 @@ def test_reduce_log_sum_default_axes(): assert np.allclose(expected, graph_result) +@xfail_issue_99962 def test_reduce_log_sum_exp(): def logsumexp(data, axis=None, keepdims=True): return np.log(np.sum(np.exp(data), axis=axis, keepdims=keepdims)) @@ -276,6 +286,7 @@ def logsumexp(data, axis=None, keepdims=True): ) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_sum_square(reduction_axes): shape = [2, 4, 3, 2] @@ -295,6 +306,7 @@ def test_reduce_sum_square(reduction_axes): assert np.allclose(expected, graph_result) +@xfail_issue_99962 def test_reduce_sum_square_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) diff --git a/src/bindings/python/tests/test_onnx/test_ops_unary.py b/src/bindings/python/tests/test_onnx/test_ops_unary.py index f02110e3771ab2..847009cbc769f8 100644 --- a/src/bindings/python/tests/test_onnx/test_ops_unary.py +++ b/src/bindings/python/tests/test_onnx/test_ops_unary.py @@ -327,7 +327,7 @@ def test_identity(): def test_cast_to_bool(val_type, input_data): expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -344,7 +344,7 @@ def test_cast_to_float(val_type, range_start, range_end, in_dtype): input_data = np.random.randint(range_start, range_end, size=(2, 2), dtype=in_dtype) expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -361,7 +361,7 @@ def test_cast_to_int(val_type): input_data = np.ceil(-8 + random_data).astype(val_type) expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -374,7 +374,7 @@ def test_cast_to_uint(val_type): input_data = np.ceil(np.random.rand(2, 3, 4) * 16).astype(val_type) expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -449,7 +449,7 @@ def test_constant(value_type): outputs=["values"], value=onnx.helper.make_tensor( name="const_tensor", - data_type=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(value_type)], + data_type=onnx.helper.np_dtype_to_tensor_dtype(np.dtype(value_type)), dims=values.shape, vals=values.flatten(), ), @@ -467,7 +467,7 @@ def test_constant_err(): outputs=["values"], value=onnx.helper.make_tensor( name="const_tensor", - data_type=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(np.float16)], + data_type=onnx.helper.np_dtype_to_tensor_dtype(np.dtype(np.float16)), dims=values.shape, vals=values.flatten(), ), diff --git a/src/bindings/python/tests/test_onnx/utils/onnx_backend.py b/src/bindings/python/tests/test_onnx/utils/onnx_backend.py index 11609f5308d035..eb7e37e225f95f 100644 --- a/src/bindings/python/tests/test_onnx/utils/onnx_backend.py +++ b/src/bindings/python/tests/test_onnx/utils/onnx_backend.py @@ -13,11 +13,11 @@ import numpy import onnx from onnx.backend.base import Backend, BackendRep -from onnx.helper import make_graph, make_model, make_tensor_value_info +from onnx.helper import make_graph, make_model, make_tensor_value_info, np_dtype_to_tensor_dtype from openvino.runtime import Model from tests.runtime import get_runtime -from tests.test_onnx.utils.onnx_helpers import import_onnx_model, np_dtype_to_tensor_type +from tests.test_onnx.utils.onnx_helpers import import_onnx_model class OpenVinoOnnxBackendRep(BackendRep): @@ -80,15 +80,15 @@ def run_node( ): # type: (...) -> Optional[Tuple[Any, ...]] """Prepare and run a computation on an ONNX node.""" # default values for input/output tensors - input_tensor_types = [np_dtype_to_tensor_type(node_input.dtype) for node_input in inputs] - output_tensor_types = [onnx.TensorProto.FLOAT for idx in range(len(node.output))] + input_tensor_types = [np_dtype_to_tensor_dtype(node_input.dtype) for node_input in inputs] + output_tensor_types = [onnx.TensorProto.FLOAT for _ in range(len(node.output))] output_tensor_shapes = [()] # type: List[Tuple[int, ...]] if outputs_info is not None: output_tensor_types = [ - np_dtype_to_tensor_type(dtype) for (dtype, shape) in outputs_info + np_dtype_to_tensor_dtype(dtype) for (dtype, _) in outputs_info ] - output_tensor_shapes = [shape for (dtype, shape) in outputs_info] + output_tensor_shapes = [shape for (_, shape) in outputs_info] input_tensors = [ make_tensor_value_info(name, tensor_type, value.shape) diff --git a/src/bindings/python/tests/test_onnx/utils/onnx_helpers.py b/src/bindings/python/tests/test_onnx/utils/onnx_helpers.py index 061e7912e26680..15970e5d43a514 100644 --- a/src/bindings/python/tests/test_onnx/utils/onnx_helpers.py +++ b/src/bindings/python/tests/test_onnx/utils/onnx_helpers.py @@ -4,20 +4,10 @@ import numpy as np import onnx -from onnx.mapping import NP_TYPE_TO_TENSOR_TYPE from openvino.runtime import Core, Tensor, Model -def np_dtype_to_tensor_type(data_type: np.dtype) -> int: - """Return TensorProto type for provided numpy dtype. - - :param data_type: Numpy data type object. - :return: TensorProto.DataType enum value for corresponding type. - """ - return NP_TYPE_TO_TENSOR_TYPE[data_type] - - def import_onnx_model(model: onnx.ModelProto) -> Model: onnx.checker.check_model(model) model_byte_string = model.SerializeToString() diff --git a/src/bindings/python/tests/test_runtime/test_core.py b/src/bindings/python/tests/test_runtime/test_core.py index 15c6a2ed553eb1..87709aa443316c 100644 --- a/src/bindings/python/tests/test_runtime/test_core.py +++ b/src/bindings/python/tests/test_runtime/test_core.py @@ -176,11 +176,13 @@ def test_get_version(device): def test_available_devices(device): core = Core() - devices = core.available_devices - assert device in devices, ( - f"Current device '{device}' is not listed in " - f"available devices '{', '.join(devices)}'" - ) + devices_attr = core.available_devices + devices_method = core.get_available_devices() + for devices in (devices_attr, devices_method): + assert device in devices, ( + f"Current device '{device}' is not listed in " + f"available devices '{', '.join(devices)}'" + ) def test_get_property(device): diff --git a/src/bindings/python/tests/test_runtime/test_dimension.py b/src/bindings/python/tests/test_runtime/test_dimension.py new file mode 100644 index 00000000000000..697e11555590b0 --- /dev/null +++ b/src/bindings/python/tests/test_runtime/test_dimension.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from openvino.runtime import Dimension + + +def test_dynamic_dimension(): + dim = Dimension() + assert dim.is_dynamic + assert str(dim) == "?" + assert dim.to_string() == "?" + assert str(dim.__repr__) == ">" + + +def test_dynamic_dimension_with_bounds(): + dim = Dimension(2, 5) + assert str(dim) == "2..5" + assert dim.to_string() == "2..5" + assert not dim.is_static + assert dim.is_dynamic + assert dim.get_min_length() == 2 + assert dim.min_length == 2 + assert dim.get_max_length() == 5 + assert dim.max_length == 5 + + +def test_static_dimension(): + dim = Dimension(2) + assert str(dim) == "2" + assert dim.to_string() == "2" + assert dim.is_static + assert not dim.is_dynamic + assert len(dim) == 2 + assert dim.get_length() == 2 + + +def test_dim_same_scheme(): + assert Dimension().same_scheme(Dimension()) is True + assert Dimension(3).same_scheme(Dimension(3)) is True + assert Dimension(3).same_scheme(Dimension(4)) is False + assert Dimension().same_scheme(Dimension(4)) is False + + +def test_dim_compatible(): + assert Dimension().compatible(Dimension()) is True + assert Dimension(3).compatible(Dimension(3)) is True + assert Dimension(3).compatible(Dimension(4)) is False + assert Dimension().compatible(Dimension(4)) is True + + +def test_dim_relax(): + assert Dimension().relaxes(Dimension()) is True + assert Dimension(3).relaxes(Dimension(3)) is True + assert Dimension(3).relaxes(Dimension(4)) is False + assert Dimension().relaxes(Dimension(4)) is True + + +def test_dim_refine(): + assert Dimension().refines(Dimension()) is True + assert Dimension(3).refines(Dimension(3)) is True + assert Dimension(3).refines(Dimension(4)) is False + assert Dimension().refines(Dimension(4)) is False diff --git a/src/bindings/python/tests/test_runtime/test_infer_request.py b/src/bindings/python/tests/test_runtime/test_infer_request.py index 4540e9d7ebe8d9..4755fde6a77ca5 100644 --- a/src/bindings/python/tests/test_runtime/test_infer_request.py +++ b/src/bindings/python/tests/test_runtime/test_infer_request.py @@ -267,7 +267,7 @@ def test_batched_tensors(device): assert np.array_equal(actual[idx], _tmp) -def test_inputs_outputs_property(device): +def test_inputs_outputs_property_and_method(device): num_inputs = 10 input_shape = [1] params = [ops.parameter(input_shape, np.uint8) for _ in range(num_inputs)] @@ -277,10 +277,14 @@ def test_inputs_outputs_property(device): request = compiled_model.create_infer_request() data = [np.atleast_1d(i) for i in range(num_inputs)] results = request.infer(data).values() - for result, output_tensor in zip(results, request.outputs): + for result, output_tensor in zip(results, request.output_tensors): assert np.array_equal(result, output_tensor.data) - for input_data, input_tensor in zip(data, request.inputs): + for input_data, input_tensor in zip(data, request.input_tensors): assert np.array_equal(input_data, input_tensor.data) + for input_tensor in request.input_tensors: + assert list(input_tensor.get_shape()) == input_shape + for output_tensor in request.output_tensors: + assert list(output_tensor.get_shape()) == input_shape @pytest.mark.skip(reason="Sporadically failed. Need further investigation. Ticket - 95967") @@ -395,7 +399,7 @@ def test_infer_mixed_values(device, ov_type, numpy_dtype, shared_flag): request.infer([tensor1, array1], shared_memory=shared_flag) - assert np.array_equal(request.outputs[0].data, np.concatenate((tensor1.data, array1))) + assert np.array_equal(request.output_tensors[0].data, np.concatenate((tensor1.data, array1))) @pytest.mark.parametrize(("ov_type", "numpy_dtype"), [ @@ -419,8 +423,7 @@ def test_async_mixed_values(device, ov_type, numpy_dtype, shared_flag): request.start_async([tensor1, array1], shared_memory=shared_flag) request.wait() - - assert np.array_equal(request.outputs[0].data, np.concatenate((tensor1.data, array1))) + assert np.array_equal(request.output_tensors[0].data, np.concatenate((tensor1.data, array1))) @pytest.mark.parametrize(("ov_type", "numpy_dtype"), [ @@ -512,6 +515,7 @@ def test_infer_queue_iteration(device): it = iter(infer_queue) infer_request = next(it) assert isinstance(infer_request, InferRequest) + assert infer_request.userdata is None with pytest.raises(StopIteration): next(it) diff --git a/src/bindings/python/tests/test_runtime/test_model.py b/src/bindings/python/tests/test_runtime/test_model.py index b13e5015275403..e0af9cbda469be 100644 --- a/src/bindings/python/tests/test_runtime/test_model.py +++ b/src/bindings/python/tests/test_runtime/test_model.py @@ -5,6 +5,7 @@ import os import numpy as np import pytest +import math import openvino.runtime.opset8 as ops from openvino.runtime import ( @@ -48,8 +49,10 @@ def test_function_add_outputs_tensor_name(): relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 new_outs = function.add_outputs("relu_t1") assert len(function.get_results()) == 2 + assert len(function.results) == 2 assert "relu_t1" in function.outputs[1].get_tensor().names assert len(new_outs) == 1 assert new_outs[0].get_node() == function.outputs[1].get_node() @@ -64,8 +67,10 @@ def test_function_add_outputs_op_name(): relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 new_outs = function.add_outputs(("relu1", 0)) assert len(function.get_results()) == 2 + assert len(function.results) == 2 assert len(new_outs) == 1 assert new_outs[0].get_node() == function.outputs[1].get_node() assert new_outs[0].get_index() == function.outputs[1].get_index() @@ -78,9 +83,9 @@ def test_function_add_output_port(): relu1.get_output_tensor(0).set_names({"relu_t1"}) relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") - assert len(function.get_results()) == 1 + assert len(function.results) == 1 new_outs = function.add_outputs(relu1.output(0)) - assert len(function.get_results()) == 2 + assert len(function.results) == 2 assert len(new_outs) == 1 assert new_outs[0].get_node() == function.outputs[1].get_node() assert new_outs[0].get_index() == function.outputs[1].get_index() @@ -94,6 +99,7 @@ def test_function_add_output_incorrect_tensor_name(): relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 with pytest.raises(RuntimeError) as e: function.add_outputs("relu_t") # Verify that absent output name is present in error message @@ -108,6 +114,7 @@ def test_function_add_output_incorrect_idx(): relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 with pytest.raises(RuntimeError) as e: function.add_outputs(("relu1", 1234)) # Verify that op name and port number are present in error message @@ -123,6 +130,7 @@ def test_function_add_output_incorrect_name(): relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 with pytest.raises(RuntimeError) as e: function.add_outputs(("relu_1", 0)) # Verify that absent op name is present in error message @@ -139,8 +147,10 @@ def test_add_outputs_several_tensors(): relu3 = ops.relu(relu2, name="relu3") function = Model(relu3, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 new_outs = function.add_outputs(["relu_t1", "relu_t2"]) assert len(function.get_results()) == 3 + assert len(function.results) == 3 assert len(new_outs) == 2 assert new_outs[0].get_node() == function.outputs[1].get_node() assert new_outs[0].get_index() == function.outputs[1].get_index() @@ -158,8 +168,10 @@ def test_add_outputs_several_ports(): relu3 = ops.relu(relu2, name="relu3") function = Model(relu3, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 new_outs = function.add_outputs([("relu1", 0), ("relu2", 0)]) assert len(function.get_results()) == 3 + assert len(function.results) == 3 assert len(new_outs) == 2 assert new_outs[0].get_node() == function.outputs[1].get_node() assert new_outs[0].get_index() == function.outputs[1].get_index() @@ -175,6 +187,7 @@ def test_add_outputs_incorrect_value(): relu2 = ops.relu(relu1, name="relu2") function = Model(relu2, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 with pytest.raises(TypeError) as e: function.add_outputs(0) assert "Incorrect type of a value to add as output." in str(e.value) @@ -187,6 +200,7 @@ def test_add_outputs_incorrect_outputs_list(): relu1.get_output_tensor(0).set_names({"relu_t1"}) function = Model(relu1, [param], "TestFunction") assert len(function.get_results()) == 1 + assert len(function.results) == 1 with pytest.raises(TypeError) as e: function.add_outputs([0, 0]) assert "Incorrect type of a value to add as output at index 0" in str(e.value) @@ -283,6 +297,9 @@ def test_get_batch(): param = model.get_parameters()[0] param.set_layout(Layout("NC")) assert get_batch(model) == 2 + param = model.parameters[0] + param.set_layout(Layout("NC")) + assert get_batch(model) == 2 def test_get_batch_chwn(): @@ -292,41 +309,53 @@ def test_get_batch_chwn(): add = ops.add(param1, param2) add2 = ops.add(add, param3) model = Model(add2, [param1, param2, param3], "TestFunction") - param = model.get_parameters()[0] - param.set_layout(Layout("CHWN")) + param_method = model.get_parameters()[0] + param_attr = model.parameters[0] + param_method.set_layout(Layout("CHWN")) + param_attr.set_layout(Layout("CHWN")) assert get_batch(model) == 4 def test_set_batch_dimension(): model = generate_add_model() - model_param1 = model.get_parameters()[0] - model_param2 = model.get_parameters()[1] + model_param1_method = model.get_parameters()[0] + model_param2_method = model.get_parameters()[1] + model_param1_attr = model.parameters[0] + model_param2_attr = model.parameters[1] # check batch == 2 - model_param1.set_layout(Layout("NC")) + model_param1_method.set_layout(Layout("NC")) + model_param1_attr.set_layout(Layout("NC")) assert get_batch(model) == 2 # set batch to 1 set_batch(model, Dimension(1)) assert get_batch(model) == 1 # check if shape of param 1 has changed - assert model_param1.get_output_shape(0) == PartialShape([1, 1]) + assert model_param1_method.get_output_shape(0) == PartialShape([1, 1]) + assert model_param1_attr.get_output_shape(0) == PartialShape([1, 1]) # check if shape of param 2 has not changed - assert model_param2.get_output_shape(0) == PartialShape([2, 1]) + assert model_param2_method.get_output_shape(0) == PartialShape([2, 1]) + assert model_param2_attr.get_output_shape(0) == PartialShape([2, 1]) def test_set_batch_int(): model = generate_add_model() - model_param1 = model.get_parameters()[0] - model_param2 = model.get_parameters()[1] + model_param1_method = model.get_parameters()[0] + model_param2_method = model.get_parameters()[1] + model_param1_attr = model.parameters[0] + model_param2_attr = model.parameters[1] # check batch == 2 - model_param1.set_layout(Layout("NC")) + model_param1_method.set_layout(Layout("NC")) + model_param1_attr.set_layout(Layout("NC")) assert get_batch(model) == 2 # set batch to 1 set_batch(model, 1) assert get_batch(model) == 1 # check if shape of param 1 has changed - assert model_param1.get_output_shape(0) == PartialShape([1, 1]) + assert model_param1_method.get_output_shape(0) == PartialShape([1, 1]) + assert model_param1_attr.get_output_shape(0) == PartialShape([1, 1]) # check if shape of param 2 has not changed - assert model_param2.get_output_shape(0) == PartialShape([2, 1]) + assert model_param2_method.get_output_shape(0) == PartialShape([2, 1]) + assert model_param2_attr.get_output_shape(0) == PartialShape([2, 1]) def test_set_batch_default_batch_size(): @@ -335,6 +364,7 @@ def test_set_batch_default_batch_size(): model_param1.set_layout(Layout("NC")) set_batch(model) assert model.is_dynamic() + assert model.dynamic def test_reshape_with_ports(): @@ -520,22 +550,6 @@ def check_rt_info(model): # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request def test_serialize_complex_rt_info(request, tmp_path): def check_rt_info(model, serialized): - if serialized: - threshold = "13.23" - min_val = "-3.24543" - max_val = "3.23422" - directed = "YES" - empty = "" - ids = "sasd fdfdfsdf" - mean = "22.3 33.11 44" - else: - threshold = 13.23 - min_val = -3.24543 - max_val = 3.234223 - directed = True - empty = [] - ids = ["sasd", "fdfdfsdf"] - mean = [22.3, 33.11, 44.0] assert model.has_rt_info(["config", "type_of_model"]) is True assert model.has_rt_info(["config", "converter_type"]) is True assert model.has_rt_info(["config", "model_parameters", "threshold"]) is True @@ -548,17 +562,29 @@ def check_rt_info(model, serialized): assert model.has_rt_info(["config", "model_parameters", "labels", "label_groups", "ids"]) is True assert model.has_rt_info(["config", "model_parameters", "mean_values"]) is True - assert model.get_rt_info(["config", "type_of_model"]) == "classification" - assert model.get_rt_info(["config", "converter_type"]) == "classification" - assert model.get_rt_info(["config", "model_parameters", "threshold"]) == threshold - assert model.get_rt_info(["config", "model_parameters", "min"]) == min_val - assert model.get_rt_info(["config", "model_parameters", "max"]) == max_val - assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "type"]) == "tree" - assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "directed"]) == directed - assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "float_empty"]) == empty - assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "nodes"]) == empty - assert model.get_rt_info(["config", "model_parameters", "labels", "label_groups", "ids"]) == ids - assert model.get_rt_info(["config", "model_parameters", "mean_values"]) == mean + assert model.get_rt_info(["config", "type_of_model"]).astype(str) == "classification" + assert model.get_rt_info(["config", "converter_type"]).astype(str) == "classification" + assert math.isclose(model.get_rt_info(["config", "model_parameters", "threshold"]).astype(float), 13.23, rel_tol=0.0001) + assert math.isclose(model.get_rt_info(["config", "model_parameters", "min"]).astype(float), -3.24543, rel_tol=0.0001) + assert math.isclose(model.get_rt_info(["config", "model_parameters", "max"]).astype(float), 3.234223, rel_tol=0.0001) + assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "type"]).astype(str) == "tree" + assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "directed"]).astype(bool) is True + + assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "float_empty"]).aslist() == [] + assert model.get_rt_info(["config", "model_parameters", "labels", "label_tree", "nodes"]).aslist() == [] + assert model.get_rt_info(["config", "model_parameters", "labels", "label_groups", "ids"]).aslist(str) == ["sasd", "fdfdfsdf"] + assert model.get_rt_info(["config", "model_parameters", "mean_values"]).aslist(float) == [22.3, 33.11, 44.0] + + rt_info = model.get_rt_info() + assert isinstance(rt_info["config"], dict) + + for key, value in rt_info.items(): + if key == "config": + for config_value in value: + assert config_value in ["type_of_model", "converter_type", "model_parameters"] + + for rt_info_val in model.get_rt_info(["config", "model_parameters", "labels", "label_tree"]).astype(dict): + assert rt_info_val in ["float_empty", "nodes", "type", "directed"] core = Core() xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path) diff --git a/src/bindings/python/tests/test_runtime/test_ovdict.py b/src/bindings/python/tests/test_runtime/test_ovdict.py new file mode 100644 index 00000000000000..460cf68d73f9bb --- /dev/null +++ b/src/bindings/python/tests/test_runtime/test_ovdict.py @@ -0,0 +1,258 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Mapping +import numpy as np +import pytest + +import openvino.runtime.opset10 as ops +from openvino.runtime import Core, ConstOutput, CompiledModel, InferRequest, Model +from openvino.runtime.ie_api import OVDict + + +def _get_ovdict( + device, + input_shape=None, + data_type=np.float32, + input_names=None, + output_names=None, + multi_output=False, + direct_infer=False, + split_num=5, +): + # Create model + # If model is multi-output (multi_output=True), input_shape must match + # requirements of split operation. + # TODO OpenSource: refactor it to be more generic + if input_shape is None: + input_shape = [1, 20] + if input_names is None: + input_names = ["data_0"] + if output_names is None: + output_names = ["output_0"] + if multi_output: + assert isinstance(output_names, (list, tuple)) + assert len(output_names) > 1 + assert len(output_names) == split_num + param = ops.parameter(input_shape, data_type, name=input_names[0]) + model = Model( + ops.split(param, 1, split_num) if multi_output else ops.abs(param), [param], + ) + # Manually name outputs + for i in range(len(output_names)): + model.output(i).tensor.names = {output_names[i]} + # Compile model + core = Core() + compiled_model = core.compile_model(model, device) + # Create test data + input_data = np.random.random(input_shape).astype(data_type) + # Two ways of infering + if direct_infer: + result = compiled_model(input_data) + assert result is not None + return result, compiled_model + + request = compiled_model.create_infer_request() + result = request.infer(input_data) + assert result is not None + return result, request + + +def _check_keys(keys, outs): + outs_iter = iter(outs) + for key in keys: + assert isinstance(key, ConstOutput) + assert key == next(outs_iter) + return True + + +def _check_values(result): + for value in result.values(): + assert isinstance(value, np.ndarray) + return True + + +def _check_items(result, outs, output_names): + i = 0 + for key, value in result.items(): + assert isinstance(key, ConstOutput) + assert isinstance(value, np.ndarray) + # Check values + assert np.equal(result[outs[i]], result[key]).all() + assert np.equal(result[outs[i]], result[i]).all() + assert np.equal(result[outs[i]], result[output_names[i]]).all() + i += 1 + return True + + +def _check_dict(result, obj, output_names=None): + if output_names is None: + output_names = ["output_0"] + + outs = obj.model_outputs if isinstance(obj, InferRequest) else obj.outputs + assert len(outs) == len(result) + assert len(outs) == len(output_names) + # Check for __iter__ + assert _check_keys(result, outs) + # Check for keys function + assert _check_keys(result.keys(), outs) + assert _check_values(result) + assert _check_items(result, outs, output_names) + assert all([output_names[i] in result.names()[i] for i in range(0, len(output_names))]) + + return True + + +@pytest.mark.parametrize("is_direct", [True, False]) +def test_ovdict_assign(device, is_direct): + result, _ = _get_ovdict(device, multi_output=False, direct_infer=is_direct) + + with pytest.raises(TypeError) as e: + result["some_name"] = 99 + assert "'OVDict' object does not support item assignment" in str(e.value) + + +@pytest.mark.parametrize("is_direct", [True, False]) +def test_ovdict_single_output_basic(device, is_direct): + result, obj = _get_ovdict(device, multi_output=False, direct_infer=is_direct) + + assert isinstance(result, OVDict) + if isinstance(obj, (InferRequest, CompiledModel)): + assert _check_dict(result, obj) + else: + raise TypeError("Unknown `obj` type!") + + +@pytest.mark.parametrize("is_direct", [True, False]) +def test_ovdict_wrong_key_type(device, is_direct): + result, _ = _get_ovdict(device, multi_output=False, direct_infer=is_direct) + + with pytest.raises(TypeError) as e: + _ = result[2.0] + assert "Unknown key type: " in str(e.value) + + +@pytest.mark.parametrize("is_direct", [True, False]) +def test_ovdict_single_output_noname(device, is_direct): + result, obj = _get_ovdict( + device, + multi_output=False, + direct_infer=is_direct, + output_names=[], + ) + + assert isinstance(result, OVDict) + + outs = obj.model_outputs if isinstance(obj, InferRequest) else obj.outputs + + assert isinstance(result[outs[0]], np.ndarray) + assert isinstance(result[0], np.ndarray) + + with pytest.raises(KeyError) as e0: + _ = result["some_name"] + assert "some_name" in str(e0.value) + + # Check if returned names are tuple with one empty set + assert len(result.names()) == 1 + assert result.names()[0] == set() + + +@pytest.mark.parametrize("is_direct", [True, False]) +def test_ovdict_single_output_wrongname(device, is_direct): + result, obj = _get_ovdict( + device, + multi_output=False, + direct_infer=is_direct, + output_names=["output_21"], + ) + + assert isinstance(result, OVDict) + + outs = obj.model_outputs if isinstance(obj, InferRequest) else obj.outputs + + assert isinstance(result[outs[0]], np.ndarray) + assert isinstance(result[0], np.ndarray) + + with pytest.raises(KeyError) as e: + _ = result["output_37"] + assert "output_37" in str(e.value) + + with pytest.raises(KeyError) as e: + _ = result[6] + assert "6" in str(e.value) + + +@pytest.mark.parametrize("is_direct", [True, False]) +@pytest.mark.parametrize("use_function", [True, False]) +def test_ovdict_single_output_dict(device, is_direct, use_function): + result, obj = _get_ovdict( + device, + multi_output=False, + direct_infer=is_direct, + ) + + assert isinstance(result, OVDict) + + outs = obj.model_outputs if isinstance(obj, InferRequest) else obj.outputs + native_dict = result.to_dict() if use_function else dict(result) + + assert issubclass(type(native_dict), dict) + assert not isinstance(native_dict, OVDict) + assert isinstance(native_dict[outs[0]], np.ndarray) + + with pytest.raises(KeyError) as e: + _ = native_dict["output_0"] + assert "output_0" in str(e.value) + + with pytest.raises(KeyError) as e: + _ = native_dict[0] + assert "0" in str(e.value) + + +@pytest.mark.parametrize("is_direct", [True, False]) +def test_ovdict_multi_output_basic(device, is_direct): + output_names = ["output_0", "output_1", "output_2", "output_3", "output_4"] + result, obj = _get_ovdict( + device, + multi_output=True, + direct_infer=is_direct, + output_names=output_names, + ) + + assert isinstance(result, OVDict) + if isinstance(obj, (InferRequest, CompiledModel)): + assert _check_dict(result, obj, output_names) + else: + raise TypeError("Unknown `obj` type!") + + +@pytest.mark.parametrize("is_direct", [True, False]) +@pytest.mark.parametrize("use_function", [True, False]) +def test_ovdict_multi_output_tuple0(device, is_direct, use_function): + output_names = ["output_0", "output_1"] + result, obj = _get_ovdict( + device, + input_shape=(1, 10), + multi_output=True, + direct_infer=is_direct, + split_num=2, + output_names=output_names, + ) + + out0, out1 = None, None + if use_function: + assert isinstance(result.to_tuple(), tuple) + out0, out1 = result.to_tuple() + else: + out0, out1 = result.values() + + assert out0 is not None + assert out1 is not None + assert isinstance(out0, np.ndarray) + assert isinstance(out1, np.ndarray) + + outs = obj.model_outputs if isinstance(obj, InferRequest) else obj.outputs + + assert np.equal(result[outs[0]], out0).all() + assert np.equal(result[outs[1]], out1).all() diff --git a/src/bindings/python/tests/test_runtime/test_properties.py b/src/bindings/python/tests/test_runtime/test_properties.py index 41558de5aa7fc8..f21a204ae8dd93 100644 --- a/src/bindings/python/tests/test_runtime/test_properties.py +++ b/src/bindings/python/tests/test_runtime/test_properties.py @@ -60,10 +60,17 @@ def test_properties_rw_base(): (properties.hint.PerformanceMode.CUMULATIVE_THROUGHPUT, "PerformanceMode.CUMULATIVE_THROUGHPUT", 3), ), ), + ( + properties.hint.SchedulingCoreType, + ( + (properties.hint.SchedulingCoreType.ANY_CORE, "SchedulingCoreType.ANY_CORE", 0), + (properties.hint.SchedulingCoreType.PCORE_ONLY, "SchedulingCoreType.PCORE_ONLY", 1), + (properties.hint.SchedulingCoreType.ECORE_ONLY, "SchedulingCoreType.ECORE_ONLY", 2), + ), + ), ( properties.hint.ExecutionMode, ( - (properties.hint.ExecutionMode.UNDEFINED, "ExecutionMode.UNDEFINED", -1), (properties.hint.ExecutionMode.PERFORMANCE, "ExecutionMode.PERFORMANCE", 1), (properties.hint.ExecutionMode.ACCURACY, "ExecutionMode.ACCURACY", 2), ), @@ -206,8 +213,8 @@ def test_properties_ro(ov_property_ro, expected_value): "AFFINITY", ((properties.Affinity.NONE, properties.Affinity.NONE),), ), - (properties.force_tbb_terminate, "FORCE_TBB_TERMINATE", ((True, True),)), - (properties.inference_precision, "INFERENCE_PRECISION_HINT", ((Type.f32, Type.f32),)), + (properties.force_tbb_terminate, "FORCE_TBB_TERMINATE", ((True, True), (False, False))), + (properties.enable_mmap, "ENABLE_MMAP", ((True, True), (False, False))), (properties.hint.inference_precision, "INFERENCE_PRECISION_HINT", ((Type.f32, Type.f32),)), ( properties.hint.model_priority, @@ -219,10 +226,35 @@ def test_properties_ro(ov_property_ro, expected_value): "PERFORMANCE_HINT", ((properties.hint.PerformanceMode.UNDEFINED, properties.hint.PerformanceMode.UNDEFINED),), ), + ( + properties.hint.enable_cpu_pinning, + "ENABLE_CPU_PINNING", + ( + (True, True), + (False, False), + (1, True), + (0, False), + ), + ), + ( + properties.hint.scheduling_core_type, + "SCHEDULING_CORE_TYPE", + ((properties.hint.SchedulingCoreType.PCORE_ONLY, properties.hint.SchedulingCoreType.PCORE_ONLY),), + ), + ( + properties.hint.enable_hyper_threading, + "ENABLE_HYPER_THREADING", + ( + (True, True), + (False, False), + (1, True), + (0, False), + ), + ), ( properties.hint.execution_mode, "EXECUTION_MODE_HINT", - ((properties.hint.ExecutionMode.UNDEFINED, properties.hint.ExecutionMode.UNDEFINED),), + ((properties.hint.ExecutionMode.PERFORMANCE, properties.hint.ExecutionMode.PERFORMANCE),), ), ( properties.hint.num_requests, @@ -247,6 +279,36 @@ def test_properties_ro(ov_property_ro, expected_value): (2.0, 2.0), ), ), + ( + properties.intel_auto.device_bind_buffer, + "DEVICE_BIND_BUFFER", + ( + (True, True), + (False, False), + (1, True), + (0, False), + ), + ), + ( + properties.intel_auto.enable_startup_fallback, + "ENABLE_STARTUP_FALLBACK", + ( + (True, True), + (False, False), + (1, True), + (0, False), + ), + ), + ( + properties.intel_auto.enable_runtime_fallback, + "ENABLE_RUNTIME_FALLBACK", + ( + (True, True), + (False, False), + (1, True), + (0, False), + ), + ), (properties.device.id, "DEVICE_ID", (("0", "0"),)), ( properties.log.level, @@ -319,12 +381,12 @@ def check(value1, value2): {"CPU": {"NUM_STREAMS": 2}}) check({"CPU": make_dict(properties.streams.num(2))}, {"CPU": {"NUM_STREAMS": properties.streams.Num(2)}}) - check({"GPU": make_dict(properties.inference_precision(Type.f32))}, + check({"GPU": make_dict(properties.hint.inference_precision(Type.f32))}, {"GPU": {"INFERENCE_PRECISION_HINT": Type.f32}}) - check({"CPU": make_dict(properties.streams.num(2), properties.inference_precision(Type.f32))}, + check({"CPU": make_dict(properties.streams.num(2), properties.hint.inference_precision(Type.f32))}, {"CPU": {"INFERENCE_PRECISION_HINT": Type.f32, "NUM_STREAMS": properties.streams.Num(2)}}) - check({"CPU": make_dict(properties.streams.num(2), properties.inference_precision(Type.f32)), - "GPU": make_dict(properties.streams.num(1), properties.inference_precision(Type.f16))}, + check({"CPU": make_dict(properties.streams.num(2), properties.hint.inference_precision(Type.f32)), + "GPU": make_dict(properties.streams.num(1), properties.hint.inference_precision(Type.f16))}, {"CPU": {"INFERENCE_PRECISION_HINT": Type.f32, "NUM_STREAMS": properties.streams.Num(2)}, "GPU": {"INFERENCE_PRECISION_HINT": Type.f16, "NUM_STREAMS": properties.streams.Num(1)}}) @@ -397,8 +459,11 @@ def test_single_property_setting(device): properties.cache_dir("./"), properties.inference_num_threads(9), properties.affinity(properties.Affinity.NONE), - properties.inference_precision(Type.f32), + properties.hint.inference_precision(Type.f32), properties.hint.performance_mode(properties.hint.PerformanceMode.LATENCY), + properties.hint.enable_cpu_pinning(True), + properties.hint.scheduling_core_type(properties.hint.SchedulingCoreType.PCORE_ONLY), + properties.hint.enable_hyper_threading(True), properties.hint.num_requests(12), properties.streams.num(5), ], @@ -409,8 +474,11 @@ def test_single_property_setting(device): properties.cache_dir(): "./", properties.inference_num_threads(): 9, properties.affinity(): properties.Affinity.NONE, - properties.inference_precision(): Type.f32, + properties.hint.inference_precision(): Type.f32, properties.hint.performance_mode(): properties.hint.PerformanceMode.LATENCY, + properties.hint.enable_cpu_pinning(): True, + properties.hint.scheduling_core_type(): properties.hint.SchedulingCoreType.PCORE_ONLY, + properties.hint.enable_hyper_threading(): True, properties.hint.num_requests(): 12, properties.streams.num(): 5, }, @@ -422,6 +490,7 @@ def test_single_property_setting(device): properties.affinity(): "NONE", "INFERENCE_PRECISION_HINT": Type.f32, properties.hint.performance_mode(): properties.hint.PerformanceMode.LATENCY, + properties.hint.scheduling_core_type(): properties.hint.SchedulingCoreType.PCORE_ONLY, properties.hint.num_requests(): 12, "NUM_STREAMS": properties.streams.Num(5), }, diff --git a/src/bindings/python/tests/test_runtime/test_type.py b/src/bindings/python/tests/test_runtime/test_type.py index b31f36edf9f7e2..28ed0708d7f3e0 100644 --- a/src/bindings/python/tests/test_runtime/test_type.py +++ b/src/bindings/python/tests/test_runtime/test_type.py @@ -64,11 +64,18 @@ def test_basic_ovtypes(ovtype, assert ovtype.is_static() is static_flag assert ovtype.is_dynamic() is dynamic_flag assert ovtype.is_real() is real_flag + assert ovtype.real is real_flag assert ovtype.is_integral() is integral_flag + assert ovtype.integral is integral_flag assert ovtype.is_signed() is signed_flag + assert ovtype.signed is signed_flag assert ovtype.is_quantized() is quantized_flag + assert ovtype.quantized is quantized_flag assert ovtype.get_type_name() == type_name + assert ovtype.type_name == type_name + assert ovtype.get_size() == type_size assert ovtype.size == type_size + assert ovtype.get_bitwidth() == type_bitwidth assert ovtype.bitwidth == type_bitwidth @@ -77,15 +84,22 @@ def test_undefined_ovtype(): assert ov_type.is_static() is True assert ov_type.is_dynamic() is False assert ov_type.is_real() is False + assert ov_type.real is False assert ov_type.is_integral() is True + assert ov_type.integral is True assert ov_type.is_signed() is False + assert ov_type.signed is False assert ov_type.is_quantized() is False + assert ov_type.quantized is False assert ov_type.get_type_name() == "undefined" + assert ov_type.type_name == "undefined" + assert ov_type.get_size() == 0 assert ov_type.size == 0 # Note: might depend on the system import sys assert ov_type.bitwidth == sys.maxsize * 2 + 1 + assert ov_type.get_bitwidth() == sys.maxsize * 2 + 1 def test_dynamic_ov_type(): @@ -98,7 +112,9 @@ def test_dynamic_ov_type(): assert ov_type.is_quantized() is False assert ov_type.get_type_name() == "dynamic" assert ov_type.size == 0 + assert ov_type.get_size() == 0 assert ov_type.bitwidth == 0 + assert ov_type.get_bitwidth() == 0 @pytest.mark.parametrize(("ovtype_one", "ovtype_two", "expected"), [ diff --git a/src/bindings/python/tests/test_transformations/test_offline_api.py b/src/bindings/python/tests/test_transformations/test_offline_api.py index fdc7eec0048ce9..e153fc9412c19a 100644 --- a/src/bindings/python/tests/test_transformations/test_offline_api.py +++ b/src/bindings/python/tests/test_transformations/test_offline_api.py @@ -354,5 +354,6 @@ def test_flush_fp32_subnormals_to_zero(): apply_moc_transformations(model, cf=False, smart_reshape=True) # apply_flush_fp32_subnormals_to_zero is called inside - assert np.all(weights.data[4:8] != subnorm_val) - assert np.all(weights.data[4:8] == 0.0) + new_weights = add_node.input_value(1).get_node() + assert np.all(new_weights.data[4:8] != subnorm_val) + assert np.all(new_weights.data[4:8] == 0.0) diff --git a/src/bindings/python/tests/test_utils/test_data_dispatch.py b/src/bindings/python/tests/test_utils/test_data_dispatch.py index 254cf890458bb8..e2ce00f10e7482 100644 --- a/src/bindings/python/tests/test_utils/test_data_dispatch.py +++ b/src/bindings/python/tests/test_utils/test_data_dispatch.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2018-2022 Intel Corporation +# Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import os @@ -157,8 +157,8 @@ def test_ndarray_copied_dispatcher(device, input_shape): result, infer_request = _run_dispatcher(device, test_data, False, input_shape) assert result == {} - assert np.array_equal(infer_request.inputs[0].data, test_data) + assert np.array_equal(infer_request.input_tensors[0].data, test_data) test_data[0] = 2.0 - assert not np.array_equal(infer_request.inputs[0].data, test_data) + assert not np.array_equal(infer_request.input_tensors[0].data, test_data) diff --git a/src/bindings/python/tests_compatibility/__init__.py b/src/bindings/python/tests_compatibility/__init__.py index 7b5d7217cd8ed1..8b4b56d04d0daa 100644 --- a/src/bindings/python/tests_compatibility/__init__.py +++ b/src/bindings/python/tests_compatibility/__init__.py @@ -86,6 +86,28 @@ def xfail_test(reason="Mark the test as expected to fail", strict=True): xfail_issue_47323 = xfail_test(reason="RuntimeError: The plugin does not support FP64") xfail_issue_73538 = xfail_test(reason="OneHot: Unsupported negative indices, " "AssertionError: Mismatched elements.") +xfail_issue_99949 = xfail_test(reason="Bitwise operators are not supported") +xfail_issue_99950 = xfail_test(reason="CenterCropPad func is not supported") +xfail_issue_99952 = xfail_test(reason="Col2Im operator is not supported") +xfail_issue_99954 = xfail_test(reason="Constant Pad - RuntimeError: Shape inference of Reference node with name y failed") +xfail_issue_99955 = xfail_test(reason="GroupNorm is not supported") +xfail_issue_99957 = xfail_test(reason="LayerNorm - RuntimeError: While validating node ''") +xfail_issue_99958 = xfail_test(reason="LogSoftmax - Results mismatch") +xfail_issue_99959 = xfail_test(reason="Mish function is not supported") +xfail_issue_99960 = xfail_test(reason="MVN - Results mismatch") +xfail_issue_99961 = xfail_test(reason="Optional has/get element operators are not supported)'") +xfail_issue_99962 = pytest.mark.skip(reason="ReduceL1/L2 - Unrecognized attribute: axes for operator ReduceL1/L2") +xfail_issue_99968 = xfail_test(reason="ReduceL1/L2 - Results mismatch or unsupported ReduceSum with " + "dynamic rank by CPU plugin") +xfail_issue_99969 = xfail_test(reason="Resize - Results mismatch / " + "RuntimeError: While validating ONNX node '' / " + "RuntimeError: Check '(false)' failed at onnx/frontend/src/op/resize.cpp") +xfail_issue_99970 = xfail_test(reason="Scatter and ScatterND - RuntimeError: Check '(reduction == none)' failed at " + "src/frontends/onnx/frontend/src/op/scatter_elements.cpp OR at " + "src/frontends/onnx/frontend/src/op/scatter_nd") +xfail_issue_99972 = xfail_test(reason="Softmax - Results mismatch") +xfail_issue_99973 = xfail_test(reason="Split - RuntimeError: While validating ONNX node " + "''") # Model MSFT issues: xfail_issue_37957 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations: " @@ -122,7 +144,6 @@ def xfail_test(reason="Mark the test as expected to fail", strict=True): xfail_issue_63033 = xfail_test(reason="BatchNormalization: Training mode is not supported") xfail_issue_63036 = xfail_test(reason="Changes in ConvTranspose padding") -xfail_issue_63039 = xfail_test(reason="Result mismatches with UINT8 operations") xfail_issue_63043 = xfail_test(reason="Recurrent node expects constants as W, R, B inputs.") skip_rng_tests = pytest.mark.skip(reason="Tests use random number generator with no seed.") diff --git a/src/bindings/python/tests_compatibility/test_ngraph/test_basic.py b/src/bindings/python/tests_compatibility/test_ngraph/test_basic.py index de83f6a77089bf..5acc1a29fd2d49 100644 --- a/src/bindings/python/tests_compatibility/test_ngraph/test_basic.py +++ b/src/bindings/python/tests_compatibility/test_ngraph/test_basic.py @@ -414,9 +414,3 @@ def test_sink_function_ctor(): assert len(function.get_results()) == 1 assert function.get_friendly_name() == "TestFunction" - -def test_node_version(): - node = ng.add([1], [2]) - - assert node.get_version() == 1 - assert node.version == 1 diff --git a/src/bindings/python/tests_compatibility/test_ngraph/test_core.py b/src/bindings/python/tests_compatibility/test_ngraph/test_core.py index 9b50732396e5a5..949946ef842bdc 100644 --- a/src/bindings/python/tests_compatibility/test_ngraph/test_core.py +++ b/src/bindings/python/tests_compatibility/test_ngraph/test_core.py @@ -250,10 +250,10 @@ def test_discrete_type_info(): assert n1.get_type_info().name == "TopK" assert n3.get_type_info().name == "Sin" assert n1.type_info.name == n2.type_info.name - assert n1.type_info.version == n2.type_info.version + assert n1.type_info.version_id == n2.type_info.version_id assert n1.type_info.parent == n2.type_info.parent assert n1.get_type_info().name == n2.get_type_info().name - assert n1.get_type_info().version == n2.get_type_info().version + assert n1.get_type_info().version_id == n2.get_type_info().version_id assert n1.get_type_info().parent == n2.get_type_info().parent assert n1.get_type_info().name != n3.get_type_info().name assert n1.get_type_info().name > n3.get_type_info().name diff --git a/src/bindings/python/tests_compatibility/test_ngraph/test_create_op.py b/src/bindings/python/tests_compatibility/test_ngraph/test_create_op.py index 7b084bc77a81d0..7ec5a26109ab49 100644 --- a/src/bindings/python/tests_compatibility/test_ngraph/test_create_op.py +++ b/src/bindings/python/tests_compatibility/test_ngraph/test_create_op.py @@ -11,6 +11,7 @@ import ngraph.opset1 as ng_opset1 import ngraph.opset5 as ng_opset5 import ngraph.opset10 as ng_opset10 +import ngraph.opset11 as ng_opset11 from ngraph.utils.types import make_constant_node from ngraph.exceptions import UserInputError from ngraph.impl import Type @@ -2259,13 +2260,33 @@ def test_interpolate_opset10(dtype, expected_shape, shape_calculation_mode): mode = "cubic" node = ng_opset10.interpolate(image=image_node, output_shape=output_shape, scales=scales, - axes=axes, - mode=mode, shape_calculation_mode=shape_calculation_mode) + axes=axes,mode=mode, shape_calculation_mode=shape_calculation_mode) assert node.get_type_name() == "Interpolate" assert node.get_output_size() == 1 assert list(node.get_output_shape(0)) == expected_shape +@pytest.mark.parametrize( + ("expected_shape", "shape_calculation_mode", "input_value"), + [ + ([1, 3, 64, 64], "scales", np.array([1 / 16, 1 / 16], dtype=np.float32)), + ([1, 3, 256, 256], "sizes", np.array([256, 256], dtype=np.int32)), + ], +) +@pytest.mark.parametrize("dtype", np_types) +def test_interpolate_opset11(dtype, expected_shape, shape_calculation_mode, input_value): + + image_shape = [1, 3, 1024, 1024] + image_node = ng.parameter(image_shape, dtype, name="Image") + axes = [2, 3] + mode = "bilinear_pillow" + + node = ng_opset11.interpolate(image=image_node, scales_or_sizes=input_value, axes=axes, mode=mode, + shape_calculation_mode=shape_calculation_mode) + assert node.get_type_name() == "Interpolate" + assert node.get_output_size() == 1 + assert list(node.get_output_shape(0)) == expected_shape + def test_is_finite_opset10(): input_shape = [1, 2, 3, 4] input_node = ng.parameter(input_shape, np.float32, name="InputData") @@ -2391,3 +2412,16 @@ def test_unique_opset10(): assert node.get_output_element_type(1) == Type.i64 assert node.get_output_element_type(2) == Type.i64 assert node.get_output_element_type(3) == Type.i64 + + +def test_topk_opset11(): + data_shape = [1, 3, 256] + data = ng.parameter(data_shape, dtype=np.int32, name="Data") + k_val = np.int32(3) + axis = np.int32(-1) + node = ng_opset11.topk(data, k_val, axis, "min", "value", stable=True) + + assert node.get_type_name() == "TopK" + assert node.get_output_size() == 2 + assert list(node.get_output_shape(0)) == [1, 3, 3] + assert list(node.get_output_shape(1)) == [1, 3, 3] diff --git a/src/bindings/python/tests_compatibility/test_onnx/test_backend.py b/src/bindings/python/tests_compatibility/test_onnx/test_backend.py index 89b7afcb47e4af..26bb2b5def0187 100644 --- a/src/bindings/python/tests_compatibility/test_onnx/test_backend.py +++ b/src/bindings/python/tests_compatibility/test_onnx/test_backend.py @@ -37,7 +37,6 @@ xfail_issue_58033, xfail_issue_63033, xfail_issue_63036, - xfail_issue_63039, xfail_issue_63043, xfail_issue_63137, xfail_issue_63138, @@ -51,6 +50,21 @@ xfail_issue_91151, xfail_issue_91490, xfail_issue_101965, + xfail_issue_99949, + xfail_issue_99950, + xfail_issue_99952, + xfail_issue_99954, + xfail_issue_99955, + xfail_issue_99957, + xfail_issue_99958, + xfail_issue_99959, + xfail_issue_99960, + xfail_issue_99961, + xfail_issue_99968, + xfail_issue_99969, + xfail_issue_99970, + xfail_issue_99972, + xfail_issue_99973, skip_dynamic_model, ) from tests_compatibility.test_onnx.utils.onnx_backend import OpenVinoTestBackend @@ -282,10 +296,6 @@ def expect_fail(test_case_path, xfail): # type: (str) -> None "OnnxBackendNodeModelTest.test_batchnorm_example_training_mode_cpu", ), (xfail_issue_63036, "OnnxBackendNodeModelTest.test_convtranspose_autopad_same_cpu"), - ( - xfail_issue_63039, - "OnnxBackendNodeModelTest.test_div_uint8_cpu", - ), ( xfail_issue_63043, "OnnxBackendNodeModelTest.test_gru_batchwise_cpu", @@ -499,7 +509,220 @@ def expect_fail(test_case_path, xfail): # type: (str) -> None "OnnxBackendNodeModelTest.test_hammingwindow_symmetric_expanded_cpu", "OnnxBackendNodeModelTest.test_hannwindow_expanded_cpu", "OnnxBackendNodeModelTest.test_hannwindow_symmetric_expanded_cpu", - ) + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_and_pad_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_keep_dims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_keep_dims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_expanded_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_pad_expanded_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_1d_opset13_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_1d_opset18_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_2d_opset13_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_2d_opset18_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_default_axis_opset13_cpu", + "OnnxBackendNodeModelTest.test_split_variable_parts_default_axis_opset18_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_keep_dims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_keep_dims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_negative_axes_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_keepdims_random_expanded_cpu", + ), + ( + xfail_issue_99949, + "OnnxBackendNodeModelTest.test_bitwise_and_i16_3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_and_i32_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_and_ui64_bcast_3v1d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_not_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_not_3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_not_4d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_i16_4d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_i32_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_ui64_bcast_3v1d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_ui8_bcast_4v3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_i16_3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_i32_2d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_and_ui8_bcast_4v3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_or_ui8_bcast_4v3d_cpu", + "OnnxBackendNodeModelTest.test_bitwise_xor_ui64_bcast_3v1d_cpu", + ), + ( + xfail_issue_99950, + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_and_pad_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_chw_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_chw_expanded_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_hwc_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_axes_hwc_expanded_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_crop_cpu", + "OnnxBackendNodeModelTest.test_center_crop_pad_pad_cpu", + ), + ( + xfail_issue_99952, + "OnnxBackendNodeModelTest.test_col2im_5d_cpu", + "OnnxBackendNodeModelTest.test_col2im_cpu", + "OnnxBackendNodeModelTest.test_col2im_dilations_cpu", + "OnnxBackendNodeModelTest.test_col2im_pads_cpu", + "OnnxBackendNodeModelTest.test_col2im_strides_cpu", + ), + ( + xfail_issue_99954, + "OnnxBackendNodeModelTest.test_constant_pad_axes_cpu", + ), + ( + xfail_issue_99955, + "OnnxBackendNodeModelTest.test_group_normalization_epsilon_cpu", + "OnnxBackendNodeModelTest.test_group_normalization_epsilon_expanded_cpu", + "OnnxBackendNodeModelTest.test_group_normalization_example_cpu", + "OnnxBackendNodeModelTest.test_group_normalization_example_expanded_cpu", + ), + ( + xfail_issue_99957, + "OnnxBackendNodeModelTest.test_layer_normalization_2d_axis1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_2d_axis_negative_1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis1_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis2_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis2_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis3_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis_negative_1_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis_negative_2_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_4d_axis_negative_3_expanded_ver18_cpu", + "OnnxBackendNodeModelTest.test_layer_normalization_default_axis_expanded_ver18_cpu", + ), + ( + xfail_issue_99958, + "OnnxBackendNodeModelTest.test_logsoftmax_large_number_expanded_ver18_cpu", + ), + ( + xfail_issue_99959, + "OnnxBackendNodeModelTest.test_mish_cpu", + ), + ( + xfail_issue_99960, + "OnnxBackendNodeModelTest.test_mvn_expanded_ver18_cpu", + ), + ( + xfail_issue_99961, + "OnnxBackendNodeModelTest.test_optional_get_element_optional_sequence_cpu", + "OnnxBackendNodeModelTest.test_optional_get_element_optional_tensor_cpu", + "OnnxBackendNodeModelTest.test_optional_get_element_tensor_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_name_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_name_tensor_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_no_input_tensor_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_empty_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_optional_input_cpu", + "OnnxBackendNodeModelTest.test_optional_has_element_tensor_input_cpu", + ), + ( + xfail_issue_99968, + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_negative_axes_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_negative_axes_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_negative_axes_keep_dims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_negative_axes_keep_dims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_asc_axes_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_asc_axes_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_desc_axes_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_desc_axes_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_negative_axes_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_max_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_mean_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_min_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_prod_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_do_not_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_negative_axes_keepdims_example_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_negative_axes_keepdims_random_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_negative_axes_keep_dims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l1_negative_axes_keep_dims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_negative_axes_keep_dims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_l2_negative_axes_keep_dims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_negative_axes_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_log_sum_exp_negative_axes_keepdims_random_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_negative_axes_keepdims_example_expanded_cpu", + "OnnxBackendNodeModelTest.test_reduce_sum_square_negative_axes_keepdims_random_expanded_cpu", + ), + ( + xfail_issue_99969, + "OnnxBackendNodeModelTest.test_resize_downsample_scales_cubic_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_scales_linear_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_cubic_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_linear_antialias_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_not_smaller_cpu", + "OnnxBackendNodeModelTest.test_resize_downsample_sizes_nearest_not_larger_cpu", + "OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_axes_2_3_cpu", + "OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_axes_3_2_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_nearest_axes_2_3_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_scales_nearest_axes_3_2_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_axes_2_3_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_axes_3_2_cpu", + "OnnxBackendNodeModelTest.test_resize_upsample_sizes_nearest_not_larger_cpu", + ), + ( + xfail_issue_99970, + "OnnxBackendNodeModelTest.test_scatter_elements_with_reduction_max_cpu", + "OnnxBackendNodeModelTest.test_scatter_elements_with_reduction_min_cpu", + "OnnxBackendNodeModelTest.test_scatternd_max_cpu", + "OnnxBackendNodeModelTest.test_scatternd_min_cpu", + ), + ( + xfail_issue_99972, + "OnnxBackendNodeModelTest.test_softmax_large_number_expanded_ver18_cpu", + ), + ( + xfail_issue_99973, + "OnnxBackendNodeModelTest.test_split_1d_uneven_split_opset18_cpu", + "OnnxBackendNodeModelTest.test_split_2d_uneven_split_opset18_cpu", + "OnnxBackendNodeModelTest.test_split_zero_size_splits_opset13_cpu", + "OnnxBackendNodeModelTest.test_split_zero_size_splits_opset18_cpu", + ), ] for test_group in tests_expected_to_fail: diff --git a/src/bindings/python/tests_compatibility/test_onnx/test_ops_reduction.py b/src/bindings/python/tests_compatibility/test_onnx/test_ops_reduction.py index 20134cddc6908f..4122d1a3158689 100644 --- a/src/bindings/python/tests_compatibility/test_onnx/test_ops_reduction.py +++ b/src/bindings/python/tests_compatibility/test_onnx/test_ops_reduction.py @@ -5,6 +5,7 @@ import onnx import pytest +from tests_compatibility import xfail_issue_99962 from tests_compatibility.runtime import get_runtime from tests_compatibility.test_onnx.utils import ( run_node, @@ -79,6 +80,7 @@ def test_reduce_operation_keepdims_none_axes(operation, ref_operation): ref_operation(reduce_data, keepdims=True)) +@xfail_issue_99962 @pytest.mark.parametrize("operation, ref_operation", reduce_operation_parameters_as_attr) @pytest.mark.parametrize("axes", reduce_axis_parameters) def test_reduce_operation_keepdims_with_axes_as_attr(operation, ref_operation, axes): @@ -93,6 +95,7 @@ def test_reduce_operation_keepdims_with_axes_as_const(operation, ref_operation, ref_operation(reduce_data, keepdims=True, axis=axes)) +@xfail_issue_99962 @pytest.mark.parametrize("axes", [ None, (0,), @@ -134,6 +137,7 @@ def test_reduce_operation_no_keepdims_axes_as_const(operation, ref_operation, ax ref_operation(reduce_data, keepdims=False)) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_l1(reduction_axes): shape = [2, 4, 3, 2] @@ -171,6 +175,7 @@ def test_reduce_l1_default_axes(): assert np.allclose(expected, ng_result) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_l2(reduction_axes): shape = [2, 4, 3, 2] @@ -209,6 +214,7 @@ def test_reduce_l2_default_axes(): assert np.allclose(expected, ng_result) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_log_sum(reduction_axes): shape = [2, 4, 3, 2] @@ -246,6 +252,7 @@ def test_reduce_log_sum_default_axes(): assert np.allclose(expected, ng_result) +@xfail_issue_99962 def test_reduce_log_sum_exp(): def logsumexp(data, axis=None, keepdims=True): return np.log(np.sum(np.exp(data), axis=axis, keepdims=keepdims)) @@ -283,6 +290,7 @@ def logsumexp(data, axis=None, keepdims=True): ) +@xfail_issue_99962 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_sum_square(reduction_axes): shape = [2, 4, 3, 2] diff --git a/src/bindings/python/tests_compatibility/test_onnx/test_ops_unary.py b/src/bindings/python/tests_compatibility/test_onnx/test_ops_unary.py index 827f226677060e..ddbd8dd53e4a4a 100644 --- a/src/bindings/python/tests_compatibility/test_onnx/test_ops_unary.py +++ b/src/bindings/python/tests_compatibility/test_onnx/test_ops_unary.py @@ -326,7 +326,7 @@ def test_identity(): def test_cast_to_bool(val_type, input_data): expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -343,7 +343,7 @@ def test_cast_to_float(val_type, range_start, range_end, in_dtype): input_data = np.random.randint(range_start, range_end, size=(2, 2), dtype=in_dtype) expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -359,7 +359,7 @@ def test_cast_to_int(val_type): input_data = np.ceil(-8 + np.random.rand(2, 3, 4) * 16) expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -372,7 +372,7 @@ def test_cast_to_uint(val_type): input_data = np.ceil(np.random.rand(2, 3, 4) * 16) expected = np.array(input_data, dtype=val_type) - model = get_node_model("Cast", input_data, opset=6, to=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[val_type]) + model = get_node_model("Cast", input_data, opset=6, to=onnx.helper.np_dtype_to_tensor_dtype(val_type)) result = run_model(model, [input_data]) assert np.allclose(result, expected) @@ -455,7 +455,7 @@ def test_constant(value_type): outputs=["values"], value=onnx.helper.make_tensor( name="const_tensor", - data_type=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(value_type)], + data_type=onnx.helper.np_dtype_to_tensor_dtype(np.dtype(value_type)), dims=values.shape, vals=values.flatten(), ), @@ -473,7 +473,7 @@ def test_constant_err(): outputs=["values"], value=onnx.helper.make_tensor( name="const_tensor", - data_type=onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(np.float16)], + data_type=onnx.helper.np_dtype_to_tensor_dtype(np.dtype(np.float16)), dims=values.shape, vals=values.flatten(), ), diff --git a/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_backend.py b/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_backend.py index 5c06a5e874eaa2..a3eea6a2ac9aed 100644 --- a/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_backend.py +++ b/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_backend.py @@ -13,11 +13,11 @@ import numpy import onnx from onnx.backend.base import Backend, BackendRep -from onnx.helper import make_graph, make_model, make_tensor_value_info +from onnx.helper import make_graph, make_model, make_tensor_value_info, np_dtype_to_tensor_dtype from ngraph.impl import Function from tests_compatibility.runtime import get_runtime -from tests_compatibility.test_onnx.utils.onnx_helpers import import_onnx_model, np_dtype_to_tensor_type +from tests_compatibility.test_onnx.utils.onnx_helpers import import_onnx_model class OpenVinoOnnxBackendRep(BackendRep): @@ -80,15 +80,15 @@ def run_node( ): # type: (...) -> Optional[Tuple[Any, ...]] """Prepare and run a computation on an ONNX node.""" # default values for input/output tensors - input_tensor_types = [np_dtype_to_tensor_type(node_input.dtype) for node_input in inputs] - output_tensor_types = [onnx.TensorProto.FLOAT for idx in range(len(node.output))] + input_tensor_types = [np_dtype_to_tensor_dtype(node_input.dtype) for node_input in inputs] + output_tensor_types = [onnx.TensorProto.FLOAT for _ in range(len(node.output))] output_tensor_shapes = [()] # type: List[Tuple[int, ...]] if outputs_info is not None: output_tensor_types = [ - np_dtype_to_tensor_type(dtype) for (dtype, shape) in outputs_info + np_dtype_to_tensor_dtype(dtype) for (dtype, _) in outputs_info ] - output_tensor_shapes = [shape for (dtype, shape) in outputs_info] + output_tensor_shapes = [shape for (_, shape) in outputs_info] input_tensors = [ make_tensor_value_info(name, tensor_type, value.shape) diff --git a/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_helpers.py b/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_helpers.py index 063af326802fb3..c5e774da297142 100644 --- a/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_helpers.py +++ b/src/bindings/python/tests_compatibility/test_onnx/utils/onnx_helpers.py @@ -1,24 +1,13 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import numpy as np import onnx -from onnx.mapping import NP_TYPE_TO_TENSOR_TYPE from openvino.inference_engine import IECore import ngraph as ng from ngraph.impl import Function -def np_dtype_to_tensor_type(data_type: np.dtype) -> int: - """Return TensorProto type for provided numpy dtype. - - :param data_type: Numpy data type object. - :return: TensorProto.DataType enum value for corresponding type. - """ - return NP_TYPE_TO_TENSOR_TYPE[data_type] - - def import_onnx_model(model: onnx.ModelProto) -> Function: onnx.checker.check_model(model) model_byte_string = model.SerializeToString() diff --git a/src/bindings/python/wheel/setup.py b/src/bindings/python/wheel/setup.py index 7af4d4221e7f2d..4e00f24586805f 100644 --- a/src/bindings/python/wheel/setup.py +++ b/src/bindings/python/wheel/setup.py @@ -8,6 +8,7 @@ import subprocess # nosec import typing import platform +import re import multiprocessing from fnmatch import fnmatchcase from pathlib import Path @@ -97,6 +98,7 @@ "name": "tbb", "prefix": "libs.tbb", "install_dir": TBB_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "pugixml_libs": { @@ -294,42 +296,72 @@ def configure(self, install_cfg): ): set_rpath(comp_data["rpath"], os.path.realpath(path)) + def get_reallink(self, link_file): + real_name = link_file + while True: + real_name = os.readlink(real_name) + if not os.path.isabs(real_name): + real_name = os.path.join(os.path.dirname(link_file), real_name) + if not Path(real_name).is_symlink(): + break + return real_name + def generate_package(self, src_dirs): """Collect package data files from preinstalled dirs and put all runtime libraries to the subpackage.""" # additional blacklist filter, just to fix cmake install issues - blacklist = [".lib", ".pdb", "_debug.dll", "_debug.dylib"] + blacklist_patterns = ["^.*\\.lib$", "^.*\\.pdb$", "^.*_debug\\.dll$", "^.*_debug\\.\\d*\\.dylib$", "^.*_debug\\.so\\.\\d*$", "^.*\\.la$"] package_dir = os.path.join(get_package_dir(PY_INSTALL_CFG), WHEEL_LIBS_INSTALL_DIR) for src_dir in src_dirs: local_base_dir = Path(src_dir) - - # skip symlinks of higher level like libX.so or libX.dylib + # Wheel package content must not contain symlinks + # the block handles two kinds of soft links, take the library on linux as an example + # the first case: there are two soft links pointing to the real file, + # input is libX.so->libX.so.Y and libX.so.Y->libX.so.Y.Z (e.g. hwloc library in oneTBB package) + # input is libX.so->libX.so.Y.Z and libX.so.Y->libX.so.Y.Z (e.g. oneTBB library) + # the second case: there is one soft link pointing to the real file + # process results of the above two cases: remove soft links(libX.so and libX.so.Y), rename libX.so.Y.Z to libX.so.Y + file_dict = {} + # step 1: + # record real files and its symlinks {real file: soft link} + # if there are two soft links pointing to the same file, like libX.so and libX.so.Y(including the above two cases), + # only record the libX.so.Y and remove libX.so for symlink in local_base_dir.rglob("*"): if symlink.is_symlink(): - file_name = os.readlink(symlink) - if not os.path.isabs(file_name): - file_name = os.path.join(os.path.dirname(symlink), file_name) - if Path(file_name).is_symlink(): - self.announce(f"Unlink symlink {symlink}, use {file_name} instead", level=3) - os.unlink(symlink) - - # transform libX.so.Y / libX.Y.dylib symlinks to real files - for symlink in local_base_dir.rglob("*"): - if symlink.is_symlink(): - file_name = os.readlink(symlink) - if not os.path.isabs(file_name): - file_name = os.path.join(os.path.dirname(symlink), file_name) - - os.unlink(symlink) - os.rename(file_name, symlink) - self.announce(f"Resolved symlink {symlink} as {file_name}", level=3) + real_name = self.get_reallink(symlink) + if real_name in file_dict: + link_file_name_old = os.path.basename(file_dict[real_name]) + link_file_name_new = os.path.basename(symlink) + if len(link_file_name_new) > len(link_file_name_old): + # replace libX.so/libX.dylib with libX.so.Y/libX.Y.dylib + self.announce(f"Unlink symlink {file_dict[real_name]}, use {symlink} instead", level=3) + os.unlink(file_dict[real_name]) + file_dict[real_name] = symlink + else: + self.announce(f"Unlink symlink {symlink}, use {file_dict[real_name]} instead", level=3) + os.unlink(symlink) + else: + file_dict[real_name] = symlink + + # step 2: + # according to the corresponding relationship (file_dict), + # remove the reserved soft link and rename the real file to the name of its soft link + for real_name, symlink in file_dict.items(): + os.unlink(symlink) + os.rename(real_name, symlink) + self.announce(f"Resolved symlink {symlink} as {real_name}", level=3) # copy so / dylib files to WHEEL_LIBS_INSTALL_DIR for file_path in local_base_dir.rglob("*"): file_name = os.path.basename(file_path) if file_path.is_symlink(): sys.exit(f"Wheel package content must not contain symlinks {file_path}") - if file_path.is_file() and not any(file_name.endswith(ext) for ext in blacklist): + blacklisted = False + for pattern in blacklist_patterns: + if re.match(pattern, file_name) is not None: + blacklisted = True + break + if file_path.is_file() and not blacklisted: dst_file = os.path.join(package_dir, os.path.relpath(file_path, local_base_dir)) os.makedirs(os.path.dirname(dst_file), exist_ok=True) copyfile(file_path, dst_file) diff --git a/src/cmake/ie_parallel.cmake b/src/cmake/ie_parallel.cmake index 1e6b942f887649..7c58235c57eb61 100644 --- a/src/cmake/ie_parallel.cmake +++ b/src/cmake/ie_parallel.cmake @@ -2,6 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # +if(NOT ANDROID) + find_package(PkgConfig QUIET) +endif() + function(_ov_get_tbb_location tbb_target _tbb_lib_location_var) if(NOT TBB_FOUND) return() @@ -63,48 +67,45 @@ macro(ov_find_package_tbb) unset(TBB_DIR) # try tbb.pc from system - if(NOT ANDROID AND ENABLE_SYSTEM_TBB) - find_package(PkgConfig QUIET) - if(PkgConfig_FOUND) - macro(_ov_pkg_config_tbb_unset) - # unset since it affects OpenVINOConfig.cmake.in - unset(tbb_FOUND) - unset(tbb_FOUND CACHE) - endmacro() - pkg_search_module(tbb QUIET - IMPORTED_TARGET GLOBAL - tbb) - if(tbb_FOUND) - # parse version - string(REGEX REPLACE "~.*" "" tbb_VERSION_PATCHED "${tbb_VERSION}") - if(tbb_VERSION_PATCHED AND tbb_VERSION_PATCHED VERSION_LESS _ov_minimal_tbb_version) - _ov_pkg_config_tbb_unset() - message(WARNING "Found TBB ${tbb_VERSION} via ${PKG_CONFIG_EXECUTABLE} while OpenVINO requies ${_ov_minimal_tbb_version} at least") - elseif(TARGET PkgConfig::tbb) - add_library(TBB::tbb ALIAS PkgConfig::tbb) - set(TBB_VERSION ${tbb_VERSION}) - set(TBB_FOUND ${tbb_FOUND}) - - # note: for python wheels we need to find and install tbbmalloc as well - _ov_get_tbb_location(PkgConfig::tbb tbb_loc) - string(REPLACE "tbb" "tbbmalloc" tbbmalloc_loc "${tbb_loc}") - if(EXISTS "${tbbmalloc_loc}") - add_library(TBB::tbbmalloc SHARED IMPORTED) - set_target_properties(TBB::tbbmalloc PROPERTIES IMPORTED_LOCATION ${tbbmalloc_loc}) - endif() - - message(STATUS "${PKG_CONFIG_EXECUTABLE}: tbb (${tbb_VERSION}) is found at ${tbb_PREFIX}") + if(ENABLE_SYSTEM_TBB AND PkgConfig_FOUND) + macro(_ov_pkg_config_tbb_unset) + # unset since it affects OpenVINOConfig.cmake.in + unset(tbb_FOUND) + unset(tbb_FOUND CACHE) + endmacro() + pkg_search_module(tbb QUIET + IMPORTED_TARGET GLOBAL + tbb) + if(tbb_FOUND) + # parse version + string(REGEX REPLACE "~.*" "" tbb_VERSION_PATCHED "${tbb_VERSION}") + if(tbb_VERSION_PATCHED AND tbb_VERSION_PATCHED VERSION_LESS _ov_minimal_tbb_version) + _ov_pkg_config_tbb_unset() + message(WARNING "Found TBB ${tbb_VERSION} via ${PKG_CONFIG_EXECUTABLE} while OpenVINO requies ${_ov_minimal_tbb_version} at least") + elseif(TARGET PkgConfig::tbb) + add_library(TBB::tbb ALIAS PkgConfig::tbb) + set(TBB_VERSION ${tbb_VERSION}) + set(TBB_FOUND ${tbb_FOUND}) + + # note: for python wheels we need to find and install tbbmalloc as well + _ov_get_tbb_location(PkgConfig::tbb tbb_loc) + string(REPLACE "tbb" "tbbmalloc" tbbmalloc_loc "${tbb_loc}") + if(EXISTS "${tbbmalloc_loc}") + add_library(TBB::tbbmalloc SHARED IMPORTED) + set_target_properties(TBB::tbbmalloc PROPERTIES IMPORTED_LOCATION ${tbbmalloc_loc}) + endif() + + message(STATUS "${PKG_CONFIG_EXECUTABLE}: tbb (${tbb_VERSION}) is found at ${tbb_PREFIX}") + else() + _ov_pkg_config_tbb_unset() + + if(CPACK_GENERATOR STREQUAL "^(DEB|RPM|CONDA-FORGE|BREW)$") + # package managers require system TBB + set(message_type FATAL_ERROR) else() - _ov_pkg_config_tbb_unset() - - if(CPACK_GENERATOR STREQUAL "^(DEB|RPM|CONDA-FORGE|BREW)$") - # package managers require system TBB - set(message_type FATAL_ERROR) - else() - set(message_type WARNING) - endif() - message(${message_type} "cmake v${CMAKE_VERSION} contains bug in function 'pkg_search_module', need to update to at least v3.16.0 version") + set(message_type WARNING) endif() + message(${message_type} "cmake v${CMAKE_VERSION} contains bug in function 'pkg_search_module', need to update to at least v3.16.0 version") endif() endif() endif() @@ -124,7 +125,7 @@ macro(ov_find_package_tbb) endif() # try to find one more time - find_package(TBB QUIET COMPONENTS tbb tbbmalloc + find_package(TBB ${_ov_minimal_tbb_version} QUIET COMPONENTS tbb tbbmalloc # TBB_DIR can be provided by ov_download_tbb HINTS ${TBB_DIR} ${_tbb_paths} @@ -143,20 +144,38 @@ macro(ov_find_package_tbb) endforeach() if(WIN32 AND TARGET TBB::tbbbind_2_5) - # Add HWLOC::hwloc_2_5 target to check via Apivalidator - get_target_property(TBB_location TBB::tbb IMPORTED_LOCATION_RELEASE) - get_filename_component(TBB_dir "${TBB_location}" DIRECTORY) - set(hwloc_dll_name "${CMAKE_SHARED_LIBRARY_PREFIX}hwloc${CMAKE_SHARED_LIBRARY_SUFFIX}") - find_file(HWLOC_DLL NAMES ${hwloc_dll_name} PATHS "${TBB_dir}" DOC "Path to hwloc.dll") - - if(NOT HWLOC_DLL) - message(FATAL_ERROR "Failed to find ${hwloc_dll_name} in ${TBB_dir}") + # some package managers provide hwloc.pc file within installation package + # let's try it first + if(PkgConfig_FOUND) + pkg_search_module(HWLOC QUIET + IMPORTED_TARGET GLOBAL + hwloc) endif() - add_library(HWLOC::hwloc_2_5 SHARED IMPORTED) - set_property(TARGET HWLOC::hwloc_2_5 APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(HWLOC::hwloc_2_5 PROPERTIES - IMPORTED_LOCATION_RELEASE "${HWLOC_DLL}") + if(TARGET PkgConfig::HWLOC) + # dependency is satisfied + else() + # Add HWLOC::hwloc_2_5 target to check via ApiValidator + get_target_property(imported_configs TBB::tbbbind_2_5 IMPORTED_CONFIGURATIONS) + foreach(imported_config RELEASE RELWITHDEBINFO DEBUG) + if(imported_config IN_LIST imported_configs) + get_target_property(TBBbind_location TBB::tbbbind_2_5 IMPORTED_LOCATION_${imported_config}) + get_filename_component(TBB_dir "${TBBbind_location}" DIRECTORY) + break() + endif() + endforeach() + + set(hwloc_dll_name "${CMAKE_SHARED_LIBRARY_PREFIX}hwloc${CMAKE_SHARED_LIBRARY_SUFFIX}") + find_file(HWLOC_DLL NAMES ${hwloc_dll_name} PATHS "${TBB_dir}" DOC "Path to hwloc.dll") + + if(NOT HWLOC_DLL) + message(FATAL_ERROR "Failed to find ${hwloc_dll_name} in ${TBB_dir}") + endif() + + add_library(HWLOC::hwloc_2_5 SHARED IMPORTED) + set_property(TARGET HWLOC::hwloc_2_5 APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + set_target_properties(HWLOC::hwloc_2_5 PROPERTIES IMPORTED_LOCATION_RELEASE "${HWLOC_DLL}") + endif() endif() endif() diff --git a/src/cmake/install_tbb.cmake b/src/cmake/install_tbb.cmake index 9e1bea9b426f62..c31c3847094a4a 100644 --- a/src/cmake/install_tbb.cmake +++ b/src/cmake/install_tbb.cmake @@ -192,7 +192,8 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND else() install(DIRECTORY "${TBBROOT}/lib" DESTINATION "${IE_TBB_DIR_INSTALL}" - COMPONENT tbb) + COMPONENT tbb + PATTERN "cmake" EXCLUDE) endif() install(FILES "${TBBROOT}/LICENSE" @@ -206,10 +207,19 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND DEPENDS tbb) list(APPEND core_dev_components tbb_dev) - install(FILES "${TBBROOT}/cmake/TBBConfig.cmake" - "${TBBROOT}/cmake/TBBConfigVersion.cmake" - DESTINATION "${IE_TBB_DIR_INSTALL}/cmake" - COMPONENT tbb_dev) + if(EXISTS "${TBBROOT}/lib/cmake") + # oneTBB case + install(DIRECTORY "${TBBROOT}/lib/cmake" + DESTINATION "${IE_TBB_DIR_INSTALL}/lib" + COMPONENT tbb_dev) + else() + # tbb2020 case + install(FILES "${TBBROOT}/cmake/TBBConfig.cmake" + "${TBBROOT}/cmake/TBBConfigVersion.cmake" + DESTINATION "${IE_TBB_DIR_INSTALL}/cmake" + COMPONENT tbb_dev) + endif() + install(DIRECTORY "${TBBROOT}/include" DESTINATION "${IE_TBB_DIR_INSTALL}" COMPONENT tbb_dev) @@ -218,7 +228,8 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND # .lib files are needed only for Windows install(DIRECTORY "${TBBROOT}/lib" DESTINATION "${IE_TBB_DIR_INSTALL}" - COMPONENT tbb_dev) + COMPONENT tbb_dev + PATTERN "cmake" EXCLUDE) endif() set(pkg_config_tbb_lib_dir "${IE_TBB_DIR_INSTALL}/lib") diff --git a/src/cmake/openvino.cmake b/src/cmake/openvino.cmake index 7870e2963e3c59..373cffcb9817e2 100644 --- a/src/cmake/openvino.cmake +++ b/src/cmake/openvino.cmake @@ -20,8 +20,10 @@ endif() add_library(${TARGET_NAME} $ + $ $ $ + $ $ $) @@ -131,7 +133,7 @@ ie_cpack_add_component(${OV_CPACK_COMP_CORE_DEV} HIDDEN DEPENDS ${OV_CPACK_COMP_CORE} ${core_dev_components}) -if(BUILD_SHARED_LIBS) +if(ENABLE_PLUGINS_XML) install(FILES $/plugins.xml DESTINATION ${OV_CPACK_PLUGINSDIR} COMPONENT ${OV_CPACK_COMP_CORE}) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index ce10d767bffa23..45f6863deceb79 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -6,7 +6,10 @@ add_subdirectory(itt) add_subdirectory(conditional_compilation) add_subdirectory(util) -add_subdirectory(snippets) +if(ENABLE_INTEL_CPU) + add_subdirectory(snippets) +endif() + add_subdirectory(transformations) add_subdirectory(offline_transformations) add_subdirectory(low_precision_transformations) diff --git a/src/common/low_precision_transformations/include/low_precision/markup_bias.hpp b/src/common/low_precision_transformations/include/low_precision/markup_bias.hpp new file mode 100644 index 00000000000000..ce4b262075b804 --- /dev/null +++ b/src/common/low_precision_transformations/include/low_precision/markup_bias.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +namespace ngraph { +namespace pass { +namespace low_precision { + +/** + * @ingroup ie_transformation_common_api + * @brief MarkupBias transformation marks biases after target layers. + * + * For more details about the transformation, refer to + * [MarkupBias](@ref openvino_docs_OV_UG_lpt_MarkupBias) page + * in the Inference Engine Developer Guide. + */ +class LP_TRANSFORMATIONS_API MarkupBias : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("MarkupBias", "0"); + MarkupBias(); +}; + +} // namespace low_precision +} // namespace pass +} // namespace ngraph \ No newline at end of file diff --git a/src/common/low_precision_transformations/include/low_precision/markup_precisions.hpp b/src/common/low_precision_transformations/include/low_precision/markup_precisions.hpp index 783dcafb13f678..97f3bc7569984e 100644 --- a/src/common/low_precision_transformations/include/low_precision/markup_precisions.hpp +++ b/src/common/low_precision_transformations/include/low_precision/markup_precisions.hpp @@ -39,12 +39,12 @@ class ngraph::pass::low_precision::MarkupPrecisions : public ngraph::pass::Funct class Restriction { public: explicit Restriction(const bool versionIsRequired) : versionIsRequired(versionIsRequired) {} - void add(const uint64_t version, const ngraph::pass::low_precision::PrecisionsRestriction::PrecisionsByPorts& precisions) { - precisionsByVersion.emplace(version, precisions); + void add(const std::string version_id, const ngraph::pass::low_precision::PrecisionsRestriction::PrecisionsByPorts& precisions) { + precisionsByVersion.emplace(version_id, precisions); } bool versionIsRequired; - std::unordered_map precisionsByVersion; + std::unordered_map precisionsByVersion; }; OPENVINO_RTTI("MarkupPrecisions", "0"); diff --git a/src/common/low_precision_transformations/include/low_precision/markup_quantization_granularity.hpp b/src/common/low_precision_transformations/include/low_precision/markup_quantization_granularity.hpp index 23558ec81a26d5..098da265cee0b4 100644 --- a/src/common/low_precision_transformations/include/low_precision/markup_quantization_granularity.hpp +++ b/src/common/low_precision_transformations/include/low_precision/markup_quantization_granularity.hpp @@ -37,12 +37,12 @@ class ngraph::pass::low_precision::MarkupQuantizationGranularity : public ngraph class PerTensorQuantization { public: explicit PerTensorQuantization(const bool versionIsRequired) : versionIsRequired(versionIsRequired) {} - void add(const uint64_t version, const std::vector& restrictions) { - portsByVersion.emplace(version, restrictions); + void add(const std::string version_id, const std::vector& restrictions) { + portsByVersion.emplace(version_id, restrictions); } bool versionIsRequired; - std::unordered_map> portsByVersion; + std::unordered_map> portsByVersion; }; OPENVINO_RTTI("MarkupPerTensorQuantization", "0"); diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/avg_pool_precision_preserved_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/avg_pool_precision_preserved_attribute.hpp index 4064edabf03006..c7c84e2122960c 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/avg_pool_precision_preserved_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/avg_pool_precision_preserved_attribute.hpp @@ -23,7 +23,7 @@ namespace ngraph { */ class LP_TRANSFORMATIONS_API AvgPoolPrecisionPreservedAttribute : public PrecisionPreservedAttribute { public: - OPENVINO_RTTI("LowPrecision::AvgPoolPrecisionPreserved", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::AvgPoolPrecisionPreserved", "", ov::RuntimeAttribute); using PrecisionPreservedAttribute::PrecisionPreservedAttribute; void merge_attributes(std::vector& attributes); bool is_skipped() const; diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/bias_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/bias_attribute.hpp new file mode 100644 index 00000000000000..fdefec850cac3b --- /dev/null +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/bias_attribute.hpp @@ -0,0 +1,21 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +namespace ov { +LP_TRANSFORMATIONS_API void mark_as_bias(const std::shared_ptr& node); + +LP_TRANSFORMATIONS_API bool marked_as_bias(const std::shared_ptr& node); + +class LP_TRANSFORMATIONS_API BiasAttribute : public ov::RuntimeAttribute { +public: + OPENVINO_RTTI("LowPrecision::Bias", "", ov::RuntimeAttribute); + bool is_copyable(const std::shared_ptr& to) const override; +}; +} // namespace ov diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/intervals_alignment_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/intervals_alignment_attribute.hpp index dcdf552856062f..99d6e814c2abe0 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/intervals_alignment_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/intervals_alignment_attribute.hpp @@ -58,7 +58,7 @@ class LP_TRANSFORMATIONS_API IntervalsAlignmentSharedValue { */ class LP_TRANSFORMATIONS_API IntervalsAlignmentAttribute : public SharedAttribute { public: - OPENVINO_RTTI("LowPrecision::IntervalsAlignment", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::IntervalsAlignment", "", ov::RuntimeAttribute); IntervalsAlignmentAttribute() = default; IntervalsAlignmentAttribute(IntervalsAlignmentSharedValue::Interval combinedInterval, size_t levels); IntervalsAlignmentAttribute( diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/precision_preserved_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/precision_preserved_attribute.hpp index 13baf73c3eb1b6..f5d14342a8f230 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/precision_preserved_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/precision_preserved_attribute.hpp @@ -22,7 +22,7 @@ namespace ngraph { */ class LP_TRANSFORMATIONS_API PrecisionPreservedAttribute : public SharedAttribute { public: - OPENVINO_RTTI("LowPrecision::PrecisionPreserved", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::PrecisionPreserved", "", ov::RuntimeAttribute); PrecisionPreservedAttribute() = default; PrecisionPreservedAttribute(const bool value); diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/precisions_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/precisions_attribute.hpp index 41f88a290b6a7c..0a69000b85e4e1 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/precisions_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/precisions_attribute.hpp @@ -26,7 +26,7 @@ namespace ngraph { */ class LP_TRANSFORMATIONS_API PrecisionsAttribute : public SharedAttribute> { public: - OPENVINO_RTTI("LowPrecision::Precisions", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::Precisions", "", ov::RuntimeAttribute); PrecisionsAttribute(const std::vector& precisions); static ov::Any create( diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_alignment_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_alignment_attribute.hpp index 0c766b71885ab5..30daa5f8073082 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_alignment_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_alignment_attribute.hpp @@ -27,7 +27,7 @@ namespace ngraph { */ class LP_TRANSFORMATIONS_API QuantizationAlignmentAttribute : public SharedAttribute { public: - OPENVINO_RTTI("LowPrecision::QuantizationAlignment", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::QuantizationAlignment", "", ov::RuntimeAttribute); QuantizationAlignmentAttribute(const bool value = false); static ov::Any create( diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_granularity_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_granularity_attribute.hpp index 84f2bf474c82a2..df466ef041cf4a 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_granularity_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_granularity_attribute.hpp @@ -22,7 +22,7 @@ namespace ngraph { */ class LP_TRANSFORMATIONS_API QuantizationGranularityAttribute : public ov::RuntimeAttribute { public: - OPENVINO_RTTI("LowPrecision::QuantizationGranularity", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::QuantizationGranularity", "", ov::RuntimeAttribute); enum class Granularity { PerChannel, diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_mode_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_mode_attribute.hpp index a3658061510ee0..3c3a454e4b9519 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_mode_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/quantization_mode_attribute.hpp @@ -12,7 +12,7 @@ namespace ngraph { class LP_TRANSFORMATIONS_API QuantizationModeAttribute : public ov::RuntimeAttribute { public: - OPENVINO_RTTI("LowPrecision::QuantizationModeAttribute", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::QuantizationModeAttribute", "", ov::RuntimeAttribute); enum class Mode { Asymmetric, diff --git a/src/common/low_precision_transformations/include/low_precision/rt_info/skip_cleanup_attribute.hpp b/src/common/low_precision_transformations/include/low_precision/rt_info/skip_cleanup_attribute.hpp index 1a11bbc2983ea2..1b323ee9424794 100644 --- a/src/common/low_precision_transformations/include/low_precision/rt_info/skip_cleanup_attribute.hpp +++ b/src/common/low_precision_transformations/include/low_precision/rt_info/skip_cleanup_attribute.hpp @@ -11,7 +11,7 @@ namespace ngraph { class LP_TRANSFORMATIONS_API SkipCleanupAttribute : public ov::RuntimeAttribute { public: - OPENVINO_RTTI("LowPrecision::SkipCleanup", "", ov::RuntimeAttribute, 0); + OPENVINO_RTTI("LowPrecision::SkipCleanup", "", ov::RuntimeAttribute); static ov::Any create(const std::shared_ptr& node); }; } // namespace ngraph diff --git a/src/common/low_precision_transformations/src/add.cpp b/src/common/low_precision_transformations/src/add.cpp index f20fad5061de4c..32dcac99b80151 100644 --- a/src/common/low_precision_transformations/src/add.cpp +++ b/src/common/low_precision_transformations/src/add.cpp @@ -15,6 +15,7 @@ #include "low_precision/common/ie_lpt_exception.hpp" #include "low_precision/network_helper.hpp" +#include "low_precision/rt_info/bias_attribute.hpp" #include "itt.hpp" namespace ngraph { @@ -29,7 +30,7 @@ std::shared_ptr replaceToSubtract(const std::shared_ptr& // - single responsibility // - keep AddTransformation and AddToSubtractTransformation transformations independent and optional const auto add = ov::as_type_ptr(op); - if (add == nullptr) { + if (add == nullptr || ov::marked_as_bias(add)) { return nullptr; } @@ -40,17 +41,8 @@ std::shared_ptr replaceToSubtract(const std::shared_ptr& if (constBranchIndex == -1) { return nullptr; } - const size_t dataBranchIndex = constBranchIndex == 0 ? 1ul : 0; - - const auto parent = add->get_input_node_shared_ptr(dataBranchIndex); - if (ov::is_type(parent) || - ov::is_type(parent) || - ov::is_type(parent) || - (ov::is_type(parent) && - (ov::is_type(parent->get_input_node_ptr(0)) || ov::is_type(parent->get_input_node_ptr(1))))) { - return nullptr; - } + const size_t dataBranchIndex = constBranchIndex == 0 ? 1ul : 0; auto constant = fold(add->input_value(constBranchIndex)); auto constOutput = constant->output(0); diff --git a/src/common/low_precision_transformations/src/concat.cpp b/src/common/low_precision_transformations/src/concat.cpp index 48e64a7772c7f7..e5fe59299ddb88 100644 --- a/src/common/low_precision_transformations/src/concat.cpp +++ b/src/common/low_precision_transformations/src/concat.cpp @@ -99,9 +99,11 @@ bool ConcatTransformation::transform(TransformationContext& context, ngraph::pat [](const FakeQuantizeDequantization& value) { return !value.isLowPrecision(); }); bool DqWithDifferentPrecision = someDqInLowPrecision && someDqInFpPrecision; + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = ngraph::normalize_axis(concat->get_friendly_name(), concat->get_axis(), concat->get_output_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END OutputVector dataNodes; NodeVector convertNodes; @@ -214,7 +216,9 @@ bool ConcatTransformation::canBeTransformed(const TransformationContext& context return false; } + OPENVINO_SUPPRESS_DEPRECATED_START const size_t normalizedAxis = ngraph::normalize_axis(concat->get_friendly_name(), axis, outRank); + OPENVINO_SUPPRESS_DEPRECATED_END if (outPShape[normalizedAxis].is_dynamic()) { return false; } diff --git a/src/common/low_precision_transformations/src/convolution.cpp b/src/common/low_precision_transformations/src/convolution.cpp index 46831f0586c5cf..4bd2dd31f24534 100644 --- a/src/common/low_precision_transformations/src/convolution.cpp +++ b/src/common/low_precision_transformations/src/convolution.cpp @@ -237,8 +237,15 @@ bool ConvolutionTransformation::transform(TransformationContext &context, ngraph Shape newScaleShape = newScalePShape.to_shape(); if (!newScaleShape.empty()) { - // that's all we need: [C, 1, 1, 1] => [C, 1, 1] - newScaleShape.pop_back(); + const auto input_shape = convolution->get_input_partial_shape(0); + const auto diff = newScaleShape.size() - input_shape.size(); + OPENVINO_ASSERT( + newScaleShape.empty() || ((0 <= diff) && (diff <= 2ull)), + "unexpected shape size on weights"); + + for (size_t i = 0; i <= diff; ++i) { + newScaleShape.pop_back(); + } } if (reshapeFromWeights != nullptr) { @@ -282,7 +289,12 @@ bool ConvolutionTransformation::transform(TransformationContext &context, ngraph const size_t weightsRankValue = weightsPShape.rank().get_length(); Shape zeroPointShape(weightsRankValue, 1ul); + // output channel or group zeroPointShape[0] = static_cast(weightsPShape[0].get_length()); + if ((reshapeFromWeights == nullptr) && (weightsRankValue == 5ull)) { + // output channel + zeroPointShape[1] = static_cast(weightsPShape[1].get_length()); + } auto zeroPointConstant = fold( subtractFromWeights->input_value(1), diff --git a/src/common/low_precision_transformations/src/eltwise_base_transformation.cpp b/src/common/low_precision_transformations/src/eltwise_base_transformation.cpp index 106eef1d7fc408..293d28dcebdda7 100644 --- a/src/common/low_precision_transformations/src/eltwise_base_transformation.cpp +++ b/src/common/low_precision_transformations/src/eltwise_base_transformation.cpp @@ -10,6 +10,7 @@ #include #include "low_precision/network_helper.hpp" +#include "low_precision/rt_info/bias_attribute.hpp" using namespace ngraph; using namespace ngraph::pass; @@ -70,8 +71,18 @@ static std::shared_ptr getDataParent(const std::shared_ptr branchDat parent = parent->get_input_node_shared_ptr(0); } - if (ov::is_type(parent) && isTargetType(parent->get_input_node_shared_ptr(0))) { - return parent->get_input_node_shared_ptr(0); + if (ov::marked_as_bias(parent)) { + const auto bias_parent = parent->get_input_node_shared_ptr(0); + // target node just before bias + if (isTargetType(bias_parent)) { + return bias_parent; + } + // between target node and bias are placed some DQ operations + const auto dq = NetworkHelper::getDequantization(parent->get_input_node_shared_ptr(0)); + const auto data_node = dq.data.get_node_shared_ptr(); + if (isTargetType(data_node)) { + return data_node; + } } return parent; } diff --git a/src/common/low_precision_transformations/src/fake_quantize.cpp b/src/common/low_precision_transformations/src/fake_quantize.cpp index 7ab60dca33908c..3dc308962b2197 100644 --- a/src/common/low_precision_transformations/src/fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/fake_quantize.cpp @@ -10,6 +10,7 @@ #include #include "low_precision/network_helper.hpp" +#include "low_precision/rt_info/bias_attribute.hpp" #include "itt.hpp" namespace ngraph { @@ -191,17 +192,8 @@ std::shared_ptr FakeQuantizeTransformation::fuseElementwis inputLowConst_f32 = fq::updateShape(fold(inputLowConst_f32, value), fakeQuantize->get_output_partial_shape(0)); inputHighConst_f32 = fq::updateShape(fold(inputHighConst_f32, value), fakeQuantize->get_output_partial_shape(0)); - } else if (ov::is_type(eltwise) && checkElementwise(eltwise)) { - if (ov::is_type(fq::getDataNode(eltwise)) || - ov::is_type(fq::getDataNode(eltwise)) || - ov::is_type(fq::getDataNode(eltwise)) || - ov::is_type(fq::getDataNode(eltwise)) || - ov::is_type(fq::getDataNode(eltwise))) { - return nullptr; - } - + } else if (ov::is_type(eltwise) && checkElementwise(eltwise) && !ov::marked_as_bias(eltwise)) { const auto value = foldConvert(constant, element::f32); - inputLowConst_f32 = fq::updateShape(fold(inputLowConst_f32, value), fakeQuantize->get_output_partial_shape(0)); inputHighConst_f32 = fq::updateShape(fold(inputHighConst_f32, value), fakeQuantize->get_output_partial_shape(0)); } else if (ov::is_type(eltwise)) { diff --git a/src/common/low_precision_transformations/src/gather.cpp b/src/common/low_precision_transformations/src/gather.cpp index daaad0b3b2710f..7079752192bc91 100644 --- a/src/common/low_precision_transformations/src/gather.cpp +++ b/src/common/low_precision_transformations/src/gather.cpp @@ -42,7 +42,9 @@ std::shared_ptr gatherDeqConstant( } const int64_t axis = ov::as_type_ptr(gather->get_input_node_shared_ptr(2))->cast_vector()[0]; + OPENVINO_SUPPRESS_DEPRECATED_START const size_t normalizedAxis = normalize_axis(gather->get_friendly_name(), axis, gather->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END // Dequantization channel matches with gather axis if (constantShape[normalizedAxis] != 1ul) { @@ -165,7 +167,10 @@ bool GatherTransformation::canBeTransformed(const TransformationContext& context } } const int64_t axis = axisConstant->cast_vector()[0]; + OPENVINO_SUPPRESS_DEPRECATED_START const size_t normalizedAxis = normalize_axis(operation->get_friendly_name(), axis, operation->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END + if (constantShape[normalizedAxis] != 1ul) { const auto indicesConstant = ov::as_type_ptr(operation->get_input_node_shared_ptr(1)); if (indicesConstant == nullptr) diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index 0a476b6f436b6b..1ed6ba9641271a 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -19,6 +19,7 @@ #include "low_precision/align_quantization_intervals.hpp" #include "low_precision/fake_quantize_decomposition.hpp" +#include "low_precision/markup_bias.hpp" #include "low_precision/markup_precisions.hpp" #include "low_precision/markup_can_be_quantized.hpp" #include "low_precision/markup_avg_pool_precision_preserved.hpp" @@ -201,6 +202,7 @@ bool ngraph::pass::low_precision::MarkupOptimizations::run_on_model(const std::s markup.register_pass(params.defaultPrecisions); markup.register_pass(params.defaultPrecisions); } + markup.register_pass(); markup.run_passes(f); return false; } diff --git a/src/common/low_precision_transformations/src/markup_bias.cpp b/src/common/low_precision_transformations/src/markup_bias.cpp new file mode 100644 index 00000000000000..50ff66c0366d31 --- /dev/null +++ b/src/common/low_precision_transformations/src/markup_bias.cpp @@ -0,0 +1,41 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "low_precision/markup_bias.hpp" + +#include +#include +#include + +#include "itt.hpp" +#include "low_precision/rt_info/bias_attribute.hpp" + +using namespace ngraph::pass::low_precision; + +MarkupBias::MarkupBias() { + MATCHER_SCOPE(MarkupBias); + auto layer_m = ov::pass::pattern::wrap_type(ov::pass::pattern::has_static_rank()); + auto bias_const_m = ov::pass::pattern::wrap_type(); + auto bias_m = ov::pass::pattern::wrap_type({layer_m, bias_const_m}); + + ov::matcher_pass_callback callback = [=](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + const auto& const_shape = pattern_map.at(bias_const_m).get_shape(); + + const bool per_channel = std::count_if(const_shape.begin(), const_shape.end(), [](size_t x) { return x > 1; }) == 1; + if (ov::shape_size(const_shape) == 1 || per_channel) { + const auto bias = pattern_map.at(bias_m).get_node_shared_ptr(); + ov::mark_as_bias(bias); + } + + return false; + }; + + auto m = std::make_shared(bias_m, matcher_name); + register_matcher(m, callback); +} diff --git a/src/common/low_precision_transformations/src/markup_precisions.cpp b/src/common/low_precision_transformations/src/markup_precisions.cpp index d5c168d4502131..a1748036012f13 100644 --- a/src/common/low_precision_transformations/src/markup_precisions.cpp +++ b/src/common/low_precision_transformations/src/markup_precisions.cpp @@ -30,10 +30,10 @@ ngraph::pass::low_precision::MarkupPrecisions::MarkupPrecisions( OPENVINO_SUPPRESS_DEPRECATED_START if (it == restrictionsByOperation.end()) { Restriction r(restriction.specifyVersion); - r.precisionsByVersion.emplace(restriction.operationType.version, restriction.precisionsByPorts); + r.precisionsByVersion.emplace(restriction.operationType.version_id, restriction.precisionsByPorts); restrictionsByOperation.emplace(restriction.operationType.name, r); } else { - it->second.add(restriction.operationType.version, restriction.precisionsByPorts); + it->second.add(restriction.operationType.version_id, restriction.precisionsByPorts); } OPENVINO_SUPPRESS_DEPRECATED_END } @@ -108,9 +108,7 @@ bool ngraph::pass::low_precision::MarkupPrecisions::run_on_model(const std::shar if (it != restrictionsByOperation.end()) { const Restriction& r = it->second; if (r.versionIsRequired) { - OPENVINO_SUPPRESS_DEPRECATED_START - const auto it2 = r.precisionsByVersion.find(typeInfo.version); - OPENVINO_SUPPRESS_DEPRECATED_END + const auto it2 = r.precisionsByVersion.find(typeInfo.version_id); if (it2 == r.precisionsByVersion.end()) { continue; } diff --git a/src/common/low_precision_transformations/src/markup_quantization_granularity.cpp b/src/common/low_precision_transformations/src/markup_quantization_granularity.cpp index 7b86c1d2203c50..bbe448d83423ec 100644 --- a/src/common/low_precision_transformations/src/markup_quantization_granularity.cpp +++ b/src/common/low_precision_transformations/src/markup_quantization_granularity.cpp @@ -20,10 +20,10 @@ ngraph::pass::low_precision::MarkupQuantizationGranularity::MarkupQuantizationGr OPENVINO_SUPPRESS_DEPRECATED_START if (it == restrictionsByOperation.end()) { PerTensorQuantization r(restriction.specifyVersion); - r.portsByVersion.emplace(restriction.operationType.version, restriction.restrictions); + r.portsByVersion.emplace(restriction.operationType.version_id, restriction.restrictions); restrictionsByOperation.emplace(restriction.operationType.name, r); } else { - it->second.add(restriction.operationType.version, restriction.restrictions); + it->second.add(restriction.operationType.version_id, restriction.restrictions); } OPENVINO_SUPPRESS_DEPRECATED_END } @@ -74,9 +74,7 @@ bool ngraph::pass::low_precision::MarkupQuantizationGranularity::run_on_model(co } if (restriction.versionIsRequired) { - OPENVINO_SUPPRESS_DEPRECATED_START - const auto it2 = restriction.portsByVersion.find(node->get_type_info().version); - OPENVINO_SUPPRESS_DEPRECATED_END + const auto it2 = restriction.portsByVersion.find(node->get_type_info().version_id); if (it2 == restriction.portsByVersion.end()) { continue; } diff --git a/src/common/low_precision_transformations/src/network_helper.cpp b/src/common/low_precision_transformations/src/network_helper.cpp index c7db834e0b2848..b5ea92e61ff66a 100644 --- a/src/common/low_precision_transformations/src/network_helper.cpp +++ b/src/common/low_precision_transformations/src/network_helper.cpp @@ -1286,7 +1286,14 @@ FakeQuantizeDequantization NetworkHelper::getDequantization(const std::shared_pt return 1ul; }; - Output dataNode = inPlace ? std::const_pointer_cast(node)->output(0) : node->input_value(parentIndex); + Output dataNode; + if (inPlace) { + dataNode = std::const_pointer_cast(node); + } else { + if (parentIndex >= node->get_input_size()) + return FakeQuantizeDequantization(); + dataNode = node->input_value(parentIndex); + } const std::shared_ptr multiply = ov::as_type_ptr(dataNode.get_node_shared_ptr()); std::shared_ptr multiplyConstant; diff --git a/src/common/low_precision_transformations/src/reduce_base_transformation.cpp b/src/common/low_precision_transformations/src/reduce_base_transformation.cpp index 87604bb193e316..27a8645884a2ba 100644 --- a/src/common/low_precision_transformations/src/reduce_base_transformation.cpp +++ b/src/common/low_precision_transformations/src/reduce_base_transformation.cpp @@ -48,7 +48,9 @@ bool ReduceBaseTransformation::canBeTransformed(const TransformationContext& con return false; } + OPENVINO_SUPPRESS_DEPRECATED_START const std::vector axes = ngraph::normalize_axes(reduce->get_friendly_name(), constData, inputRank); + OPENVINO_SUPPRESS_DEPRECATED_END const auto deqByReducedConst = [&](const std::shared_ptr& eltwise) { const auto constShape = eltwise->get_shape(); diff --git a/src/common/low_precision_transformations/src/rt_info/bias_attribute.cpp b/src/common/low_precision_transformations/src/rt_info/bias_attribute.cpp new file mode 100644 index 00000000000000..40d75e06f87939 --- /dev/null +++ b/src/common/low_precision_transformations/src/rt_info/bias_attribute.cpp @@ -0,0 +1,27 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "low_precision/rt_info/bias_attribute.hpp" +#include "low_precision/network_helper.hpp" + +#include +#include +#include +#include +#include +#include + +void ov::mark_as_bias(const std::shared_ptr& node) { + auto& rt = node->get_rt_info(); + rt[ov::BiasAttribute::get_type_info_static()] = ov::BiasAttribute(); +} + +bool ov::marked_as_bias(const std::shared_ptr& node) { + const auto& rt_info = node->get_rt_info(); + return rt_info.find(ov::BiasAttribute::get_type_info_static()) != rt_info.end(); +} + +bool ov::BiasAttribute::is_copyable(const std::shared_ptr& to) const { + return ov::is_type(to) && ngraph::pass::low_precision::NetworkHelper::getConstantInput(to) != nullptr; +} diff --git a/src/common/low_precision_transformations/src/shuffle_channels.cpp b/src/common/low_precision_transformations/src/shuffle_channels.cpp index 6979ac309bffd0..37c36284c4b73e 100644 --- a/src/common/low_precision_transformations/src/shuffle_channels.cpp +++ b/src/common/low_precision_transformations/src/shuffle_channels.cpp @@ -48,10 +48,12 @@ bool ShuffleChannelsTransformation::transform(TransformationContext& context, ng if (shape_size(constShape) == 1ul) { return NetworkHelper::toScalar(normalizedConst); } else { + OPENVINO_SUPPRESS_DEPRECATED_START const size_t normalizedAxis = ngraph::normalize_axis( shuffleChannels->get_friendly_name(), shuffleChannels->get_axis(), shuffleChannels->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END if (constShape[normalizedAxis] == 1ul) { return normalizedConst; diff --git a/src/common/low_precision_transformations/src/split.cpp b/src/common/low_precision_transformations/src/split.cpp index aec4408d38d532..d49d79861cb08b 100644 --- a/src/common/low_precision_transformations/src/split.cpp +++ b/src/common/low_precision_transformations/src/split.cpp @@ -46,7 +46,9 @@ bool SplitTransformation::transform(TransformationContext& context, ngraph::patt ngraph::copy_runtime_info(split, newSplit); const int64_t axis = ov::as_type_ptr(split->get_input_node_shared_ptr(1))->cast_vector()[0]; + OPENVINO_SUPPRESS_DEPRECATED_START const size_t normalizedAxis = normalize_axis(split->get_friendly_name(), axis, split->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END const size_t outputSize = newSplit->get_output_size(); const auto splitConstant = [&](const std::shared_ptr operation) { diff --git a/src/common/low_precision_transformations/src/weightable_layer_transformation.cpp b/src/common/low_precision_transformations/src/weightable_layer_transformation.cpp index 1cfe4bb51d3ac7..1837f21635235c 100644 --- a/src/common/low_precision_transformations/src/weightable_layer_transformation.cpp +++ b/src/common/low_precision_transformations/src/weightable_layer_transformation.cpp @@ -230,16 +230,16 @@ bool WeightableLayerTransformation::isQuantizedStatic(const std::shared_ptrget_input_node_shared_ptr(1); - if (!ov::is_type(reshape)) { - return false; - } + std::shared_ptr parent = ov::is_type(reshape) ? + reshape->get_input_node_shared_ptr(0) : + reshape; - if (ov::is_type(reshape->get_input_node_shared_ptr(0))) { - const std::shared_ptr fq = ov::as_type_ptr(reshape->get_input_node_shared_ptr(0)); + const auto fq = ov::as_type_ptr(parent); + if (fq != nullptr) { return NetworkHelper::isQuantizeSupported(fq); } - dequantizationOnWeights = NetworkHelper::getDequantization(reshape, defaultPrecisions, 0); + dequantizationOnWeights = NetworkHelper::getDequantization(parent, defaultPrecisions, 0, true); } else if (ov::is_type(layer->get_input_node_shared_ptr(1))) { const std::shared_ptr fq = ov::as_type_ptr(layer->get_input_node_shared_ptr(1)); return NetworkHelper::isQuantizeSupported(fq); diff --git a/src/common/low_precision_transformations/tests/group_convolution_transformation.cpp b/src/common/low_precision_transformations/tests/group_convolution_transformation.cpp index b20d211d651adc..d148c370dfd2f8 100644 --- a/src/common/low_precision_transformations/tests/group_convolution_transformation.cpp +++ b/src/common/low_precision_transformations/tests/group_convolution_transformation.cpp @@ -48,6 +48,7 @@ class GroupConvolutionTestValues { TestTransformationParams params; size_t group; int groupCalculationDimention; + bool addReshape; Actual actual; Expected expected; }; @@ -76,7 +77,8 @@ class GroupConvolutionTransformation : public LayerTransformation, testValues.actual.dequantizationOnWeights, ngraph::element::f32, {}, - ngraph::element::f32); + ngraph::element::f32, + testValues.addReshape); SimpleLowPrecisionTransformer transform; transform.add( @@ -101,7 +103,8 @@ class GroupConvolutionTransformation : public LayerTransformation, testValues.expected.dequantizationOnWeights, testValues.expected.precisionAfterOperation, testValues.expected.dequantizationAfter, - testValues.expected.precisionAfterDequantization); + testValues.expected.precisionAfterDequantization, + testValues.addReshape); } static std::string getTestCaseName(testing::TestParamInfo obj) { @@ -113,7 +116,9 @@ class GroupConvolutionTransformation : public LayerTransformation, result << toString(testValues.params) << "_" << inputShape << "_" << outputShape << "_" << testValues.group << "_" << testValues.groupCalculationDimention << "_" << testValues.actual.precisionBeforeDequantization << "_" << testValues.actual.dequantization << "_" - << "_weights_" << testValues.actual.weights->get_element_type() << "_" + << "_add_reshape:" << testValues.addReshape << "_" + << "_weights_type:" << testValues.actual.weights->get_element_type() << "_" + << "_weights_shape:" << testValues.actual.weights->get_shape() << "_" << "{ " << testValues.actual.weights->cast_vector()[0] << " }_" << testValues.actual.fakeQuantizeOnWeights << "_"; return result.str(); @@ -128,333 +133,520 @@ TEST_P(GroupConvolutionTransformation, CompareFunctions) { ASSERT_TRUE(LayerTransformation::allNamesAreUnique(actualFunction)) << "Not all names are unique"; } +// clang-format off namespace testValues1 { + const std::vector> shapesForGroupConv = { {{1, 6, 224, 224}, {1, 24, 218, 218}}, - {{-1, -1, -1, -1}, {-1, -1, -1, -1}}}; + {{-1, -1, -1, -1}, {-1, -1, -1, -1}} +}; const std::vector testValuesGroupConv = { // group convolution, tensor quantization, with zero point - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::u8, - {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) - }}, + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) + } + }, // group convolution, tensor quantization, with zero point - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), - 3ul, - 0, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::u8, - {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) - }}, + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), + 3ul, + 0, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) + } + }, // group convolution, tensor quantization, with zero point - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), - 3ul, - 1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::u8, - {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) - }}, + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), + 3ul, + 1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) + } + }, // group convolution, tensor quantization, with zero point - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(false), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}, - ngraph::element::f32, - {}}}, + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(false), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {}, + ngraph::element::f32, + {} + } + }, + // group convolution, tensor quantization, with zero point - {LayerTransformation::createParamsU8I8().setUpdatePrecisions(false), - 3ul, - -1, - // ActualValues - {ngraph::element::f32, - {{}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::f32, - {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) - }}, + { + LayerTransformation::createParamsU8I8().setUpdatePrecisions(false), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::f32, + {{}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::f32, + {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} // 0.0002 = 0.02 (on data) * 0.01 (on weights) + } + }, + // group convolution, per-channel quantization with different values, without zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {}, {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, - {}, - {{// 0.0002 = 0.02 (on data) * 0.01 (on weights) - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - // 0.0004 = 0.04 (on data) * 0.01 (on weights) - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - // 0.0008 = 0.08 (on data) * 0.01 (on weights) - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f}, - ngraph::element::f32, - {1, 24, 1, 1}}}, - }}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {}, {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + { + {}, + {}, + { + { + // 0.0002 = 0.02 (on data) * 0.01 (on weights) + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + // 0.0004 = 0.04 (on data) * 0.01 (on weights) + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + // 0.0008 = 0.08 (on data) * 0.01 (on weights) + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f + }, + ngraph::element::f32, + {1, 24, 1, 1} + } + }, + } + }, + // group convolution, per-channel quantization with the same values, without zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - { - ngraph::element::u8, - {{ngraph::element::f32}, {}, {{0.02f}, ngraph::element::f32, {1, 6, 1, 1}}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}, - }, - // ExpectedValues - { - ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}, - }}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {}, {{0.02f}, ngraph::element::f32, {1, 6, 1, 1}}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {}, + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}, + } + }, + // group convolution, without zero point, without convert - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::f32, - {{}, {}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::f32, - {{}, {}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-1.25f}), - {}, - {}, - ngraph::element::f32, - {}}}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::f32, + {{}, {}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::f32, + {{}, {}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-1.25f}), + {}, + {}, + ngraph::element::f32, + {} + } + }, + // group convolution, without zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{element::f32}, {}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}}}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{element::f32}, {}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} + } + }, + // per-channel quantization with different values, without zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {}, {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), - {}, - {ngraph::element::f32, {}, {0.01f}}}, - // ExpectedValues - { - ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), - {}, - {}, - ngraph::element::f32, - {{}, - {}, - {{// 0.0002 = 0.02 (on data) * 0.01 (on weights) - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - // 0.0004 = 0.04 (on data) * 0.01 (on weights) - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - // 0.0008 = 0.08 (on data) * 0.01 (on weights) - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f}, - ngraph::element::f32, - {1, 24, 1, 1}}}, - }}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {}, {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), + {}, + {ngraph::element::f32, {}, {0.01f}} + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), + {}, + {}, + ngraph::element::f32, + { + {}, + {}, + { + { + // 0.0002 = 0.02 (on data) * 0.01 (on weights) + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + // 0.0004 = 0.04 (on data) * 0.01 (on weights) + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + // 0.0008 = 0.08 (on data) * 0.01 (on weights) + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f + }, + ngraph::element::f32, + {1, 24, 1, 1} + } + }, + } + }, // per-channel quantization with different values, without zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, - {{255}, ngraph::element::f32, {}, true, 1, ngraph::element::u8, true}, - {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), - {}, - {ngraph::element::f32, {{127}, ngraph::element::f32, {}, true, 1, ngraph::element::i8, true}, {0.01f}}}, - // ExpectedValues - { - ngraph::element::u8, - {{}, {std::vector(6ul, 255.f), ngraph::element::f32, {1, 6, 1, 1}, false, 1, ngraph::element::u8}, {}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), - {}, - {{}, - {std::vector(24ul, 127.f), - ngraph::element::f32, - {24, 1, 1, 1}, - false, - 1, - ngraph::element::i8, - false, - {{ov::pass::DisableConstantFolding::get_type_info_static(), ov::pass::DisableConstantFolding()}}}, - {}}, - ngraph::element::f32, - {{}, - {}, - {{// 0.0002 = 0.02 (on data) * 0.01 (on weights) - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - 0.0002f, - // 0.0004 = 0.04 (on data) * 0.01 (on weights) - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - 0.0004f, - // 0.0008 = 0.08 (on data) * 0.01 (on weights) - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f, - 0.0008f}, - ngraph::element::f32, - {1, 24, 1, 1}}}, - }}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + { + {ngraph::element::f32}, + {{255}, ngraph::element::f32, {}, true, 1, ngraph::element::u8, true}, + {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}} + }, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), + {}, + { + ngraph::element::f32, + {{127}, ngraph::element::f32, {}, true, 1, ngraph::element::i8, true}, + {0.01f} + } + }, + // ExpectedValues + { + ngraph::element::u8, + { + {}, + {std::vector(6ul, 255.f), ngraph::element::f32, {1, 6, 1, 1}, false, 1, ngraph::element::u8}, + {} + }, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{2.f}), + {}, + { + {}, + { + std::vector(24ul, 127.f), + ngraph::element::f32, + {24, 1, 1, 1}, + false, + 1, + ngraph::element::i8, + false, + {{ov::pass::DisableConstantFolding::get_type_info_static(), ov::pass::DisableConstantFolding()}} + }, + {} + }, + ngraph::element::f32, + { + {}, + {}, + { + { + // 0.0002 = 0.02 (on data) * 0.01 (on weights) + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + // 0.0004 = 0.04 (on data) * 0.01 (on weights) + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + // 0.0008 = 0.08 (on data) * 0.01 (on weights) + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f + }, + ngraph::element::f32, + {1, 24, 1, 1} + } + }, + } + }, + + // per-channel quantization with different values, without zero point, no reshape - 5D weights + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + false, + // ActualValues + { + ngraph::element::u8, + { + {ngraph::element::f32}, + {{255}, ngraph::element::f32, {}, true, 1, ngraph::element::u8, true}, + {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}} + }, + op::Constant::create(ngraph::element::i8, ngraph::Shape{3,8,2,7,7}, std::vector{2.f}), + {}, + { + ngraph::element::f32, + {{127}, ngraph::element::f32, {}, true, 1, ngraph::element::i8, true}, + {0.01f} + } + }, + // ExpectedValues + { + ngraph::element::u8, + { + {}, + {std::vector(6ul, 255.f), ngraph::element::f32, {1, 6, 1, 1}, false, 1, ngraph::element::u8}, + {} + }, + op::Constant::create(ngraph::element::i8, ngraph::Shape{3,8,2,7,7}, std::vector{2.f}), + {}, + { + {}, + { + std::vector(24ul, 127.f), + ngraph::element::f32, + {3, 8, 1, 1, 1}, + false, + 1, + ngraph::element::i8, + false, + {{ov::pass::DisableConstantFolding::get_type_info_static(), ov::pass::DisableConstantFolding()}} + }, + {} + }, + ngraph::element::f32, + { + {}, + {}, + { + { + // 0.0002 = 0.02 (on data) * 0.01 (on weights) + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + 0.0002f, + // 0.0004 = 0.04 (on data) * 0.01 (on weights) + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + 0.0004f, + // 0.0008 = 0.08 (on data) * 0.01 (on weights) + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f, + 0.0008f + }, + ngraph::element::f32, + {1, 24, 1, 1} + } + }, + } + }, }; INSTANTIATE_TEST_SUITE_P(smoke_LPT, @@ -472,146 +664,201 @@ const std::vector> shapesF const std::vector testValuesForDepthWiseConv = { // depth-wise convolution, per-tensor quantization, with zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::u8, - {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}}}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} + } + }, + // depth-wise convolution, tensor quantization, with zero point - {LayerTransformation::createParamsU8I8().setUpdatePrecisions(false), - 3ul, - -1, - // ActualValues - {ngraph::element::f32, - {{}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::f32, - {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}}}, + { + LayerTransformation::createParamsU8I8().setUpdatePrecisions(false), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::f32, + {{}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::f32, + {{}, {{128.f}, ngraph::element::f32, {1, 6, 1, 1}, false}, {}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} + } + }, + // depth-wise convolution, per-channel quantization with different values, without zero point - {LayerTransformation::createParamsU8I8(), - 6ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {}, {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, - {}, - {{ - 0.0002f, - 0.0002f, // 0.0002 = 0.02 (on data) * 0.01 (on weights) - 0.0004f, - 0.0004f, // 0.0004 = 0.04 (on data) * 0.01 (on weights) - 0.0008f, - 0.0008f // 0.0008 = 0.08 (on data) * 0.01 (on weights) - }, - ngraph::element::f32, - {1, 6, 1, 1}}}, - }}, + { + LayerTransformation::createParamsU8I8(), + 6ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {}, {{0.02f, 0.02f, 0.04f, 0.04f, 0.08f, 0.08f}, ngraph::element::f32, {1, 6, 1, 1}}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + { + {}, + {}, + { + { + 0.0002f, + 0.0002f, // 0.0002 = 0.02 (on data) * 0.01 (on weights) + 0.0004f, + 0.0004f, // 0.0004 = 0.04 (on data) * 0.01 (on weights) + 0.0008f, + 0.0008f // 0.0008 = 0.08 (on data) * 0.01 (on weights) + }, + ngraph::element::f32, + {1, 6, 1, 1} + } + }, + } + }, + // depth-wise convolution, per-tensor quantization with the same values, without zero point - {LayerTransformation::createParamsU8I8(), - 6ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {}, {{0.02f}, ngraph::element::f32, {1, 6, 1, 1}}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - { - ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}, - }}, + { + LayerTransformation::createParamsU8I8(), + 6ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {}, {{0.02f}, ngraph::element::f32, {1, 6, 1, 1}}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}, + } + }, + // depth-wise convolution, without zero point, without convert - {LayerTransformation::createParamsU8I8(), - 6ul, - -1, - // ActualValues - {ngraph::element::f32, - {{}, {}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::f32, - {{}, {}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-1.25f}), - {}, - {}, - ngraph::element::f32, - {}}}, + { + LayerTransformation::createParamsU8I8(), + 6ul, + -1, + true, + // ActualValues + { + ngraph::element::f32, + {{}, {}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::f32, + {{}, {}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{-1.25f}), + {}, + {}, + ngraph::element::f32, + {} + } + }, + // depth-wise convolution, without zero point - {LayerTransformation::createParamsU8I8(), - 6ul, - -1, - // ActualValues - {ngraph::element::u8, - {{element::f32}, {}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::u8, - {}, - op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), - {}, - {}, - ngraph::element::f32, - {{}, {}, {{0.0002f}, ngraph::element::f32, {}}}}}, + { + LayerTransformation::createParamsU8I8(), + 6ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{element::f32}, {}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {}, + op::Constant::create(ngraph::element::i8, ngraph::Shape{}, std::vector{-125.f}), + {}, + {}, + ngraph::element::f32, + {{}, {}, {{0.0002f}, ngraph::element::f32, {}}} + } + }, + // without dequantization operations - {LayerTransformation::createParamsU8I8(), - 6ul, - -1, - // ActualValues - {ngraph::element::f32, - {}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::f32, - {}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}, - ngraph::element::f32, - {}}}, + { + LayerTransformation::createParamsU8I8(), + 6ul, + -1, + true, + // ActualValues + { + ngraph::element::f32, + {}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::f32, + {}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {}, + ngraph::element::f32, + {} + } + }, }; INSTANTIATE_TEST_SUITE_P(smoke_LPT, @@ -623,27 +870,35 @@ INSTANTIATE_TEST_SUITE_P(smoke_LPT, namespace testValues3 { const std::vector> shapesWithDynamicChannel = { - {PartialShape::dynamic(), PartialShape::dynamic()}}; + {PartialShape::dynamic(), PartialShape::dynamic()} +}; const std::vector testValuesWithDynamicChannel = { // depth-wise convolution, per-tensor quantization, with zero point - {LayerTransformation::createParamsU8I8(), - 3ul, - -1, - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}}, - // ExpectedValues - {ngraph::element::u8, - {{ngraph::element::f32}, {128.f}, {0.02f}}, - op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), - {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, - {}, - ngraph::element::f32, - {}}}, + { + LayerTransformation::createParamsU8I8(), + 3ul, + -1, + true, + // ActualValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + {{ngraph::element::f32}, {128.f}, {0.02f}}, + op::Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{2.f}), + {255ul, Shape({1, 1, 1, 1}), {0.f}, {254.f}, {-1.27f}, {1.27f}}, + {}, + ngraph::element::f32, + {} + } + }, }; INSTANTIATE_TEST_SUITE_P(smoke_LPT, @@ -652,3 +907,4 @@ INSTANTIATE_TEST_SUITE_P(smoke_LPT, ::testing::ValuesIn(testValuesWithDynamicChannel)), GroupConvolutionTransformation::getTestCaseName); } // namespace testValues3 +// clang-format on diff --git a/src/common/low_precision_transformations/tests/markup_bias_transformation.cpp b/src/common/low_precision_transformations/tests/markup_bias_transformation.cpp new file mode 100644 index 00000000000000..3fa1966596999a --- /dev/null +++ b/src/common/low_precision_transformations/tests/markup_bias_transformation.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" +#include "layer_transformation.hpp" +#include "lpt_ngraph_functions/markup_bias_function.hpp" +#include "simple_low_precision_transformer.hpp" + +using namespace testing; + +class MarkupBiasTestParams { +public: + ov::PartialShape input_shape; + ov::PartialShape bias_shape; + bool is_bias; +}; + +using MarkupBiasTestValues = std::tuple; + +class MarkupBiasTests : public testing::WithParamInterface, public LayerTransformation { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj) { + ov::element::Type precision; + MarkupBiasTestParams test_values; + std::string layer_type; + std::tie(precision, test_values, layer_type) = obj.param; + + std::ostringstream result; + result << precision << "IS=" << test_values.input_shape << "_bias_shape=" << test_values.bias_shape << "_" + << layer_type << "_is_bias=" << test_values.is_bias; + return result.str(); + } + +protected: + void SetUp() override { + ov::element::Type precision; + MarkupBiasTestParams test_values; + std::string layer_type; + std::tie(precision, test_values, layer_type) = GetParam(); + + actualFunction = ngraph::builder::subgraph::MarkupBiasFunction::get(precision, + test_values.input_shape, + test_values.bias_shape, + layer_type); + SimpleLowPrecisionTransformer transformer; + transformer.transform(actualFunction); + } +}; + +TEST_P(MarkupBiasTests, CompareFunctions) { + actualFunction->validate_nodes_and_infer_types(); + + const auto addOps = LayerTransformation::get(actualFunction); + EXPECT_EQ(1ul, addOps.size()) << "unexpected addOps size"; + + const bool is_bias = std::get<1>(GetParam()).is_bias; + auto biasAttr = ngraph::pass::low_precision::getAttribute(addOps[0]); + EXPECT_EQ(!biasAttr.empty(), is_bias) << "Bias markup failed"; +} + +namespace MarkupBiasTestsInstantiation { +std::vector precisions = { + ov::element::f32, +}; + +std::vector test_params_4d = { + {{1, 10, 16, 16}, {1, 10, 1, 1}, true}, + {{1, 10, 16, 16}, {1, 1, 1, 1}, true}, + {{1, 10, 16, 16}, {1, 10, 16, 16}, false}, + {{1, 10, 16, 16}, ov::PartialShape::dynamic(), false}, +}; + +std::vector layer_types_4d = { + "Convolution", + "GroupConvolution", + "ConvolutionBackpropData", +}; + +INSTANTIATE_TEST_SUITE_P(smoke_LPT_4D_Positive, + MarkupBiasTests, + ::testing::Combine(::testing::ValuesIn(precisions), + ::testing::ValuesIn(test_params_4d), + ::testing::ValuesIn(layer_types_4d)), + MarkupBiasTests::getTestCaseName); + +std::vector test_params_2d = { + {{1, 10}, {1, 10}, true}, + {{1, 10}, {1, 1}, true}, + {{1, 10}, ov::PartialShape::dynamic(), false}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_LPT_2D_Positive, + MarkupBiasTests, + ::testing::Combine(::testing::ValuesIn(precisions), + ::testing::ValuesIn(test_params_2d), + ::testing::Values("MatMulWithConstant")), + MarkupBiasTests::getTestCaseName); + +} // namespace MarkupBiasTestsInstantiation diff --git a/src/common/low_precision_transformations/tests/pull_reshape_through_dequantization_transformation.cpp b/src/common/low_precision_transformations/tests/pull_reshape_through_dequantization_transformation.cpp index 14e8f4361ceb22..b15a8f3b784c92 100644 --- a/src/common/low_precision_transformations/tests/pull_reshape_through_dequantization_transformation.cpp +++ b/src/common/low_precision_transformations/tests/pull_reshape_through_dequantization_transformation.cpp @@ -133,11 +133,17 @@ TEST_P(PullReshapeThroughDequantizationTransformation, CompareFunctions) { ASSERT_TRUE(res.first) << res.second; } -const std::vector inputShapes = {ngraph::Shape({1, 960, 7, 7}), ngraph::Shape({4, 960, 7, 7})}; +// clang-format off + +const std::vector inputShapes = { + ngraph::Shape({1, 960, 7, 7}), + ngraph::Shape({4, 960, 7, 7}) +}; const std::vector> dequantizationOnWeightElementwiseConstantShapes = { {ngraph::Shape({1, 960}), ngraph::Shape({960, 1, 1, 1})}, - {ngraph::Shape({9, 960}), ngraph::Shape({960, 1, 3, 3})}}; + {ngraph::Shape({9, 960}), ngraph::Shape({960, 1, 3, 3})} +}; const std::vector multiplyShapes = {ngraph::Shape({1, 1, 960, 1})}; @@ -193,37 +199,51 @@ const std::vector testValues = { // \ / // Multiply // - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, - {{0.02f}, element::f32, {}, false}}, - {std::vector{2.f}, ngraph::element::i8, {9, 960}}, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {/* from parameter */}, false}, - {{0.03f}, element::f32, {/* from parameter */}, false}}, - {{3, 3, 960, 1}}, - {{2}, element::f32, {/* from parameter: multiplyShapes */}, false}, - {{2, 3, 0, 1}}, - {{960, 1, 1, 3, 3}}, - ngraph::element::f32, - {}}, - // ExpectedValues - {ngraph::element::u8, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, - {{0.02f}, element::f32, {}, false}}, - {std::vector{2.f}, ngraph::element::i8, {960, 1, 3, 3}}, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {/* from parameter */}, false}, - {{0.06f}, element::f32, {/* from parameter */}, false}}, - {}, - {}, - {}, - {{960, 1, 1, 3, 3}}, - ngraph::element::f32, - {}}}, + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), + // ActualValues + { + ngraph::element::u8, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {}, false, 1ul, element::u8, true }, + { {0.02f}, element::f32, {}, false } + }, + { std::vector{ 2.f }, ngraph::element::i8, {9, 960}}, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {/* from parameter */}, false }, + { {0.03f}, element::f32, {/* from parameter */}, false } + }, + { {3, 3, 960, 1} }, + { {2}, element::f32, {/* from parameter: multiplyShapes */}, false }, + { {2, 3, 0, 1} }, + { {960, 1, 1, 3, 3} }, + ngraph::element::f32, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {}, false, 1ul, element::u8, true }, + { {0.02f}, element::f32, {}, false } + }, + { std::vector{ 2.f }, ngraph::element::i8, {960, 1, 3, 3}}, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {/* from parameter */}, false }, + { {0.06f}, element::f32, {/* from parameter */}, false } + }, + {}, + {}, + {}, + {{960, 1, 1, 3, 3}}, + ngraph::element::f32, + {} + } + }, // Subtract with Convert + Constant // Actual: @@ -276,37 +296,54 @@ const std::vector testValues = { // \ / // Multiply // - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, - {{0.02f}, element::f32, {}, false}}, - {std::vector{2.f}, ngraph::element::i8, {9, 960}}, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {/* from parameter */}, false, 1ul, element::i8, true}, - {{0.03f}, element::f32, {/* from parameter */}, false}}, - {{3, 3, 960, 1}}, - {{2}, element::f32, {/* from parameter: multiplyShapes */}, false}, - {{2, 3, 0, 1}}, - {{960, 1, 1, 3, 3}}, - ngraph::element::f32, - {}}, - // ExpectedValues - {ngraph::element::u8, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, - {{0.02f}, element::f32, {}, false}}, - {std::vector{2.f}, ngraph::element::i8, {960, 1, 3, 3}}, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {/* from parameter */}, false, 1ul, element::i8, true}, - {{0.06f}, element::f32, {/* from parameter */}, false}}, - {}, - {}, - {}, - {{960, 1, 1, 3, 3}}, - ngraph::element::f32, - {}}}}; + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), + // ActualValues + { + ngraph::element::u8, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {}, false, 1ul, element::u8, true }, + { {0.02f}, element::f32, {}, false } + }, + { std::vector{ 2.f }, ngraph::element::i8, {9, 960}}, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {/* from parameter */}, false, 1ul, element::i8, true }, + { {0.03f}, element::f32, {/* from parameter */}, false } + }, + { {3, 3, 960, 1} }, + { {2}, element::f32, {/* from parameter: multiplyShapes */}, false }, + { {2, 3, 0, 1} }, + { {960, 1, 1, 3, 3} }, + ngraph::element::f32, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {}, false, 1ul, element::u8, true }, + { {0.02f}, element::f32, {}, false } + }, + { std::vector{ 2.f }, ngraph::element::i8, {960, 1, 3, 3}}, + { + { ngraph::element::f32, false }, + { {127.f}, element::f32, {/* from parameter */}, false, 1ul, element::i8, true }, + { {0.06f}, element::f32, {/* from parameter */}, false } + }, + {}, + {}, + {}, + {{960, 1, 1, 3, 3}}, + ngraph::element::f32, + {} + } + } +}; + +// clang-format on INSTANTIATE_TEST_SUITE_P(smoke_LPT, PullReshapeThroughDequantizationTransformation, diff --git a/src/common/low_precision_transformations/tests/pull_transpose_through_dequantization_transformation.cpp b/src/common/low_precision_transformations/tests/pull_transpose_through_dequantization_transformation.cpp index 8b5d9a0b6447b8..6979bdcc36616d 100644 --- a/src/common/low_precision_transformations/tests/pull_transpose_through_dequantization_transformation.cpp +++ b/src/common/low_precision_transformations/tests/pull_transpose_through_dequantization_transformation.cpp @@ -126,7 +126,12 @@ TEST_P(PullTransposeThroughDequantizationTransformation, CompareFunctions) { ASSERT_TRUE(res.first) << res.second; } -const std::vector inputShapes = {ngraph::Shape({1, 960, 7, 7}), ngraph::Shape({4, 960, 7, 7})}; +// clang-format off + +const std::vector inputShapes = { + ngraph::Shape({1, 960, 7, 7}), + ngraph::Shape({4, 960, 7, 7}) +}; const std::vector> dequantizationOnWeightElementwiseConstantShapes = { {ngraph::Shape({}), ngraph::Shape({1, 1, 1, 1})}, @@ -178,37 +183,54 @@ const std::vector testValues = { // \ / // Multiply // - {LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), - // ActualValues - {ngraph::element::u8, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, - {{0.02f}, element::f32, {}, false}}, - {std::vector{2.f}, ngraph::element::i8, {3, 3, 960, 1}}, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {/* from parameter */}, false}, - {{0.03f}, element::f32, {/* from parameter */}, false}}, - {}, // reshape1 - {}, // multiply - {{2, 3, 0, 1}}, - {{960, 1, 1, 3, 3}}, - ngraph::element::f32, - {}}, - // ExpectedValues - {ngraph::element::u8, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, - {{0.02f}, element::f32, {}, false}}, - {std::vector{2.f}, ngraph::element::i8, {960, 1, 3, 3}}, - {{ngraph::element::f32, false}, - {{127.f}, element::f32, {/* from parameter */}, false}, - {{0.03f}, element::f32, {/* from parameter */}, false}}, - {}, - {}, - {}, - {{960, 1, 1, 3, 3}}, - ngraph::element::f32, - {}}}}; + { + LayerTransformation::createParamsU8I8().setSupportAsymmetricQuantization(true), + // ActualValues + { + ngraph::element::u8, + { + {ngraph::element::f32, false}, + {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, + {{0.02f}, element::f32, {}, false} + }, + {std::vector{2.f}, ngraph::element::i8, {3, 3, 960, 1}}, + { + {ngraph::element::f32, false}, + {{127.f}, element::f32, {/* from parameter */}, false}, + {{0.03f}, element::f32, {/* from parameter */}, false} + }, + {}, // reshape1 + {}, // multiply + {{2, 3, 0, 1}}, + {{960, 1, 1, 3, 3}}, + ngraph::element::f32, + {} + }, + // ExpectedValues + { + ngraph::element::u8, + { + {ngraph::element::f32, false}, + {{127.f}, element::f32, {}, false, 1ul, element::u8, true}, + {{0.02f}, element::f32, {}, false} + }, + {std::vector{2.f}, ngraph::element::i8, {960, 1, 3, 3}}, + { + {ngraph::element::f32, false}, + {{127.f}, element::f32, {/* from parameter */}, false}, + {{0.03f}, element::f32, {/* from parameter */}, false} + }, + {}, + {}, + {}, + {{960, 1, 1, 3, 3}}, + ngraph::element::f32, + {} + } + } +}; + +// clang-format on INSTANTIATE_TEST_SUITE_P(smoke_LPT, PullTransposeThroughDequantizationTransformation, diff --git a/src/common/low_precision_transformations/tests/simple_low_precision_transformer.cpp b/src/common/low_precision_transformations/tests/simple_low_precision_transformer.cpp index 686390d53fce3f..a83c6f8b5d8248 100644 --- a/src/common/low_precision_transformations/tests/simple_low_precision_transformer.cpp +++ b/src/common/low_precision_transformations/tests/simple_low_precision_transformer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,7 @@ SimpleLowPrecisionTransformer::SimpleLowPrecisionTransformer( // TODO: use one pass manager markup = std::make_shared(passConfig); + markup->register_pass(); markup->register_pass(params.defaultPrecisions); markup->register_pass(precisionRestrictions, params.defaultPrecisions); diff --git a/src/common/offline_transformations/include/mask_attribute.hpp b/src/common/offline_transformations/include/mask_attribute.hpp index 33ec516ded9c94..22ad9ba70d054a 100644 --- a/src/common/offline_transformations/include/mask_attribute.hpp +++ b/src/common/offline_transformations/include/mask_attribute.hpp @@ -28,7 +28,7 @@ namespace ngraph { class Mask : public std::vector>, public std::enable_shared_from_this { public: static const ::ov::DiscreteTypeInfo& get_type_info_static() { - static const ::ov::DiscreteTypeInfo type_info_static{"Mask", 0, "0"}; + static const ::ov::DiscreteTypeInfo type_info_static{"Mask", "0"}; return type_info_static; } diff --git a/src/common/offline_transformations/src/compress_quantize_weigths.cpp b/src/common/offline_transformations/src/compress_quantize_weigths.cpp index 819ca0752ee2ec..a1d89520cbf481 100644 --- a/src/common/offline_transformations/src/compress_quantize_weigths.cpp +++ b/src/common/offline_transformations/src/compress_quantize_weigths.cpp @@ -70,7 +70,9 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() { auto fq_users = fq->get_users(); if (fq_users.size() == 1 && has_dequantization_subgraph(fq_users[0])) { auto& first_convert = fq_users[0]; + OPENVINO_SUPPRESS_DEPRECATED_START if (auto new_weights = ov::get_constant_from_source(first_convert)) { + OPENVINO_SUPPRESS_DEPRECATED_END new_weights->set_friendly_name(first_convert->get_friendly_name()); replace_node(first_convert, new_weights); copy_runtime_info(first_convert, new_weights); @@ -108,7 +110,9 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() { // Convert quantized weights to low precision type std::shared_ptr new_weights = std::make_shared(quantize, quantized_type); // Constant fold quantized weights + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = ov::get_constant_from_source(new_weights)) { + OPENVINO_SUPPRESS_DEPRECATED_END new_weights = constant; } else { return false; @@ -146,17 +150,26 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() { std::shared_ptr scale = std::make_shared(output_range, input_range); auto descaled_output_low = std::make_shared(output_low, scale); std::shared_ptr shift = std::make_shared(new_output_low, descaled_output_low); - if (auto constant = ov::get_constant_from_source(scale)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = ov::get_constant_from_source(scale)) { + OPENVINO_SUPPRESS_DEPRECATED_END scale = constant; + } auto zero = op::Constant::create(input_type, Shape{}, {0}); auto scale_eq_zero = std::make_shared(scale, zero); // shift equals to input_low - output_low / scale // for positions where scale == 0, we put zero as shift std::shared_ptr zero_point = std::make_shared(scale_eq_zero, zero, shift); - if (auto constant = ov::get_constant_from_source(zero_point)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = ov::get_constant_from_source(zero_point)) { + OPENVINO_SUPPRESS_DEPRECATED_END zero_point = constant; - if (auto constant = ov::get_constant_from_source(scale)) + } + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = ov::get_constant_from_source(scale)) { + OPENVINO_SUPPRESS_DEPRECATED_END scale = constant; + } auto convert_to_high_prec = std::make_shared(new_weights, input_type); auto sub = register_new_node(convert_to_high_prec, zero_point); auto mul = register_new_node(sub, scale); @@ -206,7 +219,9 @@ ngraph::pass::ZeroPointOptimizer::ZeroPointOptimizer() { zero_point, std::make_shared(int8_zero_point, convert->get_element_type())); + OPENVINO_SUPPRESS_DEPRECATED_START auto adj_zero_point_const = ov::get_constant_from_source(adj_zero_point); + OPENVINO_SUPPRESS_DEPRECATED_END if (!adj_zero_point_const) return false; auto adj_zero_point_val = adj_zero_point_const->cast_vector(); @@ -222,7 +237,9 @@ ngraph::pass::ZeroPointOptimizer::ZeroPointOptimizer() { convert->get_element_type()), adj_zero_point); auto diff = std::make_shared(sub, transformed); + OPENVINO_SUPPRESS_DEPRECATED_START auto diff_const = ov::get_constant_from_source(diff); + OPENVINO_SUPPRESS_DEPRECATED_END if (!diff_const) return false; auto diff_val = diff_const->cast_vector(); @@ -233,10 +250,13 @@ ngraph::pass::ZeroPointOptimizer::ZeroPointOptimizer() { return false; std::shared_ptr new_weights = std::make_shared(weights, int8_zero_point); - if (auto constant = ov::get_constant_from_source(new_weights)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = ov::get_constant_from_source(new_weights)) { + OPENVINO_SUPPRESS_DEPRECATED_END new_weights = constant; - else + } else { return false; + } new_weights->set_friendly_name(weights->get_friendly_name()); replace_node(weights, new_weights); diff --git a/src/common/offline_transformations/src/pruning/init_masks.cpp b/src/common/offline_transformations/src/pruning/init_masks.cpp index e1c22d1d0f1aef..709090e9561ba4 100644 --- a/src/common/offline_transformations/src/pruning/init_masks.cpp +++ b/src/common/offline_transformations/src/pruning/init_masks.cpp @@ -86,7 +86,9 @@ class ngraph::pass::init_masks::InitMatMulMask : public MatcherPass { while (!ngraph::is_type(cur_node) && cur_node->inputs().size()) { weights_calculation_nodes.push_back(cur_node); if (ngraph::is_type(cur_node)) { + OPENVINO_SUPPRESS_DEPRECATED_START const auto forward_order = get_constant_from_source(cur_node->get_input_node_shared_ptr(1)); + OPENVINO_SUPPRESS_DEPRECATED_END if (!forward_order) return false; const auto forward_order_vec = forward_order->cast_vector(); diff --git a/src/common/offline_transformations/src/pruning/propagate_masks.cpp b/src/common/offline_transformations/src/pruning/propagate_masks.cpp index 717443521caa73..6f289cbff6cb24 100644 --- a/src/common/offline_transformations/src/pruning/propagate_masks.cpp +++ b/src/common/offline_transformations/src/pruning/propagate_masks.cpp @@ -397,7 +397,9 @@ class ngraph::pass::mask_propagation::GroupConvolutionReshape : public MatcherPa return false; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto constant = get_constant_from_source(m_shape.get_node_shared_ptr()); + OPENVINO_SUPPRESS_DEPRECATED_END if (!constant) { NGRAPH_DEBUG << "Can't get constant from source node " << m_shape.get_node()->get_friendly_name(); return false; @@ -1130,7 +1132,9 @@ class ngraph::pass::mask_propagation::Reshape : public MatcherPass { auto constant = std::dynamic_pointer_cast(m_weights.get_node_shared_ptr()); if (!constant) { + OPENVINO_SUPPRESS_DEPRECATED_START constant = get_constant_from_source(m_weights.get_node_shared_ptr()); + OPENVINO_SUPPRESS_DEPRECATED_END if (!constant) { NGRAPH_DEBUG << "Can't process reshape node " << m_output.get_node()->get_friendly_name() << " with no constant node " << m_weights.get_node()->get_friendly_name() @@ -1375,7 +1379,9 @@ class ngraph::pass::mask_propagation::Transpose : public MatcherPass { const auto& m_weights = pattern_map.at(weights); const auto& m_output = pattern_map.at(transpose); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_order_node = get_constant_from_source(m_weights.get_node_shared_ptr()); + OPENVINO_SUPPRESS_DEPRECATED_END if (!input_order_node) { NGRAPH_DEBUG << "Can't process transpose node " << m_output.get_node()->get_friendly_name() << " with no constant node " << m_weights.get_node()->get_friendly_name() diff --git a/src/common/offline_transformations/src/pruning/shrink_weights.cpp b/src/common/offline_transformations/src/pruning/shrink_weights.cpp index 39a18a647e4693..f0520814cbdb81 100644 --- a/src/common/offline_transformations/src/pruning/shrink_weights.cpp +++ b/src/common/offline_transformations/src/pruning/shrink_weights.cpp @@ -37,7 +37,9 @@ static bool is_static_reshape_op(std::shared_ptr node) { if (input.get_partial_shape().is_dynamic() || shape.get_partial_shape().is_dynamic()) return false; + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape_const_op = get_constant_from_source(shape); + OPENVINO_SUPPRESS_DEPRECATED_END if (!output_shape_const_op) return false; @@ -59,7 +61,9 @@ static bool maybe_adopt_reshape_node(std::shared_ptr reshape, ngraph:: return false; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto constant = get_constant_from_source(shape); + OPENVINO_SUPPRESS_DEPRECATED_END if (!constant) { return false; } @@ -314,7 +318,9 @@ bool ngraph::pass::ShrinkWeights::run_on_model(const std::shared_ptr { virtual void isApplicable(const Blob::Ptr &src, const Blob::Ptr &dst) = 0; protected: - ~IPreProcessData() = default; + virtual ~IPreProcessData() = default; }; OPENVINO_PLUGIN_API void CreatePreProcessData(std::shared_ptr& data); diff --git a/src/common/snippets/CMakeLists.txt b/src/common/snippets/CMakeLists.txt index d3a7e47c60423c..a37d5343092349 100644 --- a/src/common/snippets/CMakeLists.txt +++ b/src/common/snippets/CMakeLists.txt @@ -26,7 +26,7 @@ ie_faster_build(${TARGET_NAME} ) target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime - PRIVATE ngraph_reference ov_shape_inference openvino::runtime::dev) + PRIVATE ngraph_reference openvino::runtime::dev) target_include_directories(${TARGET_NAME} PUBLIC $ PRIVATE $) diff --git a/src/common/snippets/README.md b/src/common/snippets/README.md index eca770a584cda2..8f9d55bc11714b 100644 --- a/src/common/snippets/README.md +++ b/src/common/snippets/README.md @@ -2,12 +2,13 @@ ## Key Contacts -Please contact a member of [openvino-ie-cpu-maintainers](https://github.com/orgs/openvinotoolkit/teams/openvino-ie-cpu-maintainers) group, for assistance regarding snippets. +For assistance regarding snippets, contact a member of [openvino-ie-cpu-maintainers](https://github.com/orgs/openvinotoolkit/teams/openvino-ie-cpu-maintainers) group. * [SnippetS design guide](./docs/snippets_design_guide.md) * [CPU target for SnippetS code generator](./docs/snippets_cpu_target.md) ## See also + * [OpenVINO™ README](../../../README.md) * [OpenVINO Core Components](../../README.md) * [Developer documentation](../../../docs/dev/index.md) \ No newline at end of file diff --git a/src/common/snippets/docs/snippets_cpu_target.md b/src/common/snippets/docs/snippets_cpu_target.md index 04b70f7df8708e..68f03202c4761d 100644 --- a/src/common/snippets/docs/snippets_cpu_target.md +++ b/src/common/snippets/docs/snippets_cpu_target.md @@ -1,12 +1,12 @@ -# CPU target for SnippetS code generator +# CPU Target for SnippetS Code Generator -Snippets in its first generation can be seen as a generalization over generic eltwise node. First generation of snippets has lack of integration with oneDNN and so patterns it supports should be kept orthogonal to what is fused with post-ops. +Snippets in its first generation can be seen as a generalization over a generic eltwise node. First generation of snippets does not have integration with oneDNN, and the patterns it supports should be kept orthogonal to what is fused with post-ops. -POC CPU implementation could be found [here](https://github.com/openvinotoolkit/openvino/pull/2824) +See the example of POC CPU implementation [here](https://github.com/openvinotoolkit/openvino/pull/2824). First 8 kernel parameters are passed by structure which is unpacked inside a kernel into the registers. The rest are passed through the stack. -Loop trip count should be placed to some GP register, as well as work amount. Moreover, we need to load all the parameters into GP registers. If we assume that we have enough registers than it can be done before the loop body. +The loop trip count should be placed to a GP register, as well as the work amount. Moreover, you need to load all the parameters into GP registers. If you assume that you have enough registers, then it can be done before the loop body. ``` auto param0 = abi_params[0]; @@ -18,9 +18,9 @@ auto work_amount = abi_params[3]; ## Memory operations -Load could be Vector, Scalar and Broadcast. Only native vector size for an architecture is supported (e.g. 16 on AVX-512) +A load could be Vector, Scalar, and Broadcast. Only the native vector size for an architecture is supported (for example, 16 on AVX-512). -Memory operation also generates post increments for the pointer it uses. +Memory operation also generates post increments for the pointer it uses. - `MemoryEmitter` - `StoreEmitter` @@ -50,8 +50,8 @@ Tensor data can be passed with strides. `Data` corresponds to a constant table and wraps this entity for the CPU. ## See also + * [OpenVINO™ README](../../../../README.md) * [OpenVINO SnippetS](../README.md) * [OpenVINO Core Components](../../../README.md) * [Developer documentation](../../../../docs/dev/index.md) - \ No newline at end of file diff --git a/src/common/snippets/docs/snippets_design_guide.md b/src/common/snippets/docs/snippets_design_guide.md index 01b005b20e4ec9..d495b35a3fc437 100644 --- a/src/common/snippets/docs/snippets_design_guide.md +++ b/src/common/snippets/docs/snippets_design_guide.md @@ -1,26 +1,26 @@ -# SnippetS design guide -This document describes the design and rationale for snippets code generator. Implementation of code functionality is located [here](https://github.com/openvinotoolkit/openvino/tree/master/src/common/snippets). Proposal for CPU backend integration is [here](https://github.com/openvinotoolkit/openvino/pull/2824). +# SnippetS Design Guide +This document describes the design and rationale for a snippets code generator. Implementation of code functionality is located [here](https://github.com/openvinotoolkit/openvino/tree/master/src/common/snippets). A proposal for CPU backend integration is [here](https://github.com/openvinotoolkit/openvino/pull/2824). ## Rationale -We believe that core **CNN operators (convolution, gemm, fully connected) are limited by compute, the rest is memory bound**. Math approximations (like transcendental functions) are rare in emerging workloads and could be treated with the same machinery. **Snippets are designed to optimize topology for memory**, while leaving compute intensive kernels for backend developers. +Core **CNN operators (convolution, gemm, fully connected) are limited by compute, the rest is memory bound**. Math approximations (like transcendental functions) are rare in emerging workloads and could be treated with the same machinery. **Snippets are designed to optimize topology for memory**, while leaving compute intensive kernels for backend developers. -We believe **potential speedup is proportional to shrink in memory-walked bytes**. So we can transform the problem to a task to optimize for memory walks, whatever pattern snippet has and operations it contains. Number of memory walks should be less or equal to handcrafted optimizations. This guarantees performance improvements over the previous approach (excluding corner cases caused by cache effects). *Shrinkage factor might be encoded to some cost function in future evolution of code generator*. Snippets generator provides diagnostics to estimate this shrinkage factor with `ngraph::snippets::op::Subgraph::print_statistics(bool verbose)` member. +The **potential speedup is proportional to shrink in memory-walked bytes**. Therefore, you can transform the problem to a task to optimize for memory walks, whatever pattern snippet has and operations it contains. The number of memory walks should be less or equal to handcrafted optimizations. This guarantees performance improvements over the previous approach (excluding corner cases caused by cache effects). *Shrinkage factor might be encoded to some cost function in future evolution of code generator*. Snippets generator provides diagnostics to estimate this shrinkage factor with `ngraph::snippets::op::Subgraph::print_statistics(bool verbose)` member. -We design SnippetS generator for back-end developers. The main purpose of inventing snippets code generator is an **operator fusion**, **register allocation** and **target kernel generation** decomposition. This allows modifications (like new fusion support) and feature extensions (like new operation support) to be done in a single point of modification and avoid combinatorial explosion for fusions/types/architectures etc. +The SnippetS generator is designed for back-end developers. The main purpose of inventing the snippets code generator is an **operator fusion**, **register allocation** and **target kernel generation** decomposition. This allows modifications (like new fusion support) and feature extensions (like new operation support) to be done in a single point of modification and avoid combinatorial explosion for fusions/types/architectures etc. -We believe that creating a full-fledged compiler or usage of existing compiler infrastructure (like LLVM & MLIR) is superfluous at this point of evelition. We aim to provide a **flexible and performant framework for operation fusions**, leaving micro optimizations (e.g. instruction scheduling) to the backend H/W. +Creating a full-fledged compiler or usage of existing compiler infrastructure (like LLVM & MLIR) is superfluous at this point of evolution. The aim is to provide a **flexible and performant framework for operation fusions**, leaving micro optimizations (for example, instruction scheduling) to the backend H/W. -We do not aim to invent a DSL for SnippetS and would like to keep it this way. DSL gives users more flexibility to express uncommon operations. However, the shift towards an approach to encode topologies with elementary operations followed by smart enough fusions is already expressive and performant enough. +There are no plans to invent a DSL for SnippetS. DSL gives users more flexibility to express uncommon operations. However, the shift towards an approach to encode topologies with elementary operations followed by smart enough fusions is already expressive and performant enough. -**Snippet** is a compiled compute **kernel** generated from a subgraph using SnippetS code generator for specific architecture with a **scheduling domain**. Using this scheduling domain and calling convention backend can execute generated compute kernels. For the first generation, snippets are **statically scheduled towards the output domain**. Multi-output snippets are supported if all outputs are broadcast-compatible in a sense that domains for all outputs can be broadcasted from one root domain which defines snippet schedule. It’s a subject of extension for future generations. +**Snippet** is a compiled compute **kernel** generated from a subgraph using the SnippetS code generator for a specific architecture with a **scheduling domain**. Using this scheduling domain and calling convention backend can execute generated compute kernels. For the first generation, snippets are **statically scheduled towards the output domain**. Multi-output snippets are supported if all outputs are broadcast-compatible in a sense that domains for all outputs can be broadcasted from one root domain that defines snippet schedule. It is a subject of extension for future generations. -We use nGraph as the highest level IR for subgraph representation and lowering transformations. **Opset1** is a base operation set for code generation. We aim to **keep the minimal possible and sufficient operation set** (or ISA) and keep it **RISC-like** (memory and compute decomposed). +nGraph is used as the highest level IR for subgraph representation and lowering transformations. **Opset1** is a base operation set for code generation. The aim is to **keep the minimal possible and sufficient operation set** (or ISA) and keep it **RISC-like** (memory and compute decomposed). -**One subgraph corresponds to one snippet**. Operations which cannot be scheduled by a single schedule should not be placed in the same subgraph. Snippet somewhat conceptually close to OpenCL kernel without a restriction to express only embarrassingly parallel tasks. +**One subgraph corresponds to one snippet**. Operations which cannot be scheduled by a single schedule should not be placed in the same subgraph. A snippet is somewhat conceptually close to OpenCL kernel without a restriction to express only embarrassingly parallel tasks. **Subgraph** once extracted from full topology IR is **treated as an operation and data flow descriptor in scalar notation** (similar to OpenCL/CUDA). Tensor sizes are used for defining scheduling domain and detecting broadcasts/reductions. -We split operations into 3 groups: **layout-oblivious (LOO), layout-aware(-tolerant) and layout-dependent**. **Layout-oblivious** operation semantics and implementation are completely agnostic to a specific layout in which tensors are placed in memory. For example, elements-wise math and ReLU does in this category. Implementation **layout-aware** operation depends on the layout of input/output tensors. For example, convolutions and other block-wise kernels or layout repaks. For **layout-specific** operation semantics and implementation depends on the layout. For example, the Yolo region. Patterns to fuse constructed in terms of taxonomy above. +Operations are split into 3 groups: **layout-oblivious (LOO), layout-aware(-tolerant) and layout-dependent(-specific)**. **Layout-oblivious** operation semantics and implementation are completely agnostic to a specific layout in which tensors are placed in memory. For example, like elements-wise math and ReLU in this category. Implementation of **layout-aware** operation depends on the layout of input/output tensors. For example, convolutions and other block-wise kernels or layout repacks. **Layout-specific** operation semantics and implementation depend on the layout. For example, the Yolo region. Patterns to fuse are constructed in terms of taxonomy above. ## Design @@ -28,19 +28,19 @@ Code generation is split into 2 phases, **tokenization** and **lowering**. ### Tokenization -Tokenization runs on full topology nGraph function inside a specific plugin in a stage of common transformations. Input of tokenization is a topology graph. Output is a modified topology graph with `ngraph::snippets::op::Subgraph` operations installed. Each subgraph contains nGraph function (called **body**) which holds a part of original topology legal for snippet generation (can be scheduled with a single schedule) +Tokenization runs on full topology nGraph function inside a specific plugin in a stage of common transformations. Input of tokenization is a topology graph. Output is a modified topology graph with `ngraph::snippets::op::Subgraph` operations installed. Each subgraph contains nGraph function (called **body**) which holds a part of original topology legal for snippet generation (can be scheduled with a single schedule). -Procedure of finding subgraphs suitable for code generation is called **tokenization**, meaning that we split the topology tree into subgraphs in the same greedy approach which is used for parsing input stream of characters into the tokens. It also could be seen as and modified into a basic block construction problem, since we also find a leader and potentially terminators. Implementation can be found [here](https://github.com/openvinotoolkit/openvino/blob/master/src/common/snippets/src/pass/collapse_subgraph.cpp). +A procedure of finding subgraphs suitable for code generation is called **tokenization**. During tokenization the topology tree is split into subgraphs in the same greedy approach which is used for parsing input stream of characters into the tokens. It may also be seen as and modified into a basic block construction problem, since there is a leader and potentially terminators. See the example of implementation [here](https://github.com/openvinotoolkit/openvino/blob/master/src/common/snippets/src/pass/collapse_subgraph.cpp). -Tokenization has an advantage over the pattern matching approach (used in traditional and MLIR-based compilers) since it can handle arbitrary patterns of operations. Pattern matching deduces specific configuration of operations to translate to another one, more suitable for target machine or further lowering. This means that relations between operations are fixed. Tokenization on the other hand has the only limitation on specific operation types which are **suitable and profitable** to fuse with respect to original topology correctness (keeping it as a direct acyclic graph). +Tokenization has an advantage over the pattern matching approach (used in traditional and MLIR-based compilers) since it can handle arbitrary patterns of operations. Pattern matching deduces specific configuration of operations to translate to another one, more suitable for target machine or further lowering. This means that relations between operations are fixed. Tokenization, on the other hand, has the only limitation on specific operation types which are **suitable and profitable** to fuse, respecting original topology correctness (keeping it as a direct acyclic graph). -The extracted body comes to a plug-in wrapped as a composite `Subgraph` operation which is seen as a block box from a plugin standpoint and can participate in any plugin specific subroutines (e.g. layout assignment, memory allocation, etc.). +The extracted body comes to a plug-in wrapped as a composite `Subgraph` operation which is seen as a block box from a plugin standpoint and can participate in any plugin specific subroutines (for example, layout assignment, memory allocation, etc.). ### Supported subgraph patterns -Subgraph accepts arbitrary numbers of inputs and outputs. There is 1:1 mapping for external (subgraph node’s) and internal (body) parameters indexes. +Subgraph accepts arbitrary numbers of inputs and outputs. There is 1:1 mapping for external (subgraph node’s) and internal (body) parameters indexes. -Pattern here is an exact subgraph configuration (nodes and edges between them). **The first generation of snippets supports only layout-oblivious operations which may have broadcast on inputs and broadcast-compatible outputs**. For example Shapes `<1, 42, 17, 31>`, `<1, 42, 17, 1>` and `<1, 42, 1, 31>` are considered as broadcast-compatible. Layout-oblivious operation with multiple outputs as a snippet leader and forms a new subgraph. The most beneficial patterns are subgraphs with complex control flow but minimal number of inputs/and outputs. For example, GeLU has a 5x shrinkage factor from original unfused subgraph in number of bytes walked. Subgraph below could be considered as an example of such a subgraph. Leader detection procedure aims to find such subgraphs. +Pattern here is an exact subgraph configuration (nodes and edges between them). **The first generation of snippets supports only layout-oblivious operations which may have broadcast on inputs and broadcast-compatible outputs**. For example Shapes `<1, 42, 17, 31>`, `<1, 42, 17, 1>` and `<1, 42, 1, 31>` are considered as broadcast-compatible. Layout-oblivious operation with multiple outputs serves as a snippet leader and forms a new subgraph. The most beneficial patterns are subgraphs with complex control flow but minimal number of inputs/and outputs. For example, GeLU has a 5x shrinkage factor from original unfused subgraph in number of bytes walked. Subgraph below could be considered as an example of such a subgraph. Leader detection procedure aims to find such subgraphs. ```mermaid flowchart LR @@ -60,12 +60,12 @@ class nodeA3 steel1 ``` Operations are greedily added to the subgraph until -1. New operation doesn’t introduce a loop in a topology function. +1. New operation does not introduce a loop in a topology function. 1. Number of inputs and outputs satisfies target criteria. 1. Operation is not a predecessor of topology output. -1. Resulting subgraph can be scheduled (all outputs are broadcast-compatible). +1. Resulting subgraph can be scheduled (all outputs are broadcast-compatible). -If a potential subgraph doesn’t meet any of criteria above, the procedure continues to find a new leader. +If a potential subgraph does not meet any of the criteria above, the procedure continues to find a new leader. ### Lowering @@ -82,27 +82,27 @@ Lowering is a sequence of subgraph (snippet body) traversal passes to generate a #### Common optimizations -Constants are treated as inputs for a subgraph with an exception for scalar cases (since we don’t need to schedule them). `snippets::op::Scalar` is used to represent this kind of constants. +Constants are treated as inputs for a subgraph with an exception for scalar cases (since they do not need to be scheduled). `snippets::op::Scalar` is used to represent this kind of constants. -If such Scalar comes as a second input of Power operation, it’s replaced with `snippets::op::PowerStatic`. +If such Scalar comes as a second input of Power operation, it is replaced with `snippets::op::PowerStatic`. #### Canonicalization -The goal of this step is to apply target independent and schedule related optimizations and to make snippet **schedulable**. +The goal of this step is to apply target-independent and schedule-related optimizations and to make a snippet **schedulable**. ##### Domain normalization All input and output shapes are normalized to 6D for future schedule generation. If shape propagation fails or leads to inconsistent output shapes an exception is raised. -Layout assigned by user code and passed to a `generate` function is propagated through subgraph on this step as well. Layout is passed to a generate function as a `BlockedShapeVector` which is a `std::vector` , while `BlockedShape` is `std::tuple`. For example, if backend supports `NCHW16c` layout and tensor has size of `<1, 42, 17, 31>` and hold single precision floating point this structure should be `std::make_tuple(ngraph::Shape {1, 3, 17, 31, 16}, ngraph::AxisVector {0, 1, 2, 3, 1}, ngraph::element::f32);`. This allows generic layout representation. +The layout assigned by a user code and passed to a `generate` function is propagated through a subgraph on this step as well. The layout is passed to a `generate` function as a `BlockedShapeVector` which is a `std::vector` , while `BlockedShape` is `std::tuple`. For example, if backend supports `NCHW16c` layout and a tensor has a size of `<1, 42, 17, 31>` and holds single precision floating point, this structure should be `std::make_tuple(ngraph::Shape {1, 3, 17, 31, 16}, ngraph::AxisVector {0, 1, 2, 3, 1}, ngraph::element::f32);`. This allows generic layout representation. ##### Dialect conversion -The goal for this step is to transform a subgraph (body function) into a form possible to code generation. Input for this step is subgraph in a canonical form output is a subgraph in snippets dialect. +The goal for this step is to transform a subgraph (body function) into a form possible for code generation. Input for this step is a subgraph in a canonical form. Output is a subgraph in snippets dialect. -Snippet or kernel is formed around the subgraph body in a sequence of traversal steps. Let’s walk through these steps with the smallest possible subgraph which contains out of single `[Add]` operation. +A snippet or a kernel is formed around the subgraph body in a sequence of traversal steps. Let us walk through these steps with the smallest possible subgraph which contains a single `[Add]` operation. -While we extract subgraphs with the tokenization part we explicitly insert Parameters and Results to its body to form a complete nGraph Function. +When subgraphs are extracted with the tokenization part, Parameters and Results are explicitly inserted to its body to form a complete nGraph Function. ```mermaid flowchart LR @@ -118,11 +118,11 @@ class nodeA8 steel1 class nodeA1,nodeA3 steel1 ``` -This function represents operation dependencies in scalar (similar to OpenCL) notation while shapes of tensors are used to generate schedules. At this point kernel-schedule decomposition is made (similar to Halide/OpenCL/TVM) +This function represents operation dependencies in scalar (similar to OpenCL) notation while shapes of tensors are used to generate schedules. At this point, kernel-schedule decomposition is made (similar to Halide/OpenCL/TVM). ###### Explicit memory operations -As a next step explicit memory operations are placed for each input and output. `InsertLoad` and `InsertStore` passes derived from `MatcherPass`. +As a next step, explicit memory operations are placed for each input and output. `InsertLoad` and `InsertStore` passes derive from `MatcherPass`. ```mermaid flowchart LR @@ -142,16 +142,16 @@ class nodeA8 carbon1 class nodeA1,nodeA3,nodeA6,nodeA7 steel1 ``` -By default, memory operations assumes vector memory access, if scalar access is needed special passes `ReplaceLoadsWithScalarLoads` and `ReplaceStoresWithScalarStores` should be executed. +By default, memory operations assume vector memory access. If scalar access is needed, special `ReplaceLoadsWithScalarLoads` and `ReplaceStoresWithScalarStores` passes should be executed. ###### Explicit broadcast -For each operation in body function inputs are checked against broadcasting. In case of parameters to be broadcasted explicit broadcast operation is generated. For example, if for the subgraph above we have `<1, 42, 17, 31>` and `<1, 42, 17, 1>` resulting subgraph is going to be +For each operation in body function inputs are checked against broadcasting. When Parameters are to be broadcasted, an explicit broadcast operation is generated. For example, with `<1, 42, 17, 31>` and `<1, 42, 17, 1>` for the subgraph above, the resulting subgraph will be: ```mermaid flowchart LR - nodeA1("Parameter\n<1, 42, 17, 1>") --> node6("Load\n<1, 42, 17, 1>") - node6("Load\n<1, 42, 17, 1>") --> nodeA9("BroadcastMove\n<1, 42, 17, 31>") + nodeA1("Parameter\n<1, 42, 17, 1>") --> nodeA6("Load\n<1, 42, 17, 1>") + nodeA6("Load\n<1, 42, 17, 1>") --> nodeA9("BroadcastMove\n<1, 42, 17, 31>") nodeA9("BroadcastMove\n<1, 42, 17, 31>") --> nodeA2(Add) nodeA3("Parameter\n<1, 42, 17, 31>") --> nodeA7("Load\n<1, 42, 17, 31>") nodeA7("Load\n<1, 42, 17, 31>") ---> nodeA2(Add) @@ -164,10 +164,10 @@ classDef daisy1 fill:#FFE17A, stroke: #FEC91B, color: #262626 class nodeA2 daisy1 class nodeA5 moss1 class nodeA8,nodeA9 carbon1 -class nodeA1,nodeA3,node6,nodeA7 steel1 +class nodeA1,nodeA3,nodeA6,nodeA7 steel1 ``` -If load followed by broadcast is detected then this pair is replaced by a single Broadcast load instruction. Like the following +If Load followed by Broadcast is detected, then this pair is replaced by a single BroadcastLoad instruction: ```mermaid flowchart LR @@ -187,7 +187,7 @@ class nodeA8 carbon1 class nodeA1,nodeA3,nodeA6,nodeA7 steel1 ``` -Broadcast and regular streaming vector load is possible from the same pointer. Broadcast load should always go before streaming load. Broadcast load for non the most varying dimension is not generated, however it affects the generated schedule. +Broadcast and regular streaming vector load is possible from the same pointer. BroadcastLoad should always go before streaming load. BroadcastLoad for non the most varying dimension is not generated, however it affects the generated schedule. #### Target-specific optimizations @@ -197,13 +197,13 @@ Target developers can plug in to the code generation pipeline some specific opti #### Register allocation -Canonicalized subgraph in a snippets dialect forms a basic block or region inside a snippet (kernel). Registers are allocated globally for the whole subgraph. Since all operations for a subgraph are assumed to be vector, only vector registers are allocated for the first generation of SnippetS. Linear scan register allocation algorithm is used. Register allocator is implemented as a function pass `ngraph::snippets::pass::AssignRegisters` and store allocated registers for each node into `rt_info`. `rt_info` for a node holds a register for Node's output. *However, this part should be refactored batter, either to become target independent or use target specific abstraction to acquire a new register* +Canonicalized subgraph in a snippets dialect forms a basic block or region inside a snippet (kernel). Registers are allocated globally for the whole subgraph. Since all operations for a subgraph are assumed to be vector, only vector registers are allocated for the first generation of SnippetS. Linear scan register allocation algorithm is used. Register allocator is implemented as the `ngraph::snippets::pass::AssignRegisters` function pass and store allocated registers for each node into `rt_info`. `rt_info` for a node holds a register for Node's output. *However, this part should be refactored better, either to become target independent or to use target-specific abstraction to acquire a new register* -#### Schedule generation +#### Schedule generation -The goal of this step is to transform subgraphs in a scalar notation into kernel functions callable from user code. `Kernel` and `Tile` operations are introduced for this purpose. Each of this operation has a constructor from code region described as a collection of operation and operands pairs `Kernel(const std::vector, ngraph::snippets::RegInfo>>& region);`. +The goal of this step is to transform subgraphs in a scalar notation into kernel functions callable from user code. The `Kernel` and `Tile` operations are introduced for this purpose. Each of these operations has a constructor from code region described as a collection of operation and operand pairs `Kernel(const std::vector, ngraph::snippets::RegInfo>>& region);`. -If we return to example above this comes to a following hierarchical IR. If we limit scope to layout oblivious operations with broadcasting support, tile could be generated as a single loop over the most warning dimension. The second `Tile` is generated to handle tails and can be omitted if not needed. Special pass replaces memory operations on vector to scalar versions for tail subgraph. +The example above can be used for the following hierarchical IR. If the scope to layout oblivious operations with broadcasting support is limited, `Tile` could be generated as a single loop over the most warning dimension. The second `Tile` is generated to handle tails and can be omitted if not needed. A special pass replaces memory operations on vector with scalar versions for tail subgraph. ```mermaid graph LR @@ -244,13 +244,13 @@ class nodeD1 no-stroke ``` Where -* `Kernel` constants a collection of the tiles, corresponds to a Subgraph node and responsible for function signature generation, calls generators for all tiles and data sections -* `Tile` contains single subgraph body, vector or scalar -* `Data` corresponds to data section aggregated for all nodes in all Tile’s subgraphs +* `Kernel` is a collection of the tiles, corresponds to a Subgraph node and is responsible for function signature generation. It calls generators for all tiles and data sections. +* `Tile` contains a single subgraph body, a vector or a scalar. +* `Data` corresponds to data section aggregated for all nodes in all Tile’s subgraphs. #### Target code emission -Target code emission is table based. Target is responsible for filling `jitters` table field in `Generator` class. +A target code emission is table based. A target is responsible for filling `jitters` table field in `Generator` class. ``` std::map(std::shared_ptr)>> jitters; @@ -260,9 +260,9 @@ std::map( An OpenVINO plugin is treated as a target for snippets. -Each nGraph node is mapped to a convertor function which creates `Emitter` form this node. Each specific emitter should extend from `Emitter`. It is used to map this node to target code and has `emit_code` and `emit_data` methods. `emit_data` is used during data section generation. All operations from snippets dialect which are legal for code generation should be expressed as operations derived from nGraph Op as well as Emitter derived snippets::Emitter class which knows how to translate this Op to Target specific ISA. (ex. xbyak is a jit backend for CPU plugin). +Each nGraph node is mapped to a converter function which creates `Emitter` form of the node. Each specific emitter should extend from `Emitter`. It is used to map the node to the target code and has `emit_code` and `emit_data` methods. The `emit_data` is used during data section generation. All operations from snippets dialect which are legal for code generation should be expressed as operations derived from nGraph Op as well as `Emitter` derived `snippets::Emitter` class which knows how to translate this Op to Target-specific ISA. (for example, xbyak is a jit backend for CPU plugin). -For minimal code generator support target should provide emitters for the following operations +For minimal code generator support, a target should provide emitters for the following operations: * `Kernel` * `Tile` @@ -273,29 +273,29 @@ For minimal code generator support target should provide emitters for the follow * `Store` * `ScalarStore` -Once a schedule is generated, target code is emitted from a kernel in Generator::generate method by executing Kernel::emit_code function. Since Kernel and Tile represents hierarchical +Once a schedule is generated, a target code is emitted from a kernel in `Generator::generate` method by executing `Kernel::emit_code` function. Since `Kernel` and `Tile` represent hierarchical IR. ##### Dialect extensibility -Target can potentially extend snippets dialect with target specific operation for code emission. It should implement: +A target can potentially extend the snippets dialect with a target-specific operation for code emission. It should implement: -* nGraph operation (ex. `class FMA : public ngraph::op::Op`) -* Emitter for this operation (ex. `class FmaEmitter : public Emitter` ) -* register this pair in `jitters` map +* nGraph operation (for example, `class FMA : public ngraph::op::Op`) +* Emitter for the operation (for example, `class FmaEmitter : public Emitter` ) +* register the pair in `jitters` map ### Calling convention -Parameters for a generated snippet are split into schedule-invariant and schedule-dependent. Schedule-invariant parameters include pointers to input/output tensors and strides for each of them with the same rank as scheduling domain. +Parameters for a generated snippet are split into schedule invariant and schedule dependent. Schedule-invariant parameters include pointers to input/output tensors and strides for each of them with the same rank as the scheduling domain. ### Diagnostics #### Reference mode -Subgraph can be executed with nGraph references if no generator is present. +A subgraph can be executed with nGraph references if no generator is present. ## See also + * [OpenVINO™ README](../../../../README.md) * [OpenVINO SnippetS](../README.md) * [OpenVINO Core Components](../../../README.md) * [Developer documentation](../../../../docs/dev/index.md) - diff --git a/src/common/snippets/include/snippets/generator.hpp b/src/common/snippets/include/snippets/generator.hpp index ab3156a108e3e1..48715235c11f42 100644 --- a/src/common/snippets/include/snippets/generator.hpp +++ b/src/common/snippets/include/snippets/generator.hpp @@ -16,6 +16,8 @@ namespace snippets { auto getRegisters(std::shared_ptr& n) -> ngraph::snippets::RegInfo; +typedef std::pair(const std::shared_ptr&)>, + std::function>(const std::shared_ptr&)>> jitters_value; /** * @interface TargetMachine * @brief Base class Target machine representation. Target derives from this class to provide generator information about supported emitters @@ -41,7 +43,6 @@ class TargetMachine { */ virtual size_t get_lanes() const = 0; - /** * @brief called by generator to all the emitter for a target machine * @return a map by node's type info with callbacks to create an instance of emitter for corresponding operation type @@ -51,7 +52,16 @@ class TargetMachine { if (jitter == jitters.end()) { throw ngraph_error(std::string("Target code emitter is not available for ") + type.name + " operation."); } - return jitter->second; + return jitter->second.first; + } + + std::function>(const std::shared_ptr&)> + get_supported_precisions(const ngraph::DiscreteTypeInfo type) const { + auto jitter = jitters.find(type); + if (jitter == jitters.end()) { + throw ngraph_error(std::string("Target code emitter is not available for ") + type.name + " operation."); + } + return jitter->second.second; } /** @@ -64,7 +74,7 @@ class TargetMachine { virtual ~TargetMachine() = default; protected: - std::map(std::shared_ptr)>> jitters; + std::map jitters; }; /** @@ -144,7 +154,29 @@ class Generator { */ std::shared_ptr get_target_machine() const; + /** + * @interface opRegType + * @brief Register type of operations + * Note that currently there are 4 types of ops: + * gpr->gpr: (Parameter, Result, LoopBegin, LoopEnd etc) + * gpr->vec: or vec->gpr Load/LoadConvert, Store/StoreConvert, BroadcastLoad etc. + * vec->vec: all other "normal" operations that perform calculations on vector registers: Add, BroadcastMove, Power, etc. + */ + enum opRegType {gpr2gpr, gpr2vec, vec2gpr, vec2vec}; + /** + * @brief gets register type by op type + * TODO: Should be static attribute of emitters + * @return register type + */ + opRegType get_op_reg_type(const std::shared_ptr& op) const; + protected: + /** + * @brief gets register type by specific plugin op type + * @return register type + */ + virtual opRegType get_specific_op_reg_type(const std::shared_ptr& op) const; + std::shared_ptr target; // todo: we need to save lowered code to access compiled brgemm kernels on execution time (normally lowered is destructed by then). // This is temporary solution, remove this when kernel caching is implemented. Don't forget to make generate const method. diff --git a/src/common/snippets/include/snippets/op/brgemm.hpp b/src/common/snippets/include/snippets/op/brgemm.hpp index 2746d974a06400..58c70f164799a6 100644 --- a/src/common/snippets/include/snippets/op/brgemm.hpp +++ b/src/common/snippets/include/snippets/op/brgemm.hpp @@ -5,7 +5,7 @@ #pragma once #include "ngraph/op/op.hpp" -#include "ngraph/op/matmul.hpp" +#include "memory_access.hpp" namespace ngraph { namespace snippets { @@ -16,30 +16,25 @@ namespace op { * @brief Brgemm is a batch-reduced matrix multiplication with the support of arbitrary strides between matrices rows * @ingroup snippets */ -class Brgemm : public ngraph::op::v0::MatMul { +class Brgemm : public MemoryAccess { public: - OPENVINO_OP("Brgemm", "SnippetsOpset", ngraph::op::v0::MatMul); - Brgemm(const Output& A, const Output& B, const size_t offset_a = 0lu, const size_t offset_b = 0lu, const size_t offset_c = 0lu); + OPENVINO_OP("Brgemm", "SnippetsOpset", MemoryAccess); + Brgemm(const Output& A, const Output& B, + const size_t offset_a = 0lu, const size_t offset_b = 0lu, const size_t offset_c = 0lu); Brgemm() = default; - bool visit_attributes(AttributeVisitor& visitor) override; + size_t get_offset_a() const { return get_input_offset(0); } + size_t get_offset_b() const { return get_input_offset(1); } + size_t get_offset_c() const { return get_output_offset(0); } + void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; bool has_evaluate() const override { return false; } - size_t get_offset_a() const { return m_offset_a; } - size_t get_offset_b() const { return m_offset_b; } - size_t get_offset_c() const { return m_offset_c; } - - void set_offset_a(const size_t offset) { m_offset_a = offset; } - void set_offset_b(const size_t offset) { m_offset_b = offset; } - void set_offset_c(const size_t offset) { m_offset_c = offset; } - -private: - size_t m_offset_a = 0lu; // offset for first input - size_t m_offset_b = 0lu; // offset for second input - size_t m_offset_c = 0lu; // offset for output +protected: + ov::element::Type get_output_type() const; + ov::PartialShape get_output_partial_shape(const std::vector& input_shapes) const; }; } // namespace op diff --git a/src/common/snippets/include/snippets/op/broadcastload.hpp b/src/common/snippets/include/snippets/op/broadcastload.hpp index 43f3a329adcf27..edcbe170a371f6 100644 --- a/src/common/snippets/include/snippets/op/broadcastload.hpp +++ b/src/common/snippets/include/snippets/op/broadcastload.hpp @@ -4,7 +4,7 @@ #pragma once -#include +#include #include "ngraph/op/op.hpp" @@ -17,22 +17,21 @@ namespace op { * @brief Is generated for broadcasting by least varying dimension for non-blocked cases and the second varying dimension for blocked * @ingroup snippets */ -class BroadcastLoad : public BroadcastMove { +class BroadcastLoad : public MemoryAccess { public: - OPENVINO_OP("BroadcastLoad", "SnippetsOpset", ngraph::snippets::op::BroadcastMove); + OPENVINO_OP("BroadcastLoad", "SnippetsOpset", ngraph::snippets::op::MemoryAccess); BroadcastLoad(const Output& x, ov::PartialShape output_shape, size_t offset = 0lu); BroadcastLoad() = default; - size_t get_offset() const { return m_offset; } - void set_offset(const size_t offset) { m_offset = offset; } + size_t get_offset() const { return get_input_offset(0); } bool visit_attributes(AttributeVisitor& visitor) override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; void validate_and_infer_types() override; private: - size_t m_offset = 0lu; + ov::PartialShape output_shape; }; } // namespace op diff --git a/src/common/snippets/include/snippets/op/buffer.hpp b/src/common/snippets/include/snippets/op/buffer.hpp index f75fc95e742edb..8c6f98ac894e93 100644 --- a/src/common/snippets/include/snippets/op/buffer.hpp +++ b/src/common/snippets/include/snippets/op/buffer.hpp @@ -12,10 +12,9 @@ namespace op { /** * @interface Buffer - * @brief The operation is for intermediate data storage - * - m_allocation_rank - rank of shape for memory allocation: shape[shape_rank - normalize(m_allocation_rank) : shape_rank]. - * It's needed to allocate needed memory size that depends on Tile rank, for example. - * Default value is -1 (full shape) + * @brief This is a base class for memory storage. + * If Buffer has a parent, the operation is for intermediate data storage - IntermediateMemory type. + * Otherwise, the operation is for allocation of new empty memory with shape `m_shape` - NewMemory type * Notes: * - All buffers in a graph have the same memory pointer. So if we have a few buffers, * each the corresponding MemoryAccess op for Buffer should have offset for common memory pointer of this Buffer @@ -25,21 +24,30 @@ namespace op { class Buffer : public ngraph::op::Op { public: OPENVINO_OP("Buffer", "SnippetsOpset"); - - Buffer(const Output& x, const int32_t allocation_rank = -1); Buffer() = default; + Buffer(const ov::Shape& shape); + Buffer(const ov::Output& arg, const ov::Shape& shape); + Buffer(const ov::Output& arg, int32_t allocation_rank = -1); + + bool visit_attributes(AttributeVisitor& visitor) override; + void validate_and_infer_types() override; + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - int32_t get_allocation_rank() const { return m_allocation_rank; } - void set_allocation_rank(int32_t rank) { m_allocation_rank = rank; } + enum Type { + NewMemory, + IntermediateMemory + }; + Type get_type() const { return m_type; } + ov::Shape get_allocation_shape() const { return m_shape; } size_t get_byte_size() const; - bool visit_attributes(AttributeVisitor& visitor) override; - std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - void validate_and_infer_types() override; + bool is_intermediate_memory() const { return m_type == Type::IntermediateMemory; } + bool is_new_memory() const { return m_type == Type::NewMemory; } private: - int32_t m_allocation_rank = -1; + Type m_type = Type::IntermediateMemory; + ov::Shape m_shape = {}; }; } // namespace op diff --git a/src/common/snippets/include/snippets/op/load.hpp b/src/common/snippets/include/snippets/op/load.hpp index bd0a4c5463f560..38acd0e8a10255 100644 --- a/src/common/snippets/include/snippets/op/load.hpp +++ b/src/common/snippets/include/snippets/op/load.hpp @@ -20,11 +20,18 @@ namespace op { */ class Load : public MemoryAccess { public: - OPENVINO_OP("Load", "SnippetsOpset"); + OPENVINO_OP("Load", "SnippetsOpset", MemoryAccess); Load(const Output& x, const size_t count = 1lu, const size_t offset = 0lu); Load() = default; + size_t get_offset() const { return get_input_offset(0); } + size_t get_count() const { return get_input_count(0); } + + void set_offset(size_t offset) { set_input_offset(offset, 0); } + void set_count(size_t count) { set_input_count(count, 0); } + + void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; }; @@ -41,6 +48,9 @@ class LoadReshape : public Load { LoadReshape(const Output& x, size_t count = 1lu, const size_t offset = 0lu, std::vector order = {}); LoadReshape() = default; + void set_offset(size_t offset) { set_output_offset(offset, 0); } + void set_count(size_t count) { set_output_count(count, 0); } + bool visit_attributes(AttributeVisitor& visitor) override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; void validate_and_infer_types() override; diff --git a/src/common/snippets/include/snippets/op/memory_access.hpp b/src/common/snippets/include/snippets/op/memory_access.hpp index f1b2d8ebb2f00d..7b090c8f65d528 100644 --- a/src/common/snippets/include/snippets/op/memory_access.hpp +++ b/src/common/snippets/include/snippets/op/memory_access.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2022 Intel Corporation +// Copyright (C) 2018-2023 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -13,9 +13,9 @@ namespace op { /** * @interface MemoryAccess * @brief This is a base class for memory access operations (like Load and Store). - * It provides universal set/get interface to manipulate the number - * of elements accessed during one operation call ("count"). - * Default "count" value is "1" - it means to load/store one element + * It provides universal interface to manipulate with memory: load/store. + * @param m_input_ports - vector of input descriptors: variables of PortDescriptor class + * @param m_output_ports - vector of output descriptors: variables of PortDescriptor class * @ingroup snippets */ @@ -23,18 +23,54 @@ class MemoryAccess : public ngraph::op::Op { public: OPENVINO_OP("MemoryAccess", "SnippetsOpset"); - size_t get_count() const; - size_t get_offset() const; - void set_count(const size_t count); - void set_offset(const size_t offset); + /** + * @interface PortDescriptor + * @brief This class describes port of MemoryAccess operation + * @param m_count - count of elements to load/store + * @param m_offset - starting index of elements to load/store + * @param m_index - port index + * @ingroup snippets + */ + struct PortDescriptor { + PortDescriptor(size_t count, size_t offset) : count(count), offset(offset) {} + PortDescriptor() = default; + + size_t count = 0lu; + size_t offset = 0lu; + size_t index = 0lu; + + private: + PortDescriptor(size_t count, size_t offset, size_t index) : count(count), offset(offset), index(index) {} + + friend class MemoryAccess; + }; + + void set_input_count(size_t count, size_t idx = 0); + void set_output_count(size_t count, size_t idx = 0); + void set_input_offset(size_t offset, size_t idx = 0); + void set_output_offset(size_t offset, size_t idx = 0); + + size_t get_input_count(size_t idx = 0) const; + size_t get_output_count(size_t idx = 0) const; + size_t get_input_offset(size_t idx = 0) const; + size_t get_output_offset(size_t idx = 0) const; + + size_t get_input_port_count() const { return m_input_ports.size(); } + size_t get_output_port_count() const { return m_output_ports.size(); } + bool visit_attributes(AttributeVisitor& visitor) override; - void validate_and_infer_types() override; protected: - explicit MemoryAccess(const Output& x, size_t count = 1lu, size_t offset = 0lu); + explicit MemoryAccess(const OutputVector& arguments, size_t input_count = 0, size_t output_count = 0); MemoryAccess() = default; - size_t m_count = 0lu; - size_t m_offset = 0lu; + + void set_input_port_descriptor(const PortDescriptor& desc, const size_t i); + void set_output_port_descriptor(const PortDescriptor& desc, const size_t i); + const PortDescriptor& get_input_port_descriptor(const size_t i) const; + const PortDescriptor& get_output_port_descriptor(const size_t i) const; + + std::vector m_input_ports; + std::vector m_output_ports; }; } // namespace op diff --git a/src/common/snippets/include/snippets/op/store.hpp b/src/common/snippets/include/snippets/op/store.hpp index 38715cffc6c74c..b62a4c6ccb18b7 100644 --- a/src/common/snippets/include/snippets/op/store.hpp +++ b/src/common/snippets/include/snippets/op/store.hpp @@ -20,11 +20,18 @@ namespace op { */ class Store : public MemoryAccess { public: - OPENVINO_OP("Store", "SnippetsOpset"); + OPENVINO_OP("Store", "SnippetsOpset", MemoryAccess); Store(const Output& x, const size_t count = 1lu, const size_t offset = 0lu); Store() = default; + size_t get_offset() const { return get_output_offset(0); } + size_t get_count() const { return get_output_count(0); } + + void set_offset(size_t offset) { set_output_offset(offset, 0); } + void set_count(size_t count) { set_output_count(count, 0); } + + void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; }; diff --git a/src/common/snippets/include/snippets/op/subgraph.hpp b/src/common/snippets/include/snippets/op/subgraph.hpp index ec55f076301c64..46e6633f61b8aa 100644 --- a/src/common/snippets/include/snippets/op/subgraph.hpp +++ b/src/common/snippets/include/snippets/op/subgraph.hpp @@ -101,11 +101,17 @@ class Subgraph : public ov::op::util::SubGraphOp { bool is_quantized() const { return config.m_is_quantized; } bool has_type_relaxed_ops() const { return config.m_has_type_relaxed_ops; } bool has_domain_sensitive_ops() const { return config.m_has_domain_sensitive_ops; } - - snippets::Schedule generate(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes, ngraph::pass::Manager& opt, + snippets::Schedule generate(const BlockedShapeVector& output_shapes, + const BlockedShapeVector& input_shapes, + ngraph::pass::Manager& pre_dialect, + ngraph::pass::Manager& post_dialect, + ngraph::pass::Manager& post_precision, const void* compile_params = nullptr); snippets::Schedule generate(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes, const void* compile_params = nullptr); - snippets::Schedule generate(ngraph::pass::Manager &opt, const void* compile_params = nullptr); + snippets::Schedule generate(ngraph::pass::Manager& pre_dialect, + ngraph::pass::Manager& post_dialect, + ngraph::pass::Manager& post_precision, + const void* compile_params = nullptr); snippets::Schedule generate(const void* compile_params = nullptr); ov::PartialShape canonicalize(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes); std::vector reshape_body(const std::vector& input_shapes); @@ -132,6 +138,8 @@ class Subgraph : public ov::op::util::SubGraphOp { // This check returns True if Constant op which is input of this op should be inside Subgraph body static auto constant_input_should_be_inside_body(const std::shared_ptr& node) -> bool; + static bool check_broadcast(const std::shared_ptr& node) noexcept; + private: void align_element_types(const BlockedShapeVector& outputShapes, const BlockedShapeVector& inputShapes); void convert_to_snippet_dialect(); @@ -164,8 +172,6 @@ class Subgraph : public ov::op::util::SubGraphOp { public: // True if Subgraph contains FakeQuantize -> FQ decomposition should be called bool m_is_quantized = false; - // True if we should align element types indise body - bool m_is_needed_to_align_precision = false; // True if Subgraph contains TypeRelaxed nodes -> for several streams in tp mode we should copy body using mutexes // because TypeRelaxed::copy_with_new_inputs() isn't save-thread method bool m_has_type_relaxed_ops = false; diff --git a/src/common/snippets/include/snippets/pass/align_element_type.hpp b/src/common/snippets/include/snippets/pass/align_element_type.hpp deleted file mode 100644 index 0b1f831091c4cc..00000000000000 --- a/src/common/snippets/include/snippets/pass/align_element_type.hpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include - -namespace ngraph { -namespace snippets { -namespace pass { - -/** - * @interface AlignElementType - * @brief Wrap sequence of operations which doesn't support execution on original element type by ConvertSaturation - * and reset element type for type relaxed nodes inside body to align element type between nodes. - * Example 1: - * - After FQ decomposition there may be Convert[U8/I8]. If after the Convert there are other operations - * that don't support U8/I8, new ConvertSaturation[exec_type] will be inserted after the FQ decomposition - * to execute these operations on supported element type - * Example 2: - * - Input[I8] -> Unsupported I8 op -> Movement op -> Output[I8]. There will be inserted two ConvertSaturation: - * * ConvertSatiration[exec_type] before op which is unsupported I8 - * * ConvertSaturation[I8] before Movement op to return original low precision. - * Note: We cannot just remove original Convert[I8/U8] in Example 1 because we should cover two things: - * * allow execution of operations on supported element type for them - * * keep computations mathematically equivalent to the original function - * Thus, for these cases we should have the following pipeline: FP32 -> Convert[I8/U8] -> Convert[FP32] -> FP32 - * Note: We shouldn't call validate_and_infer_type() after Convert insertions to avoid element type conflicts on inputs of ops - * @ingroup snippets - */ -class AlignElementType: public ngraph::pass::FunctionPass { -public: - OPENVINO_RTTI("AlignElementType", "0"); - AlignElementType(const ov::element::Type exec_type = ov::element::f32); - bool run_on_model(const std::shared_ptr& m) override; - - static bool opNeedsAlignElementType(const std::shared_ptr& n, const ov::element::Type exec_type = ov::element::f32); -private: - ov::element::Type exec_type; -}; - -} // namespace pass -} // namespace snippets -} // namespace ngraph diff --git a/src/common/snippets/include/snippets/pass/assign_registers.hpp b/src/common/snippets/include/snippets/pass/assign_registers.hpp index 144a8678784451..81a5e3b2b29d62 100644 --- a/src/common/snippets/include/snippets/pass/assign_registers.hpp +++ b/src/common/snippets/include/snippets/pass/assign_registers.hpp @@ -6,6 +6,8 @@ #include +#include "snippets/generator.hpp" + namespace ngraph { namespace snippets { namespace pass { @@ -18,10 +20,13 @@ namespace pass { */ class AssignRegisters : public ngraph::pass::FunctionPass { public: - explicit AssignRegisters() { + explicit AssignRegisters(const std::function& op)>& mapper) : m_reg_type_mapper(mapper) { set_property(ngraph::pass::PassProperty::REQUIRE_STATIC_SHAPE, true); } bool run_on_model(const std::shared_ptr& m) override; + +private: + std::function& op)> m_reg_type_mapper; }; } // namespace pass diff --git a/src/common/snippets/include/snippets/pass/fq_decomposition.hpp b/src/common/snippets/include/snippets/pass/fq_decomposition.hpp index 284640d8c18122..cfb9ff41955867 100644 --- a/src/common/snippets/include/snippets/pass/fq_decomposition.hpp +++ b/src/common/snippets/include/snippets/pass/fq_decomposition.hpp @@ -29,7 +29,7 @@ namespace pass { * * Expand brackets: * round(x * (levels-1) / (ih - il) - il * (levels-1) / (ih - il)) * (oh - ol) / (levels-1) + ol - * + * * Marking: * - isc := (levels-1) / (ih - il) * - ish := -il * isc @@ -37,7 +37,7 @@ namespace pass { * - osh := ol * Final expression: * round(x * isc + ish) * osc + osh - * + * * Some optimizations (example for scalars): * 1. If output element type of FQ is U8 and il = 0, ish = 0, osc = 1, osh = 0, there is enough expression: x * isc * 2. If output element type of FQ is I8 and ish ~= 128, osc = 1, osh ~= -128, il * isc ~= -128, ih * isc ~= 127 there is enough expression: x * isc @@ -54,7 +54,6 @@ class FakeQuantizeDecomposition : public ngraph::pass::MatcherPass { public: FakeQuantizeDecomposition(); - static bool isAllScalarConstant(const std::shared_ptr& node); static bool getScalesAndShifts(const std::shared_ptr& fq_node, std::vector& cl, std::vector& ch, diff --git a/src/common/snippets/include/snippets/pass/propagate_precision.hpp b/src/common/snippets/include/snippets/pass/propagate_precision.hpp new file mode 100644 index 00000000000000..d0920766f632fd --- /dev/null +++ b/src/common/snippets/include/snippets/pass/propagate_precision.hpp @@ -0,0 +1,48 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include "snippets/generator.hpp" + +namespace ngraph { +namespace snippets { +namespace pass { + +/** + * @class PropagatePrecision + * @ingroup snippets + * @brief PropagatePrecision transformation propagate precision from parameters to results. + */ +class PropagatePrecision: public ngraph::pass::FunctionPass { +public: + OPENVINO_RTTI("PropagatePrecision", "0"); + PropagatePrecision(const std::shared_ptr& target_machine); + bool run_on_model(const std::shared_ptr& m) override; + + static std::vector get_precisions( + const std::vector& input_precisions, + const std::set>& supported_precisions) noexcept; + + // if can_be_removed returns true then actual convertion (actual_before => actual_after) + // can be replaced to required (actual_before => required_after) + static bool can_be_removed( + const element::Type& actual_before, + const element::Type& actual_after, + const element::Type& required_after) noexcept; + + // if can_be_fused returns true then actual convertion can be replaced to required + static bool can_be_fused( + const element::Type& actual, + const element::Type& required) noexcept; + +private: + const std::shared_ptr target_machine; +}; + +} // namespace pass +} // namespace snippets +} // namespace ngraph diff --git a/src/common/snippets/include/snippets/utils.hpp b/src/common/snippets/include/snippets/utils.hpp index 253785b516dda5..3325ff42446594 100644 --- a/src/common/snippets/include/snippets/utils.hpp +++ b/src/common/snippets/include/snippets/utils.hpp @@ -29,10 +29,31 @@ ov::PartialShape get_port_planar_shape(const Output& out); ov::PartialShape get_reordered_planar_shape(const ov::PartialShape& shape, const std::vector& layout); std::vector get_node_output_layout(const std::shared_ptr& node); std::vector get_node_output_layout(const Node* node); +void set_transpose_output_layout(const ov::Output& port, const std::shared_ptr& node); +void set_output_layout(const ov::Output& port, const std::vector& layout); inline ov::Dimension get_inner_dim(const ov::PartialShape &shape) { return *(shape.rbegin()); } inline ov::Dimension get_outer_dim(const ov::PartialShape &shape) { return *(shape.rbegin() + 1); } +inline auto normalize_rank(int32_t allocation_rank, const size_t shape_rank) -> int32_t { + return allocation_rank < 0 ? allocation_rank + static_cast(shape_rank) + 1 : allocation_rank; +} + +template +constexpr bool one_of(T val, P item) { return val == item; } + +template +constexpr bool one_of(T val, P item, Args... item_others) { + return val == item || one_of(val, item_others...); +} + +template +constexpr bool everyone_is(T val, P item) { return val == item; } + +template +constexpr bool everyone_is(T val, P item, Args... item_others) { + return val == item && everyone_is(val, item_others...); +} } // namespace utils } // namespace snippets } // namespace ngraph diff --git a/src/common/snippets/src/generator.cpp b/src/common/snippets/src/generator.cpp index 5ff9b9e19e27d0..dba0f139fda495 100644 --- a/src/common/snippets/src/generator.cpp +++ b/src/common/snippets/src/generator.cpp @@ -77,8 +77,15 @@ auto tail_transformations(NodeVector& tail, const size_t tail_size, const ngraph } } } else if (const auto memory_access = std::dynamic_pointer_cast(op)) { - if (memory_access->get_count() != 1) { - memory_access->set_count(tail_size); + for (size_t i = 0; i < memory_access->get_input_port_count(); ++i) { + if (memory_access->get_input_count(i) > 1) { + memory_access->set_input_count(tail_size, i); + } + } + for (size_t i = 0; i < memory_access->get_output_port_count(); ++i) { + if (memory_access->get_output_count(i) > 1) { + memory_access->set_output_count(tail_size, i); + } } } updated_tile.push_back(op); @@ -220,5 +227,41 @@ std::shared_ptr Generator::get_target_machine() const { return target; } +Generator::opRegType Generator::get_op_reg_type(const std::shared_ptr& op) const { + if (std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op)) + return gpr2gpr; + else if (std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op)) + return gpr2vec; + else if (std::dynamic_pointer_cast(op)) + return vec2gpr; + else if (ov::op::util::is_unary_elementwise_arithmetic(op) || + ov::op::util::is_binary_elementwise_arithmetic(op) || + ov::op::util::is_binary_elementwise_comparison(op) || + ov::op::util::is_binary_elementwise_logical(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op) || + std::dynamic_pointer_cast(op)) + return vec2vec; + else + return get_specific_op_reg_type(op); +} + +Generator::opRegType Generator::get_specific_op_reg_type(const std::shared_ptr& op) const { + throw ov::Exception("Register type of the operation " + std::string(op->get_type_name()) + " isn't determined!"); +} + + }// namespace snippets }// namespace ngraph diff --git a/src/common/snippets/src/op/brgemm.cpp b/src/common/snippets/src/op/brgemm.cpp index 7bf999cb15e423..743653099b8601 100644 --- a/src/common/snippets/src/op/brgemm.cpp +++ b/src/common/snippets/src/op/brgemm.cpp @@ -7,56 +7,123 @@ #include "ngraph/runtime/host_tensor.hpp" #include "openvino/core/rt_info.hpp" #include "snippets/utils.hpp" -#include "matmul_shape_inference.hpp" namespace ngraph { namespace snippets { namespace op { -Brgemm::Brgemm(const Output& A, const Output& B, const size_t offset_a, const size_t offset_b, const size_t offset_c) - : MatMul(), m_offset_a(offset_a), m_offset_b(offset_b), m_offset_c(offset_c) { - set_arguments({A, B}); +Brgemm::Brgemm(const Output& A, const Output& B, + const size_t offset_a, const size_t offset_b, const size_t offset_c) : MemoryAccess({A, B}, 2, 1) { set_output_size(1); + set_input_offset(offset_a, 0); + set_input_offset(offset_b, 1); + set_output_offset(offset_a, 0); constructor_validate_and_infer_types(); } -bool Brgemm::visit_attributes(AttributeVisitor& visitor) { - MatMul::visit_attributes(visitor); - visitor.on_attribute("offset_a", m_offset_a); - visitor.on_attribute("offset_b", m_offset_b); - visitor.on_attribute("offset_c", m_offset_c); - return true; -} - void Brgemm::validate_and_infer_types() { INTERNAL_OP_SCOPE(Brgemm_validate_and_infer_types); - element::Type result_et; - NODE_VALIDATION_CHECK(this, - element::Type::merge(result_et, get_input_element_type(0), get_input_element_type(1)), - "Arguments do not have the same element type (arg0 element type: ", - get_input_element_type(0), - ", arg1 element type: ", - get_input_element_type(1), - ")."); // If no leading dimensions are provided, assume dense row-major inputs-outputs NODE_VALIDATION_CHECK(this, get_input_partial_shape(0).is_static() && get_input_partial_shape(1).is_static(), "Brgemm currently supports only static shapes."); - std::vector planar_input_shapes; - for (const auto& in : input_values()) - planar_input_shapes.emplace_back(utils::get_port_planar_shape(in)); + std::vector planar_input_shapes = { + utils::get_port_planar_shape(input_value(0)), + utils::get_port_planar_shape(input_value(1)) + }; - std::vector output_shapes = {ov::PartialShape{}}; - ov::op::v0::shape_infer(this, planar_input_shapes, output_shapes); + auto output_shape = get_output_partial_shape(planar_input_shapes); const auto& output_layout = utils::get_node_output_layout(this); - output_shapes[0] = utils::get_reordered_planar_shape(output_shapes[0], output_layout); - set_output_type(0, result_et, output_shapes[0]); + set_output_type(0, + get_output_type(), + utils::get_reordered_planar_shape(output_shape, output_layout)); } std::shared_ptr Brgemm::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(Brgemm_clone_with_new_inputs); check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), new_args.at(1), m_offset_a, m_offset_b, m_offset_c); + return std::make_shared(new_args.at(0), new_args.at(1), get_offset_a(), get_offset_b(), get_offset_c()); +} + +ov::element::Type Brgemm::get_output_type() const { + const auto element_type_a = get_input_element_type(0); + const auto element_type_b = get_input_element_type(1); + const bool is_f32 = utils::everyone_is(element::f32, element_type_a, element_type_b); + const bool is_int8 = utils::one_of(element_type_a, element::i8, element::u8) && element_type_b == element::i8; + const bool is_bf16 = utils::everyone_is(element::bf16, element_type_a, element_type_b); + if (is_f32 || is_bf16) { + return element::f32; + } else if (is_int8) { + return element::i32; + } else { + throw ngraph_error("BrgemmCPU node has incompatible input element types: " + + element_type_a.get_type_name() + + " and " + + element_type_b.get_type_name()); + } +} + +ov::PartialShape Brgemm::get_output_partial_shape(const std::vector& input_shapes) const { + NGRAPH_CHECK(input_shapes.size() == 2, "BRGEMM expects 2 input shapes for shape inference"); + + // Note: All majors checks are missed because Brgemm is transformed from MatMul with whole shape infer support + + const auto arg0_shape = input_shapes[0]; + const auto arg1_shape = input_shapes[1]; + + size_t arg0_rank = arg0_shape.size(), arg1_rank = arg1_shape.size(); + + // temporary shapes to calculate output shape + ov::PartialShape arg0_shape_tmp(arg0_shape), arg1_shape_tmp(arg1_shape); + + // one-dimensional tensors unsqueezing is applied to each input independently. + if (arg0_rank == 1) { + // If the first input is 1D tensor, it is unsqueezed to 2D tensor (row vector) + // by adding axes with size 1 at ROW_INDEX_DIM, to the left of the shape. + // For example {S} will be reshaped to {1, S}. + arg0_shape_tmp.insert(arg0_shape_tmp.begin(), 1); + arg0_rank = arg0_shape_tmp.size(); + } + if (arg1_rank == 1) { + // If the second input is 1D tensor, it is unsqueezed to 2D tensor (column vector) + // by adding axes with size 1 at COL_INDEX_DIM, to the right of the shape. + // For example {S} will be reshaped to {S, 1}. + arg1_shape_tmp.insert(arg1_shape_tmp.end(), 1); + arg1_rank = arg1_shape_tmp.size(); + } + // Check matrices dimensions compatibility, + using DimType = typename std::iterator_traits::value_type; + auto merged_dimension = DimType(); + auto arg0_col_dim = arg0_shape_tmp[arg0_rank - 1]; + auto arg1_row_dim = arg1_shape_tmp[arg1_rank - 2]; + OPENVINO_ASSERT(DimType::merge(merged_dimension, arg0_col_dim, arg1_row_dim) || arg0_col_dim.is_dynamic() || arg1_row_dim.is_dynamic(), + "Incompatible Brgemm matrix dimension"); + + // add 1 to begin to align shape ranks if needed + if (arg0_rank < arg1_rank) + arg0_shape_tmp.insert(arg0_shape_tmp.begin(), arg1_rank - arg0_rank, 1); + else if (arg0_rank > arg1_rank) + arg1_shape_tmp.insert(arg1_shape_tmp.begin(), arg0_rank - arg1_rank, 1); + + size_t max_rank = arg0_shape_tmp.size(); + std::vector output_shape(max_rank); + for (size_t i = 0; i < max_rank - 2; ++i) { + OPENVINO_ASSERT(DimType::broadcast_merge(output_shape[i], arg0_shape_tmp[i], arg1_shape_tmp[i]) || + arg0_shape_tmp[i].is_dynamic() || + arg1_shape_tmp[i].is_dynamic(), + "Incompatible Brgemm batch dimension"); + } + output_shape[output_shape.size() - 2] = arg0_shape_tmp[arg0_shape_tmp.size() - 2]; // M + output_shape[output_shape.size() - 1] = arg1_shape_tmp[arg1_shape_tmp.size() - 1]; // N + + // removing the temporary axes from originally 1D tensors. + if (arg0_shape.rank().get_length() == 1) { + output_shape.erase(output_shape.begin() + output_shape.size() - 2); + } + if (arg1_shape.rank().get_length() == 1) { + output_shape.erase(output_shape.begin() + output_shape.size() - 1); + } + return output_shape; } } // namespace op diff --git a/src/common/snippets/src/op/broadcastload.cpp b/src/common/snippets/src/op/broadcastload.cpp index 0f4e6c7667e2d1..ccbb5f9b9af9a7 100644 --- a/src/common/snippets/src/op/broadcastload.cpp +++ b/src/common/snippets/src/op/broadcastload.cpp @@ -12,20 +12,20 @@ using namespace std; using namespace ngraph; snippets::op::BroadcastLoad::BroadcastLoad(const Output& x, ov::PartialShape shape, size_t offset) - : BroadcastMove(x, std::move(shape)), m_offset(offset) { + : MemoryAccess({x}, 1, 0), output_shape(std::move(shape)) { + set_input_port_descriptor({1, offset}, 0); constructor_validate_and_infer_types(); } bool snippets::op::BroadcastLoad::visit_attributes(AttributeVisitor& visitor) { - BroadcastMove::visit_attributes(visitor); - visitor.on_attribute("offset", m_offset); + MemoryAccess::visit_attributes(visitor); return true; } std::shared_ptr snippets::op::BroadcastLoad::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(BroadcastLoad); check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), output_shape, m_offset); + return std::make_shared(new_args.at(0), output_shape, get_offset()); } void snippets::op::BroadcastLoad::validate_and_infer_types() { diff --git a/src/common/snippets/src/op/buffer.cpp b/src/common/snippets/src/op/buffer.cpp index ad05ae2e046932..8a3963119b832b 100644 --- a/src/common/snippets/src/op/buffer.cpp +++ b/src/common/snippets/src/op/buffer.cpp @@ -6,8 +6,8 @@ #include "snippets/op/buffer.hpp" #include "snippets/snippets_isa.hpp" +#include "snippets/utils.hpp" -#include using namespace std; using namespace ngraph; @@ -16,38 +16,64 @@ auto normalize_rank(int32_t allocation_rank, const size_t shape_rank) -> int32_t return allocation_rank < 0 ? allocation_rank + static_cast(shape_rank) : allocation_rank; } -snippets::op::Buffer::Buffer(const Output& x, const int32_t allocation_rank) : Op({x}), m_allocation_rank(allocation_rank) { +snippets::op::Buffer::Buffer(const ov::Shape& shape) + : Op(), m_type(Type::NewMemory), m_shape(shape) { + constructor_validate_and_infer_types(); +} + +snippets::op::Buffer::Buffer(const ov::Output& arg, const ov::Shape& shape) + : Op({arg}), m_type(Type::IntermediateMemory), m_shape(shape) { + constructor_validate_and_infer_types(); +} + +snippets::op::Buffer::Buffer(const ov::Output& arg, int32_t allocation_rank) + : Op({arg}), m_type(Type::IntermediateMemory) { + const auto pshape = arg.get_partial_shape(); + OPENVINO_ASSERT(pshape.is_static(), "Buffer supports only static input shape"); + const auto shape = pshape.get_shape(); + const auto normalize_rank = utils::normalize_rank(static_cast(allocation_rank), shape.size()); + const auto offset = static_cast(shape.size()) - normalize_rank; + m_shape = {shape.begin() + offset, shape.end()}; constructor_validate_and_infer_types(); } bool snippets::op::Buffer::visit_attributes(AttributeVisitor& visitor) { INTERNAL_OP_SCOPE(Buffer_visit_attributes); - visitor.on_attribute("allocation_rank", m_allocation_rank); + visitor.on_attribute("allocation_shape", m_shape); return true; } +void snippets::op::Buffer::validate_and_infer_types() { + INTERNAL_OP_SCOPE(Buffer_validate_and_infer_types); + ov::element::Type output_type; + ov::Shape output_shape; + if (m_type == Type::NewMemory) { + OPENVINO_ASSERT(get_input_size() == 0, "Buffer with new allocated memory must to not have arguments!"); + output_shape = m_shape; + output_type = ov::element::u8; // 1Byte + } else if (m_type == Type::IntermediateMemory) { + const auto input_shape = get_input_partial_shape(0); + OPENVINO_ASSERT(input_shape.is_static(), "Buffer supports only static input shape"); + output_type = get_input_element_type(0); + output_shape = input_shape.get_shape(); + } else { + throw ov::Exception("Buffer supports only the following types: NewMemory and IntermediateMemory"); + } + set_output_type(0, output_type, output_shape); +} + std::shared_ptr snippets::op::Buffer::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(Buffer_clone_with_new_inputs); check_new_args_count(this, new_args); - auto new_buffer = std::make_shared(new_args.at(0), m_allocation_rank); - return new_buffer; -} - -void snippets::op::Buffer::validate_and_infer_types() { - INTERNAL_OP_SCOPE(Buffer_validate_and_infer_types); - const auto shape_rank = get_input_partial_shape(0).rank(); - if (shape_rank.is_static()) { - const auto normalized_rank = normalize_rank(m_allocation_rank, shape_rank.get_length()); - NGRAPH_CHECK(normalized_rank >= 0 && normalized_rank <= shape_rank.get_length(), - "Buffer has incorrect allocation rank: " + std::to_string(m_allocation_rank)); + if (m_type == Type::NewMemory) { + return std::make_shared(m_shape); + } else if (m_type == Type::IntermediateMemory) { + return std::make_shared(new_args.at(0), m_shape); } - set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); + throw ov::Exception("Buffer supports only the following types: NewMemory and IntermediateMemory"); } size_t ngraph::snippets::op::Buffer::get_byte_size() const { - const auto pshape = get_input_partial_shape(0); - NGRAPH_CHECK(pshape.is_static(), "Buffer should have static shapes for memory allocation"); - const auto shape = pshape.get_shape(); - const auto normalized_rank = normalize_rank(m_allocation_rank, shape.size()); - return ngraph::shape_size(shape.rbegin(), shape.rbegin() + normalized_rank) * get_element_type().size(); + const auto shape = get_allocation_shape(); + return ngraph::shape_size(shape) * get_element_type().size(); } diff --git a/src/common/snippets/src/op/load.cpp b/src/common/snippets/src/op/load.cpp index 8ee227c7afb69e..f1f5bc42c7a3da 100644 --- a/src/common/snippets/src/op/load.cpp +++ b/src/common/snippets/src/op/load.cpp @@ -12,17 +12,24 @@ namespace ngraph { namespace snippets { namespace op { -Load::Load(const Output& x, const size_t count, const size_t offset) : MemoryAccess({x}, count, offset) { +Load::Load(const Output& x, const size_t count, const size_t offset) : MemoryAccess({x}, 1, 0) { + set_input_port_descriptor({count, offset}, 0); constructor_validate_and_infer_types(); } +void snippets::op::Load::validate_and_infer_types() { + // Load has memory access port only on output + OPENVINO_ASSERT(get_input_port_count() == 1, "Load node must have memory access input port"); + OPENVINO_ASSERT(get_output_port_count() == 0, "Load node mustn't have memory access output port"); + set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); +} + std::shared_ptr Load::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(Load); check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), m_count, m_offset); + return std::make_shared(new_args.at(0), get_count(), get_offset()); } - LoadReshape::LoadReshape(const Output& x, const size_t count, const size_t offset, std::vector order) : Load(x, count, offset), m_order(std::move(order)) { const auto& in_shape = x.get_partial_shape(); @@ -33,6 +40,8 @@ LoadReshape::LoadReshape(const Output& x, const size_t count, const si *std::min_element(m_order.begin(), m_order.end()) == 0, "LoadReshape detected invalid values in new_order"); const std::set unique_dims(order.begin(), order.end()); NGRAPH_CHECK(unique_dims.size() == order.size(), "LoadReshape order must not contain repeated elements"); + m_input_ports.resize(get_input_size()); + set_input_port_descriptor({count, offset}, 0); constructor_validate_and_infer_types(); } @@ -53,7 +62,7 @@ bool snippets::op::LoadReshape::visit_attributes(AttributeVisitor& visitor) { std::shared_ptr snippets::op::LoadReshape::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(LoadReshape); check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), m_count, m_offset, m_order); + return std::make_shared(new_args.at(0), get_count(), get_offset(), m_order); } }// namespace op diff --git a/src/common/snippets/src/op/memory_access.cpp b/src/common/snippets/src/op/memory_access.cpp index 2530ea77b6352b..ea0e4649f9e5de 100644 --- a/src/common/snippets/src/op/memory_access.cpp +++ b/src/common/snippets/src/op/memory_access.cpp @@ -3,43 +3,80 @@ // #include - #include "snippets/op/memory_access.hpp" -#include - namespace ngraph { namespace snippets { namespace op { -MemoryAccess::MemoryAccess(const Output& x, const size_t count, const size_t offset) : Op({x}), m_count(count), m_offset(offset) {} +MemoryAccess::MemoryAccess(const OutputVector& arguments, size_t input_count, size_t output_count) : Op(arguments) { + while (m_input_ports.size() < input_count) { + m_input_ports.push_back({0, 0, m_input_ports.size()}); + } + while (m_output_ports.size() < output_count) { + m_output_ports.push_back({0, 0, m_output_ports.size()}); + } +} bool MemoryAccess::visit_attributes(AttributeVisitor& visitor) { - visitor.on_attribute("count", m_count); - visitor.on_attribute("offset", m_offset); + for (size_t i = 0; i < m_input_ports.size(); ++i) { + auto port = m_input_ports[i]; + visitor.on_attribute("count_in_" + std::to_string(i), port.count); + visitor.on_attribute("offset_in_" + std::to_string(i), port.offset); + } + for (size_t i = 0; i < m_output_ports.size(); ++i) { + auto port = m_output_ports[i]; + visitor.on_attribute("count_out_" + std::to_string(i), port.count); + visitor.on_attribute("offset_out_" + std::to_string(i), port.offset); + } return true; } -size_t MemoryAccess::get_count() const { - return m_count; +void MemoryAccess::set_input_port_descriptor(const PortDescriptor& desc, const size_t i) { + NGRAPH_CHECK(i < m_input_ports.size(), "Index of input port descriptor should be less than count of input ports"); + m_input_ports[i] = { desc.count, desc.offset, i}; } -size_t MemoryAccess::get_offset() const { - return m_offset; +void MemoryAccess::set_output_port_descriptor(const PortDescriptor& desc, const size_t i) { + NGRAPH_CHECK(i < m_output_ports.size(), "Index of output port descriptor should be less than count of output ports"); + m_output_ports[i] = { desc.count, desc.offset, i}; } -void MemoryAccess::set_count(const size_t count) { - m_count = count; +const MemoryAccess::PortDescriptor& MemoryAccess::get_input_port_descriptor(const size_t i) const { + NGRAPH_CHECK(i < m_input_ports.size(), "Index of input port descriptor should be less than count of input ports"); + return m_input_ports[i]; } -void MemoryAccess::set_offset(const size_t offset) { - m_offset = offset; +const MemoryAccess::PortDescriptor& MemoryAccess::get_output_port_descriptor(const size_t i) const { + NGRAPH_CHECK(i < m_output_ports.size(), "Index of output port descriptor should be less than count of output ports"); + return m_output_ports[i]; } -void MemoryAccess::validate_and_infer_types() { - set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); +void MemoryAccess::set_input_count(size_t count, size_t idx) { + set_input_port_descriptor({count, get_input_port_descriptor(idx).offset, idx}, idx); +} +void MemoryAccess::set_output_count(size_t count, size_t idx) { + set_output_port_descriptor({count, get_output_port_descriptor(idx).offset, idx}, idx); +} +void MemoryAccess::set_input_offset(size_t offset, size_t idx) { + set_input_port_descriptor({get_input_port_descriptor(idx).count, offset, idx}, idx); +} +void MemoryAccess::set_output_offset(size_t offset, size_t idx) { + set_output_port_descriptor({get_output_port_descriptor(idx).count, offset, idx}, idx); +} +size_t MemoryAccess::get_input_count(size_t idx) const { + return get_input_port_descriptor(idx).count; +} +size_t MemoryAccess::get_output_count(size_t idx) const { + return get_output_port_descriptor(idx).count; +} +size_t MemoryAccess::get_input_offset(size_t idx) const { + return get_input_port_descriptor(idx).offset; +} +size_t MemoryAccess::get_output_offset(size_t idx) const { + return get_output_port_descriptor(idx).offset; } } // namespace op } // namespace snippets -} // namespace ngraph \ No newline at end of file +} // namespace ngraph diff --git a/src/common/snippets/src/op/store.cpp b/src/common/snippets/src/op/store.cpp index 2cee1b207517c4..8ac2c4cdf1704e 100644 --- a/src/common/snippets/src/op/store.cpp +++ b/src/common/snippets/src/op/store.cpp @@ -12,13 +12,22 @@ namespace ngraph { namespace snippets { namespace op { -snippets::op::Store::Store(const Output& x, const size_t count, const size_t offset) : MemoryAccess({x}, count, offset) { +snippets::op::Store::Store(const Output& x, const size_t count, const size_t offset) : MemoryAccess({x}, 0, 1) { + set_output_port_descriptor({count, offset}, 0); constructor_validate_and_infer_types(); } + +void snippets::op::Store::validate_and_infer_types() { + // Store has memory access port only on output + OPENVINO_ASSERT(get_input_port_count() == 0, "Store node mustn't have memory access input port"); + OPENVINO_ASSERT(get_output_port_count() == 1, "Store node must have memory access output port"); + set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); +} + std::shared_ptr snippets::op::Store::clone_with_new_inputs(const OutputVector& new_args) const { INTERNAL_OP_SCOPE(Store_clone_with_new_inputs); check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), m_count, m_offset); + return std::make_shared(new_args.at(0), get_count(), get_offset()); } } // namespace op diff --git a/src/common/snippets/src/op/subgraph.cpp b/src/common/snippets/src/op/subgraph.cpp index 07f13ae8defb57..f8953745520aff 100644 --- a/src/common/snippets/src/op/subgraph.cpp +++ b/src/common/snippets/src/op/subgraph.cpp @@ -11,6 +11,7 @@ #include "snippets/pass/insert_movebroadcast.hpp" #include "snippets/pass/broadcast_to_movebroadcast.hpp" #include "snippets/pass/load_movebroadcast_to_broadcastload.hpp" +#include "snippets/pass/propagate_precision.hpp" #include "snippets/pass/assign_registers.hpp" #include "snippets/pass/convert_constants.hpp" #include "snippets/pass/convert_power_to_powerstatic.hpp" @@ -18,7 +19,6 @@ #include "snippets/pass/insert_loops.hpp" #include "snippets/pass/transpose_decomposition.hpp" #include "snippets/pass/transform_convert.hpp" -#include "snippets/pass/align_element_type.hpp" #include "snippets/pass/matmul_to_brgemm.hpp" #include "snippets/pass/fuse_transpose_brgemm.hpp" #include "snippets/pass/softmax_decomposition.hpp" @@ -62,10 +62,6 @@ void snippets::op::Subgraph::init_config() { ov::is_type(op); config.m_has_type_relaxed_ops = config.m_has_type_relaxed_ops || std::dynamic_pointer_cast(op); - config.m_is_needed_to_align_precision = config.m_is_needed_to_align_precision || - is_quantized() || - has_type_relaxed_ops() || - snippets::pass::AlignElementType::opNeedsAlignElementType(op, execution_element_type); config.m_has_domain_sensitive_ops = config.m_has_domain_sensitive_ops || ov::is_type(op) || ov::is_type(op) || @@ -359,6 +355,14 @@ ov::PartialShape snippets::op::Subgraph::canonicalize(const BlockedShapeVector& return master_shape; } +bool snippets::op::Subgraph::check_broadcast(const std::shared_ptr& node) noexcept { + const auto elementwise = std::dynamic_pointer_cast(node); + return + (elementwise == nullptr) || + (elementwise->get_input_partial_shape(0).size() == elementwise->get_input_partial_shape(1).size()) || + (elementwise->get_autob().m_type != ov::op::AutoBroadcastType::PDPD); +} + void snippets::op::Subgraph::align_element_types(const BlockedShapeVector& outputShapes, const BlockedShapeVector& inputShapes) { // We should insert Convert before Results to set original output element type if needed @@ -369,35 +373,34 @@ void snippets::op::Subgraph::align_element_types(const BlockedShapeVector& outpu const auto convert = std::make_shared( body_results[i]->get_input_node_shared_ptr(0), needed_out_type); body_results[i]->set_argument(0, convert); + body_results[i]->validate_and_infer_types(); } } // We should change existing element type to original for Parameters if needed - const auto& body_parameters = body_ptr()->get_parameters(); + const auto& parameters = body_ptr()->get_parameters(); for (size_t i = 0; i < inputShapes.size(); ++i) { const auto needed_in_type = std::get<2>(inputShapes[i]); - if (body_parameters[i]->get_element_type() != needed_in_type) { - body_parameters[i]->set_element_type(needed_in_type); - config.m_is_needed_to_align_precision = true; - } - } + const auto& parameter = parameters[i]; + if (parameter->get_element_type() != needed_in_type) { + const auto parameter_output = parameter->output(0); + const auto convert = std::make_shared( + parameter_output, + parameter_output.get_element_type()); + ngraph::copy_runtime_info(parameter, convert); + + for (const auto input : parameter_output.get_target_inputs()) { + const auto& input_node = input.get_node(); + if (input_node == convert.get()) { + continue; + } + input_node->set_argument(input.get_index(), convert->output(0)); + } - // We should align element type inside body using the corresponding pass: - // - Insert Convert before operations that doesn't support original element type for execution - // - Insert reverse Convert before operations that support original element type - // but have inputs that doesn't support it (because before them will be inserted Convert with exec_type - first point) - // - Then we should use ConstantFolding pass to convert element type of Scalars before inference. - // - Eliminate redundant Converts which can be inserted in AlignElementType() pass - ngraph::pass::Manager manager; - if (config.m_is_needed_to_align_precision) { - manager.register_pass(execution_element_type); - manager.register_pass(); - // TODO [100041] : In some cases AlignElementType pass can insert extra Convert because - // the pass doesn't know real precisions in real time. - // We call EliminateConverts pass to remove them - manager.register_pass(); + parameter->set_element_type(needed_in_type); + parameter->validate_and_infer_types(); + } } - manager.run_passes(body_ptr()); } void snippets::op::Subgraph::initialize_buffer_scratchpad_size() { @@ -431,22 +434,21 @@ void snippets::op::Subgraph::initialize_buffer_scratchpad_size() { // Propagate to up: in Store. Buffer can have only one Store { - auto parent = buffer->get_input_node_shared_ptr(0); - auto idx = buffer->input(0).get_source_output().get_index(); - // There may be graph with several LoopBegin and LoopEnd between Store/Brgemm and Buffer, - // so we should iterate through LoopBase - while (ov::is_type(parent)) { - const auto source_output = parent->input_value(idx); - parent = source_output.get_node_shared_ptr(); - idx = source_output.get_index(); - } - if (auto store = ov::as_type_ptr(parent)) { - store->set_offset(offset); - } else if (const auto brgemm = ov::as_type_ptr(parent)) { - // Brgemm encapsulates work with loading and storing of data - brgemm->set_offset_c(offset); - } else { - throw ngraph_error("Buffer::set_offset() was called when Buffer didn't have the corresponding Store op for offset propagation"); + if (buffer->is_intermediate_memory()) { + OPENVINO_ASSERT(buffer->get_input_size() == 1, "Buffer with intermediate memory must have one parent"); + auto parent = buffer->get_input_node_shared_ptr(0); + auto idx = buffer->input(0).get_source_output().get_index(); + while (ov::is_type(parent)) { + const auto source_output = parent->input_value(idx); + parent = source_output.get_node_shared_ptr(); + idx = source_output.get_index(); + } + if (auto memory_access = ov::as_type_ptr(parent)) { + memory_access->set_output_offset(offset, idx); + } else { + throw ngraph_error( + "Buffer::set_offset() was called when Buffer didn't have the corresponding MemoryAccess op for offset propagation"); + } } } @@ -463,17 +465,10 @@ void snippets::op::Subgraph::initialize_buffer_scratchpad_size() { for (const auto loop_target_output : child->output(index).get_target_inputs()) { propagate_down(loop_target_output); } - } else if (const auto load = ov::as_type_ptr(child)) { - load->set_offset(offset); - } else if (const auto brgemm = ov::as_type_ptr(child)) { - // Brgemm encapsulates work with loading and storing of data - if (target_input.get_index() == 0) { - brgemm->set_offset_a(offset); - } else if (target_input.get_index() == 1) { - brgemm->set_offset_b(offset); - } + } else if (auto memory_access = ov::as_type_ptr(child)) { + memory_access->set_input_offset(offset, target_input.get_index()); } else { - throw ngraph_error("Buffer::set_offset() was called when Buffer didn't have the corresponding Load op for offset propagation"); + throw ngraph_error("Buffer::set_offset() was called when Buffer didn't have the corresponding MemoryAccess op for offset propagation"); } }; @@ -494,26 +489,25 @@ void snippets::op::Subgraph::initialize_buffer_scratchpad_size() { continue; } - // Transpose and MatMul ops should have different memories on inputs and outputs to avoid data corruption, - // so after them, we should allocate new memory. Other operations (Eltwises, Convert) can be executed inplace. - const auto parent = buffer->get_input_node_shared_ptr(0); - if (ov::is_type(parent) || is_transpose_loop(parent)) { + if (buffer->is_intermediate_memory()) { + // Transpose, MatMul and other non-decomposed ops should have different memories on inputs and outputs to avoid data corruption, + // so after them, we should allocate new memory. Other operations (Eltwises, Convert) can be executed inplace inside Loop. + OPENVINO_ASSERT(buffer->get_input_size() == 1, "Buffer with intermediate memory must have one parent"); + const auto parent = buffer->get_input_node_shared_ptr(0); + if (!ov::is_type(parent) || is_transpose_loop(parent)) { + offset = m_buffer_scratchpad; + propagate_offset(buffer, offset); + m_buffer_scratchpad += buffer_size; + continue; + } + + propagate_offset(buffer, offset); + } else { + // Single Buffer without input should allocate new memory offset = m_buffer_scratchpad; propagate_offset(buffer, offset); m_buffer_scratchpad += buffer_size; - continue; } - - // If Buffer op requires memory size more that has been already allocated, - // we increase current memory size to the needed size - // For example, it's possible when we have a sequence of Eltwise ops with broadcasting - const auto current_allocated_memory_size = m_buffer_scratchpad - offset; - if (buffer_size > current_allocated_memory_size) { - m_buffer_scratchpad += (buffer_size - current_allocated_memory_size); - // Note: we don't update offset because we just add memory to needed size - } - - propagate_offset(buffer, offset); } } } @@ -602,31 +596,49 @@ snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& ou snippets::Schedule snippets::op::Subgraph::generate(const BlockedShapeVector& output_shapes, const BlockedShapeVector& input_shapes, - ngraph::pass::Manager& opt, + ngraph::pass::Manager& pre_dialect, + ngraph::pass::Manager& post_dialect, + ngraph::pass::Manager& post_precision, const void* compile_params) { canonicalize(output_shapes, input_shapes); - return generate(opt, compile_params); + return generate(pre_dialect, post_dialect, post_precision, compile_params); } snippets::Schedule snippets::op::Subgraph::generate(const void* compile_params) { auto mngr = ngraph::pass::Manager(); - return generate(mngr, compile_params); + return generate(mngr, mngr, mngr, compile_params); } -snippets::Schedule snippets::op::Subgraph::generate(ngraph::pass::Manager& opt, const void* compile_params) { +snippets::Schedule snippets::op::Subgraph::generate( + ngraph::pass::Manager& pre_dialect, + ngraph::pass::Manager& post_dialect, + ngraph::pass::Manager& post_precision, + const void* compile_params) { INTERNAL_OP_SCOPE(Subgraph); OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::generate") NGRAPH_CHECK(m_generator != nullptr, "generate is called while generator is not set"); + pre_dialect.run_passes(body_ptr()); convert_to_snippet_dialect(); - opt.run_passes(body_ptr()); + post_dialect.run_passes(body_ptr()); + + ngraph::pass::Manager precision_manager; + precision_manager.register_pass(m_generator->get_target_machine()); + precision_manager.register_pass(); + precision_manager.register_pass(); + precision_manager.run_passes(body_ptr()); + + post_precision.run_passes(body_ptr()); // After all passes, when all optimizations are completed and all MemoryAccess ops are inserted, // we can calculate common buffer scratchpad size and propagate offset from Buffer to the corresponding MemoryAccess ops if (config.m_has_domain_sensitive_ops) initialize_buffer_scratchpad_size(); - snippets::pass::AssignRegisters().run_on_model(body_ptr()); + std::function& op)> reg_type_mapper = [=](const std::shared_ptr& op) -> Generator::opRegType { + return m_generator->get_op_reg_type(op); + }; + snippets::pass::AssignRegisters(reg_type_mapper).run_on_model(body_ptr()); const auto ops = body_ptr()->get_ops(); ngraph::snippets::Generator::GeneratorConfig generatorConfig; diff --git a/src/common/snippets/src/pass/align_element_type.cpp b/src/common/snippets/src/pass/align_element_type.cpp deleted file mode 100644 index abd50a9e44605c..00000000000000 --- a/src/common/snippets/src/pass/align_element_type.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include - -#include "snippets/snippets_isa.hpp" -#include "snippets/op/convert_saturation.hpp" -#include "snippets/pass/align_element_type.hpp" -#include "snippets/utils.hpp" -#include "ov_ops/type_relaxed.hpp" -#include "ngraph/op/util/op_types.hpp" - -#include - -namespace { - -inline auto is_in_op(const std::shared_ptr& n) -> bool { - return ov::is_type(n) - || ov::is_type(n); -} - -// At the moment Subgraph supports only Eltwise, Select, Convert, Broadcast and FQ (which is decomposed into Eltwises and Convert) with -// Softmax (which is decomposed into Eltwises as well) -// And only Eltwise and Select ops supports execution only in "exec_type". So we can check op type from the opposite -// NOTE: This check is only for executable which isn't Parameter/Constant/Result -inline auto op_supports_only_exec_type(const std::shared_ptr& n) -> bool { - return !is_in_op(n) && - !ov::is_type(n) && - !ov::is_type(n) && - !ov::is_type(n) && - !ov::is_type(n) && - !ov::is_type(n); -} - -} // namespace - -ngraph::snippets::pass::AlignElementType::AlignElementType(const ov::element::Type exec_type) : exec_type(exec_type) { } - -bool ngraph::snippets::pass::AlignElementType::run_on_model(const std::shared_ptr &m) { - RUN_ON_FUNCTION_SCOPE(AlignElementType); - - auto insertConvert = [](const std::shared_ptr& op, const size_t idx, const ov::element::Type& element_type) -> void { - auto convert = std::make_shared(op->input(idx).get_source_output(), element_type); - ngraph::copy_runtime_info(op->get_input_node_shared_ptr(idx), convert); - op->set_argument(idx, convert); - }; - - // NOTE: We don't call validate_and_infer_types() to avoid precision conflicts on inputs - bool rewritten = false; - auto ops = m->get_ordered_ops(); - for (auto& op : ops) { - if (is_in_op(op)) { - continue; - } - - if (op_supports_only_exec_type(op)) { - for (size_t i = 0; i < op->inputs().size(); i++) { - auto shared_input = op->get_input_node_shared_ptr(i); - auto existing_convert = ov::as_type_ptr(shared_input); - // We should insert Convert before Ops, which supports only exec element type, only when: - // - Input is Convert with unsupported destination type - // - Input is Op which support any element type - // We couldn't unite these conditions and just check that element type isn't supported exec type - // because we don't call validate_and_infer_types() so we don't know new precisions after setting of original - // input and output element types - if ((existing_convert && existing_convert->get_destination_type() != exec_type) || - (!op_supports_only_exec_type(shared_input))) { - insertConvert(op, i, exec_type); - rewritten |= true; - } - } - if (auto tr_node = std::dynamic_pointer_cast(op)) { - tr_node->set_overridden_output_type(exec_type, 0); - rewritten |= true; - } - } else { // branch for Movement ops, MatMul ops in the future and for the Convert, Result - for (size_t i = 0; i < op->inputs().size(); i++) { - auto shared_input = op->get_input_node_shared_ptr(i); - // it's original element type because we don't use validate_and_infer_type() anywhere - const auto original_eltype = op->input(i).get_element_type(); - // If before op there is another op that doesn't support execution on original element type, we know that - // before this op will be inserted reverse Convert to support execution on supported element type (first branch of condition). - // So we should return original element type for operations that can support low precision - if (op_supports_only_exec_type(shared_input) && original_eltype != exec_type) { - insertConvert(op, i, original_eltype); - rewritten |= true; - } - } - } - } - - return rewritten; -} - -bool ngraph::snippets::pass::AlignElementType::opNeedsAlignElementType(const std::shared_ptr& op, const ov::element::Type exec_type) { - // At the moment Snippets support only Eltwise/Convert/FQ/Select/Softmax/Broadcast which one output so we can just call get_element_type() - return op_supports_only_exec_type(op) && op->get_element_type() != exec_type; -} diff --git a/src/common/snippets/src/pass/assign_registers.cpp b/src/common/snippets/src/pass/assign_registers.cpp index 3de3138db60767..c9af20443b8938 100644 --- a/src/common/snippets/src/pass/assign_registers.cpp +++ b/src/common/snippets/src/pass/assign_registers.cpp @@ -14,6 +14,7 @@ namespace { constexpr size_t reg_count = 16lu; +using opRegType = ngraph::snippets::Generator::opRegType; } // namespace bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr& f) { @@ -22,31 +23,12 @@ bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr using Reg = size_t; using tensor = std::shared_ptr; auto ops = f->get_ordered_ops(); - // Note that currently there are 3 types of ops: - // * gpr->gpr: (Parameter, Result, LoopBegin, LoopEnd) will also be Buffer? - // * gpr->vec: or vec->gpr Load/LoadConvert, Store/StoreConvert, BroadcastLoad etc. - // * vec->vec: all other "normal" operations that perform calculations on vector registers: Add, BroadcastMove, Power, etc. - enum op_reg_type {gpr2gpr, gpr2vec, vec2gpr, vec2vec}; - auto get_op_reg_type = [](const std::shared_ptr& op) { - if (std::dynamic_pointer_cast(op) || - std::dynamic_pointer_cast(op) || - std::dynamic_pointer_cast(op) || - std::dynamic_pointer_cast(op) || - std::dynamic_pointer_cast(op) || - std::dynamic_pointer_cast(op)) - return gpr2gpr; - else if (std::dynamic_pointer_cast(op) || - std::dynamic_pointer_cast(op)) - return gpr2vec; - else if (std::dynamic_pointer_cast(op)) - return vec2gpr; - else - return vec2vec; - }; - std::vector>> typed_ops; - for (const auto& op : ops) - typed_ops.emplace_back(std::make_pair(get_op_reg_type(op), op)); + std::vector>> typed_ops; + for (const auto& op : ops) { + typed_ops.emplace_back(std::make_pair(m_reg_type_mapper(op), op)); + } + size_t counter_vec = 0; size_t counter_gpr = 0; std::map regs_vec, regs_gpr; @@ -64,10 +46,12 @@ bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr // here we use the fact that Result input & output tensors are identical by construction manually_assigned_gprs[op->output(0).get_tensor_ptr()] = static_cast(f->get_result_index(result) + num_parameters); - } else if (const auto& buffer = ov::as_type_ptr(op)) { + } else if (const auto buffer = ov::as_type_ptr(op)) { // All buffers have one common data pointer - manually_assigned_gprs[op->input(0).get_tensor_ptr()] = - static_cast(num_results + num_parameters); + if (buffer->is_intermediate_memory()) { + manually_assigned_gprs[op->input(0).get_tensor_ptr()] = + static_cast(num_results + num_parameters); + } manually_assigned_gprs[op->output(0).get_tensor_ptr()] = static_cast(num_results + num_parameters); } else if (ov::is_type(op) || ov::is_type(op)) { @@ -114,12 +98,12 @@ bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr }; for (const auto& t_op : typed_ops) { switch (t_op.first) { - case vec2vec: - case gpr2vec: + case opRegType::vec2vec: + case opRegType::gpr2vec: enumerate_out_tensors(t_op.second, regs_vec, manually_assigned_vecs, counter_vec); break; - case gpr2gpr: - case vec2gpr: + case opRegType::gpr2gpr: + case opRegType::vec2gpr: enumerate_out_tensors(t_op.second, regs_gpr, manually_assigned_gprs, counter_gpr); break; } @@ -144,24 +128,25 @@ bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr for (size_t i = 0; i < typed_ops.size(); i++) { const auto& t_op = typed_ops[i]; std::vector used_tensors, defined_tensors; - for (const auto& in : t_op.second->inputs()) + for (const auto& in : t_op.second->inputs()) { used_tensors.push_back(in.get_tensor_ptr()); + } for (const auto& out : t_op.second->outputs()) defined_tensors.push_back(out.get_tensor_ptr()); switch (t_op.first) { - case vec2vec: + case opRegType::vec2vec: used_vec[i] = tensor2reg(used_tensors, regs_vec); defined_vec[i] = tensor2reg(defined_tensors, regs_vec); break; - case gpr2gpr: + case opRegType::gpr2gpr: used_gpr[i] = tensor2reg(used_tensors, regs_gpr); defined_gpr[i] = tensor2reg(defined_tensors, regs_gpr); break; - case gpr2vec: + case opRegType::gpr2vec: used_gpr[i] = tensor2reg(used_tensors, regs_gpr); defined_vec[i] = tensor2reg(defined_tensors, regs_vec); break; - case vec2gpr: + case opRegType::vec2gpr: used_vec[i] = tensor2reg(used_tensors, regs_vec); defined_gpr[i] = tensor2reg(defined_tensors, regs_gpr); break; @@ -196,12 +181,12 @@ bool ngraph::snippets::pass::AssignRegisters::run_on_model(const std::shared_ptr if (k == ops.size()) throw ngraph_error("assign registers can't find target op in the body"); switch (typed_ops[k].first) { - case vec2vec: - case vec2gpr: + case opRegType::vec2vec: + case opRegType::vec2gpr: life_out_vec[n].insert(life_in_vec[k].begin(), life_in_vec[k].end()); break; - case gpr2gpr: - case gpr2vec: + case opRegType::gpr2gpr: + case opRegType::gpr2vec: life_out_gpr[n].insert(life_in_gpr[k].begin(), life_in_gpr[k].end()); break; } diff --git a/src/common/snippets/src/pass/collapse_subgraph.cpp b/src/common/snippets/src/pass/collapse_subgraph.cpp index cd3eb887481031..dc737d78b794a1 100644 --- a/src/common/snippets/src/pass/collapse_subgraph.cpp +++ b/src/common/snippets/src/pass/collapse_subgraph.cpp @@ -49,9 +49,16 @@ auto outputs_are_not_broadcastable(const std::shared_ptr& node) -> b auto is_supported_op(const std::shared_ptr &n) -> bool { OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::is_supported_op") auto is_supported_matmul = [](const std::shared_ptr& n) -> bool { - const auto& matmul = is_type(n); + const auto& matmul = ov::as_type_ptr(n); const auto& out_shape = n->get_output_partial_shape(0); - return matmul && out_shape.is_static() && out_shape.size() == 4; + if (!matmul || out_shape.is_dynamic() || out_shape.size() != 4) + return false; + const auto intype_0 = matmul->get_input_element_type(0); + const auto intype_1 = matmul->get_input_element_type(1); + const bool is_f32 = intype_0 == element::f32 && intype_1 == element::f32; + const bool is_int8 = (intype_0 == element::i8 || intype_0 == element::u8) && (intype_1 == element::i8); + const bool is_bf16 = intype_0 == element::bf16 && intype_1 == element::bf16; + return is_f32 || is_bf16 || is_int8; }; auto is_supported_transpose = [](const std::shared_ptr& n) -> bool { const auto& transpose = as_type_ptr(n); @@ -135,7 +142,9 @@ auto is_supported_op(const std::shared_ptr &n) -> bool { int64_t axis = -1; const auto rank = n->get_input_partial_shape(0).rank(); if (const auto softmax_v8 = ngraph::as_type_ptr(n)) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(n->get_friendly_name(), softmax_v8->get_axis(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else if (const auto softmax_v1 = ngraph::as_type_ptr(n)) { axis = softmax_v1->get_axis(); } else { @@ -212,7 +221,11 @@ const std::set ngraph::snippets::pass::TokenizeSnippets:: { ngraph::element::f32, ngraph::element::bf16, ngraph::element::i8, ngraph::element::u8 }; bool TokenizeSnippets::AppropriateForSubgraph(const std::shared_ptr &node) { - return is_supported_op(node) && has_supported_in_out(node) && node->get_control_dependencies().empty(); + return + is_supported_op(node) && + has_supported_in_out(node) && + node->get_control_dependencies().empty() && + snippets::op::Subgraph::check_broadcast(node); } TokenizeSnippets::TokenizeSnippets() { diff --git a/src/common/snippets/src/pass/common_optimizations.cpp b/src/common/snippets/src/pass/common_optimizations.cpp index 787fb8f650d5be..04278526ce8c34 100644 --- a/src/common/snippets/src/pass/common_optimizations.cpp +++ b/src/common/snippets/src/pass/common_optimizations.cpp @@ -17,7 +17,7 @@ #include "snippets/utils.hpp" #include "snippets/itt.hpp" -NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::CommonOptimizations, "Snippets::CommonOptimizations", 0); +NGRAPH_RTTI_DEFINITION(ngraph::snippets::pass::CommonOptimizations, "Snippets::CommonOptimizations"); namespace ngraph { namespace snippets { diff --git a/src/common/snippets/src/pass/fq_decomposition.cpp b/src/common/snippets/src/pass/fq_decomposition.cpp index 5c2cfd6b0f82c3..9688e0a0e22940 100644 --- a/src/common/snippets/src/pass/fq_decomposition.cpp +++ b/src/common/snippets/src/pass/fq_decomposition.cpp @@ -36,11 +36,6 @@ bool isValidRangesInputs(const std::shared_ptr& fq }); } -bool is_scalar_constant(const std::shared_ptr& source_output_node) { - return ngraph::is_type(source_output_node) && - ngraph::shape_size(source_output_node->get_shape()) == 1; -} - } // namespace ngraph::snippets::pass::FakeQuantizeDecomposition::FakeQuantizeDecomposition() { @@ -182,13 +177,6 @@ ngraph::snippets::pass::FakeQuantizeDecomposition::FakeQuantizeDecomposition() { register_matcher(m, callback); } -bool ngraph::snippets::pass::FakeQuantizeDecomposition::isAllScalarConstant(const std::shared_ptr& node) { - return is_scalar_constant(node->get_input_node_shared_ptr(1)) && - is_scalar_constant(node->get_input_node_shared_ptr(2)) && - is_scalar_constant(node->get_input_node_shared_ptr(3)) && - is_scalar_constant(node->get_input_node_shared_ptr(4)); -} - bool ngraph::snippets::pass::FakeQuantizeDecomposition::getScalesAndShifts( const std::shared_ptr& fq_node, std::vector& cl, diff --git a/src/common/snippets/src/pass/fuse_transpose_brgemm.cpp b/src/common/snippets/src/pass/fuse_transpose_brgemm.cpp index f50731bcf7cd51..62dd1292b3ffce 100644 --- a/src/common/snippets/src/pass/fuse_transpose_brgemm.cpp +++ b/src/common/snippets/src/pass/fuse_transpose_brgemm.cpp @@ -49,13 +49,8 @@ FuseTransposeBrgemm::FuseTransposeBrgemm() { auto callback = [=](pattern::Matcher& m) { OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "FuseTransposeBrgemm") - auto set_layout_from_order = [](const std::shared_ptr& node, const ov::Output& port) { - const auto& const_order = as_type_ptr(node->get_input_node_shared_ptr(1)); - std::vector layout = const_order->cast_vector(); - auto& rt_info = port.get_node_shared_ptr()->get_rt_info(); - rt_info["Layout"] = layout; - }; auto brgemm = as_type_ptr(m.get_match_root()); + // Transpose on the Brgemm's output if (!brgemm) { brgemm = as_type_ptr(m.get_match_root()->get_input_node_shared_ptr(0)); @@ -63,13 +58,13 @@ FuseTransposeBrgemm::FuseTransposeBrgemm() { const auto& transpose_out = m.get_match_value(); for (const auto& in : transpose_out.get_target_inputs()) in.replace_source_output(brgemm->output(0)); - set_layout_from_order(as_type_ptr(transpose_out.get_node_shared_ptr()), brgemm_out); + utils::set_transpose_output_layout(brgemm_out, as_type_ptr(transpose_out.get_node_shared_ptr())); } for (size_t i = 0; i < brgemm->get_input_size(); i++) { const auto& in_value = brgemm->input_value(i); if (transpose_matcher->match(in_value)) { const auto& transpose = as_type_ptr(in_value.get_node_shared_ptr()); - set_layout_from_order(transpose, transpose->input_value(0)); + utils::set_transpose_output_layout(transpose->input_value(0), transpose); brgemm->set_argument(i, transpose->input_value(0)); } } diff --git a/src/common/snippets/src/pass/insert_buffer.cpp b/src/common/snippets/src/pass/insert_buffer.cpp index 1b080bc0b0c041..e7f4c90ae028ed 100644 --- a/src/common/snippets/src/pass/insert_buffer.cpp +++ b/src/common/snippets/src/pass/insert_buffer.cpp @@ -31,7 +31,7 @@ ngraph::snippets::pass::InsertBuffer::InsertBuffer(const int32_t allocation_rank if (!ov::is_type(input_node) && !ov::is_type(input_node) && !ov::is_type(input_node)) { - const auto buffer = std::make_shared(input_node, allocation_rank); + const auto buffer = std::make_shared(input_node, allocation_rank); root->set_argument(input.get_index(), buffer); rewritten |= true; } @@ -68,7 +68,7 @@ ngraph::snippets::pass::InsertBuffer::InsertBuffer(const int32_t allocation_rank } } - const auto buffer = std::make_shared(output, allocation_rank); + const auto buffer = std::make_shared(output, allocation_rank); for (const auto& consumer : output.get_target_inputs()) { const auto output_node = consumer.get_node()->shared_from_this(); if (output_node != buffer && diff --git a/src/common/snippets/src/pass/insert_load_store.cpp b/src/common/snippets/src/pass/insert_load_store.cpp index ef0fed11b50574..114393bd872f96 100644 --- a/src/common/snippets/src/pass/insert_load_store.cpp +++ b/src/common/snippets/src/pass/insert_load_store.cpp @@ -30,7 +30,7 @@ ngraph::snippets::pass::InsertLoad::InsertLoad(const size_t count) { const auto& consumer_node = consumer.get_node(); if (ov::is_type(consumer_node) || ov::is_type(consumer_node) || - ov::is_type(consumer_node) || + ov::is_type(consumer_node) || ov::is_type(consumer_node)) { return false; } @@ -67,7 +67,7 @@ ngraph::snippets::pass::InsertStore::InsertStore(const size_t count) { const auto& parent_node = input.get_source_output().get_node(); if (ov::is_type(parent_node) || ov::is_type(parent_node) || - ov::is_type(parent_node) || + ov::is_type(parent_node) || ov::is_type(parent_node)) { return false; } diff --git a/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp b/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp index b4fdb2506dc008..7aa69d65bbde28 100644 --- a/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp +++ b/src/common/snippets/src/pass/load_movebroadcast_to_broadcastload.cpp @@ -24,20 +24,20 @@ ngraph::snippets::pass::LoadMoveBroadcastToBroadcastLoad::LoadMoveBroadcastToBro auto root = m.get_match_root(); const auto &pm = m.get_pattern_value_map(); - const auto input = pm.at(load_pattern).get_node_shared_ptr(); + const auto load = ov::as_type_ptr(pm.at(load_pattern).get_node_shared_ptr()); const auto param = pm.at(param_pattern).get_node_shared_ptr(); // Cannot rewrite Broadcast + Load if load has more than 1 user // or more than one input, or if Broadcast has several inputs - if (input->output(0).get_target_inputs().size() != 1 || - root->inputs().size() != 1 || input->inputs().size() != 1) { + if (load->output(0).get_target_inputs().size() != 1 || + root->inputs().size() != 1 || load->inputs().size() != 1) { return false; } auto inshape = root->input(0).get_partial_shape(); auto outshape = root->output(0).get_partial_shape(); - auto broadcastload = std::make_shared(param, outshape, ov::as_type_ptr(input)->get_offset()); + auto broadcastload = std::make_shared(param, outshape, load->get_offset()); ngraph::copy_runtime_info(root, broadcastload); ngraph::replace_node(root, broadcastload); diff --git a/src/common/snippets/src/pass/loop_fusion.cpp b/src/common/snippets/src/pass/loop_fusion.cpp index 18287d4464f40a..2291e0746075d9 100644 --- a/src/common/snippets/src/pass/loop_fusion.cpp +++ b/src/common/snippets/src/pass/loop_fusion.cpp @@ -73,7 +73,6 @@ auto get_buffer_and_loop_end(const std::shared_ptr(parent_shared); if (buffer) { if (buffer->output(0).get_target_inputs().size() == 0 || - buffer->get_input_size() != 1 || buffer->get_input_source_output(0).get_target_inputs().size() != 1) return false; diff --git a/src/common/snippets/src/pass/matmul_to_brgemm.cpp b/src/common/snippets/src/pass/matmul_to_brgemm.cpp index b74fb3e68cc47e..add672b0fef3ea 100644 --- a/src/common/snippets/src/pass/matmul_to_brgemm.cpp +++ b/src/common/snippets/src/pass/matmul_to_brgemm.cpp @@ -6,7 +6,7 @@ #include "snippets/pass/matmul_to_brgemm.hpp" -#include "snippets/op/brgemm.hpp" +#include "snippets/snippets_isa.hpp" #include "ngraph/opsets/opset1.hpp" #include "ngraph/rt_info.hpp" @@ -30,9 +30,13 @@ MatMulToBrgemm::MatMulToBrgemm() { return false; auto brgemm = std::make_shared(matmul->get_input_source_output(0), matmul->get_input_source_output(1)); + ov::NodeVector nodes = { brgemm }; + if (brgemm->get_output_element_type(0) != matmul->get_output_element_type(0)) { + nodes.emplace_back(std::make_shared(brgemm, matmul->get_output_element_type(0))); + } brgemm->set_friendly_name(matmul->get_friendly_name()); - ngraph::copy_runtime_info(matmul, brgemm); - ngraph::replace_node(matmul, brgemm); + ngraph::copy_runtime_info(matmul, nodes); + ngraph::replace_node(matmul, nodes.back()); return true; }; diff --git a/src/common/snippets/src/pass/mha_tokenization.cpp b/src/common/snippets/src/pass/mha_tokenization.cpp index 69a166140b4093..12161d5214c03c 100644 --- a/src/common/snippets/src/pass/mha_tokenization.cpp +++ b/src/common/snippets/src/pass/mha_tokenization.cpp @@ -184,7 +184,9 @@ ngraph::snippets::pass::TokenizeMHASnippets::TokenizeMHASnippets() { int64_t axis = 0; const auto rank = interm_op->get_input_partial_shape(0).rank(); if (const auto softmax_v8 = ngraph::as_type_ptr(interm_op)) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(interm_op->get_friendly_name(), softmax_v8->get_axis(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else if (const auto softmax_v1 = ngraph::as_type_ptr(interm_op)) { axis = softmax_v1->get_axis(); } else { diff --git a/src/common/snippets/src/pass/propagate_precision.cpp b/src/common/snippets/src/pass/propagate_precision.cpp new file mode 100644 index 00000000000000..19be34b4e97648 --- /dev/null +++ b/src/common/snippets/src/pass/propagate_precision.cpp @@ -0,0 +1,293 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/pass/propagate_precision.hpp" + +#include +#include +#include "ov_ops/type_relaxed.hpp" +#include "snippets/itt.hpp" +#include "ngraph/rt_info.hpp" + +using namespace ngraph; + +ngraph::snippets::pass::PropagatePrecision::PropagatePrecision( + const std::shared_ptr& target_machine) : target_machine(target_machine) { +} + +bool ngraph::snippets::pass::PropagatePrecision::run_on_model(const std::shared_ptr& f) { + RUN_ON_MODEL_SCOPE(PropagatePrecision); + OV_ITT_SCOPED_TASK(ngraph::pass::itt::domains::SnippetsTransform, "Snippets::op::PropagatePrecision") + + std::unordered_map, element::Type> result_types; + auto results = f->get_results(); + for (auto& result : results) { + result_types.emplace(result, result->get_input_element_type(0)); + } + + bool was_updated = true; + for (const auto& op : f->get_ordered_ops()) { + auto type_info = op->get_type_info(); + OPENVINO_ASSERT( + target_machine->has(type_info), + "operation '" + std::string(type_info.version_id) + "::" + std::string(type_info.name) + "' was not found in target machine"); + + auto exec = target_machine->get_supported_precisions(type_info); + const auto supported_precisions = exec(op); + if (supported_precisions.empty()) { + continue; + } + + // There are two operation types which break precision propagation: + // 1) Existing convertion operations. Solution: remove convertion + // operation before general algo + // 2) Type relaxed based operations. Will be resolved by snippet opset. + + auto input_precisions_were_changed = false; + + for (const auto& input : op->inputs()) { + const auto convert = ngraph::as_type(input.get_source_output().get_node()); + if (convert == nullptr) { + continue; + } + + const auto precision_before = convert->get_input_element_type(0); + const auto precision_after = convert->get_output_element_type(0); + if (can_be_removed(precision_before, precision_after, precision_before)) { + op->set_argument(input.get_index(), convert->input(0).get_source_output()); + input_precisions_were_changed = true; + } + } + + std::vector input_precisions; + for (const auto& input : op->inputs()) { + const auto input_precision = input.get_source_output().get_element_type(); + input_precisions.push_back(input_precision); + } + + assert(std::all_of( + supported_precisions.begin(), + supported_precisions.end(), + [&input_precisions](const std::vector& precisions) { + return precisions.size() == input_precisions.size(); + }) && "input precisions count is not equal for supported precisions"); + + // update input precisions + // if possible then convert precisions to supported + if (!supported_precisions.empty() && + std::all_of( + supported_precisions.begin(), + supported_precisions.end(), + [&input_precisions](const std::vector& precisions) { + return precisions != input_precisions; + })) { + auto precisions = get_precisions(input_precisions, + supported_precisions); + OPENVINO_ASSERT( + !precisions.empty(), + "there are no supported precisions for operation '" + std::string(type_info.version_id) + "::" + std::string(type_info.name) + "'"); + + auto find_convert = []( + const ngraph::Output parent_output, + const ngraph::element::Type convert_type) -> snippets::op::ConvertSaturation* { + for (const auto& input : parent_output.get_target_inputs()) { + const auto child = ngraph::as_type(input.get_node()); + if ((child != nullptr) && (child->get_output_element_type(0) == convert_type)) { + return child; + } + } + return nullptr; + }; + + for (size_t i = 0; i < op->get_input_size(); ++i) { + const auto& op_input = op->input(i); + const auto& required_after = precisions[i]; + auto parent_output = op_input.get_source_output(); + const auto actual_before = parent_output.get_element_type(); + if (actual_before != required_after) { + was_updated = true; + input_precisions_were_changed = true; + auto existing_convert = ngraph::as_type( + parent_output.get_node()); + + if (existing_convert == nullptr) { + existing_convert = find_convert(parent_output, required_after); + if (existing_convert != nullptr) { + // reuse existing convert + op->set_argument(op_input.get_index(), existing_convert->shared_from_this()); + continue; + } + } + + if (existing_convert == nullptr) { + // create new Convert + auto convert = std::make_shared( + parent_output, + required_after); + ngraph::copy_runtime_info(parent_output.get_node_shared_ptr(), convert); + op->set_argument(op_input.get_index(), convert); + continue; + } + + const auto actual_before = existing_convert->get_input_element_type(0); + const auto actual_after = existing_convert->get_output_element_type(0); + + if (can_be_removed(actual_before, actual_after, required_after)) { + // remove existing convert + existing_convert->output(0).replace(parent_output); + continue; + } + + if (can_be_fused(actual_after, required_after)) { + // fuse existing convert + auto convert = std::make_shared( + existing_convert->get_input_node_shared_ptr(0), + required_after); + ngraph::copy_runtime_info(parent_output.get_node_shared_ptr(), convert); + op->set_argument(op_input.get_index(), convert); + continue; + } + + // create new convert + auto convert = std::make_shared( + existing_convert->output(0), + required_after); + ngraph::copy_runtime_info(existing_convert->output(0).get_node()->shared_from_this(), convert); + op->set_argument(op_input.get_index(), convert); + } + } + } + + auto type_relaxed_node = std::dynamic_pointer_cast(op); + if (input_precisions_were_changed || (type_relaxed_node != nullptr)) { + // update output precision + std::vector op_output_types; + for (auto& output : op->outputs()) { + op_output_types.push_back(output.get_element_type()); + } + + if (type_relaxed_node != nullptr) { + // TODO: user story 104284 + // to keep previous functionality + // unary and binary element-wise operations are supported + // will be replaced to snippets opset later + const auto op_element_type = op->get_input_element_type(0); + if (type_relaxed_node->get_overridden_output_type(0) != op_element_type) { + was_updated = true; + OPENVINO_ASSERT(op->get_output_size() == 1ull, "operation with several output is not supported"); + + type_relaxed_node->set_overridden_output_type(op_element_type, 0); + op->validate_and_infer_types(); + } + } else { + op->validate_and_infer_types(); + } + + for (size_t i = 0; i < op->get_output_size(); ++i) { + auto output = op->output(i); + + if (output.get_element_type() != op_output_types[i]) { + was_updated = true; + auto convert = std::make_shared( + output, + op_output_types[i]); + ngraph::copy_runtime_info(output.get_node_shared_ptr(), convert); + + for (auto& input : output.get_target_inputs()) { + auto child = input.get_node(); + if (child == convert.get()) { + continue; + } + + input.replace_source_output(convert->output(0)); + + + if (ngraph::is_type(input.get_node())) { + input.get_tensor_ptr()->add_names(output.get_tensor_ptr()->get_names()); + + const std::string original_name = op->get_friendly_name(); + op->set_friendly_name(original_name + "_original"); + convert->set_friendly_name(original_name); + } + } + output.get_tensor_ptr()->set_names({}); + } + } + } + } + + for (auto it = result_types.begin(); it != result_types.end(); ++it) { + const auto result = it->first; + const auto actual_type = result->get_input_element_type(0); + const auto expected_type = it->second; + if (actual_type != it->second) { + was_updated = true; + auto convert = std::make_shared( + result->get_input_node_shared_ptr(0), + expected_type); + ngraph::copy_runtime_info(result->get_input_node_shared_ptr(0), convert); + result->set_argument(0, convert); + } + } + + return was_updated; +} + +bool ngraph::snippets::pass::PropagatePrecision::can_be_removed( + const element::Type& actual_before, + const element::Type& actual_after, + const element::Type& required_after) noexcept { + if (actual_before != required_after) { + return false; + } + + return can_be_fused(actual_after, actual_before); +} + +bool ngraph::snippets::pass::PropagatePrecision::can_be_fused( + const element::Type& actual, + const element::Type& required) noexcept { + // custom conditions: between int & float precisions + if (((actual == element::bf16) || (actual == element::f16) || (actual == element::f32)) && + ((required == element::u8) || (required == element::i8))) { + return true; + } + + if ((actual == element::f32) && ((required == element::u16) || (required == element::i16))) { + return true; + } + + // general conditions: any new added precision will support + return + (actual.is_real() == required.is_real()) && + (actual.bitwidth() >= required.bitwidth()); +} + +std::vector ngraph::snippets::pass::PropagatePrecision::get_precisions( + const std::vector& input_precisions, + const std::set>& supported_precisions_pack) noexcept { + bool was_found = false; + for (const auto& supported_precisions : supported_precisions_pack) { + for (size_t i = 0; i < supported_precisions.size(); ++i) { + const auto& supported_precision = supported_precisions[i]; + const auto& input_precision = input_precisions[i]; + if ((supported_precision.is_real() != input_precision.is_real()) || + (input_precision.bitwidth() > supported_precision.bitwidth())) { + was_found = false; + break; + } + + was_found = true; + } + if (was_found) { + return supported_precisions; + } + } + + if (!supported_precisions_pack.empty()) { + return *supported_precisions_pack.begin(); + } + + return {}; +} diff --git a/src/common/snippets/src/pass/reset_buffer.cpp b/src/common/snippets/src/pass/reset_buffer.cpp index bae2ac58ccdb15..54bdfef03f7f13 100644 --- a/src/common/snippets/src/pass/reset_buffer.cpp +++ b/src/common/snippets/src/pass/reset_buffer.cpp @@ -79,10 +79,9 @@ ngraph::snippets::pass::ResetBufferState::ResetBufferState() { // If after Loop there is immediately Buffer, we should reset the Buffer ptr for the next calculations for (size_t i = 0; i < o_size; ++i) { - const auto result_shape = body_shapes[i_size + i].get_shape(); // check for first target input is enough for Buffer searching because operations can have only single Buffer per each output port as op const auto consumer = loop_end->output(i).get_target_inputs().begin()->get_node(); - if (ov::is_type(consumer)) { + if (const auto buffer = ov::as_type_ptr(consumer->shared_from_this())) { // To calculate finalization offset we should know index of nesting Loop auto loop_index = 0lu; auto loop = loop_end->input_value(i).get_node_shared_ptr(); @@ -93,7 +92,8 @@ ngraph::snippets::pass::ResetBufferState::ResetBufferState() { port_idx = source_output.get_index(); loop_index++; } - + const auto result_shape = buffer->get_allocation_shape(); + NGRAPH_CHECK(loop_index < result_shape.size(), "Buffer has invalid Loop index and allocation shape rank"); const auto work_amount = std::accumulate(result_shape.rbegin(), result_shape.rbegin() + loop_index + 1, size_t(1), std::multiplies()); finalization_offsets[i_size + i] = calculate_required_finalization_offsets(work_amount, *(result_shape.rbegin() + loop_index)); diff --git a/src/common/snippets/src/pass/softmax_decomposition.cpp b/src/common/snippets/src/pass/softmax_decomposition.cpp index 8c1c79a4b544db..a4a7ad77773327 100644 --- a/src/common/snippets/src/pass/softmax_decomposition.cpp +++ b/src/common/snippets/src/pass/softmax_decomposition.cpp @@ -33,7 +33,9 @@ ngraph::snippets::pass::SoftmaxDecomposition::SoftmaxDecomposition(const size_t int64_t axis = 0; if (const auto softmax_v8 = ngraph::as_type_ptr(root)) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(root->get_friendly_name(), softmax_v8->get_axis(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else if (const auto softmax_v1 = ngraph::as_type_ptr(root)) { axis = softmax_v1->get_axis(); } else { @@ -126,7 +128,7 @@ ngraph::snippets::pass::SoftmaxDecomposition::SoftmaxDecomposition(const size_t apply_increments_sum, finalization_offsets_sum); const auto horizon_sum = std::make_shared(sum); - const auto buffer_exp = std::make_shared(loop_sum_end->output(0), buffer_allocation_rank); + const auto buffer_exp = std::make_shared(loop_sum_end->output(0), buffer_allocation_rank); /* =========================================== */ diff --git a/src/common/snippets/src/pass/softmax_reshape_elimination.cpp b/src/common/snippets/src/pass/softmax_reshape_elimination.cpp index f770f4e80668cd..7229b29cc54ccd 100644 --- a/src/common/snippets/src/pass/softmax_reshape_elimination.cpp +++ b/src/common/snippets/src/pass/softmax_reshape_elimination.cpp @@ -34,7 +34,9 @@ ngraph::snippets::pass::SoftmaxReshapeElimination::SoftmaxReshapeElimination() { const auto softmax_rank = softmax->get_input_partial_shape(0).rank(); int64_t axis = 0; if (const auto softmax_v8 = ngraph::as_type_ptr(softmax)) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(softmax->get_friendly_name(), softmax_v8->get_axis(), softmax_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else if (const auto softmax_v1 = ngraph::as_type_ptr(softmax)) { axis = softmax_v1->get_axis(); } else { diff --git a/src/common/snippets/src/pass/vector_to_scalar.cpp b/src/common/snippets/src/pass/vector_to_scalar.cpp index 512a0731062cf7..4f98a49de4eedd 100644 --- a/src/common/snippets/src/pass/vector_to_scalar.cpp +++ b/src/common/snippets/src/pass/vector_to_scalar.cpp @@ -24,7 +24,7 @@ ngraph::snippets::pass::SetScalarCountForLoad::SetScalarCountForLoad() { if (!load) return false; - load->set_count(1lu); + load->set_input_count(1lu, 0); return true; }); } @@ -43,7 +43,7 @@ ngraph::snippets::pass::SetScalarCountForStore::SetScalarCountForStore() { if (!store) return false; - store->set_count(1lu); + store->set_output_count(1lu, 0); return true; }); } diff --git a/src/common/snippets/src/utils.cpp b/src/common/snippets/src/utils.cpp index 3018d99e95f6f1..6587ff93fa69d2 100644 --- a/src/common/snippets/src/utils.cpp +++ b/src/common/snippets/src/utils.cpp @@ -115,6 +115,17 @@ ov::PartialShape get_port_planar_shape(const Output& out) { return get_reordered_planar_shape(tensor_shape, layout); } +void set_transpose_output_layout(const ov::Output& port, const std::shared_ptr& node) { + const auto& const_order = as_type_ptr(node->get_input_node_shared_ptr(1)); + OPENVINO_ASSERT(const_order != nullptr, "Transpose order must be Constant to set layout!"); + set_output_layout(port, const_order->cast_vector()); +} + +void set_output_layout(const ov::Output& port, const std::vector& layout) { + auto& rt_info = port.get_node_shared_ptr()->get_rt_info(); + rt_info["Layout"] = layout; +} + } // namespace utils } // namespace snippets } // namespace ngraph diff --git a/src/common/snippets/tests/include/lowering_utils.hpp b/src/common/snippets/tests/include/lowering_utils.hpp index be2e0f2e756044..7dfa71a4b6a7f7 100644 --- a/src/common/snippets/tests/include/lowering_utils.hpp +++ b/src/common/snippets/tests/include/lowering_utils.hpp @@ -16,7 +16,7 @@ using BlockedShapeVector = ngraph::snippets::op::Subgraph::BlockedShapeVector; class DummyEmitter : public ngraph::snippets::Emitter { public: // Here I pass Add to Emitter, but could be any other op, since it's ignored anyway. - DummyEmitter() : ngraph::snippets::Emitter(std::make_shared()) {} + DummyEmitter(const std::vector& custom_opset = {}) : ngraph::snippets::Emitter(std::make_shared()) {} void emit_code(const std::vector&, const std::vector&, const std::vector&, @@ -36,6 +36,9 @@ class DummyGenerator : public ngraph::snippets::Generator { public: DummyGenerator() : ngraph::snippets::Generator(std::make_shared()) {} DummyGenerator(const std::shared_ptr& t) : ngraph::snippets::Generator(t) {} + +protected: + opRegType get_specific_op_reg_type(const std::shared_ptr& op) const override { return vec2vec; }; }; class LoweringTests : public TransformationTestsF { @@ -49,7 +52,9 @@ class LoweringTests : public TransformationTestsF { static std::shared_ptr getSubgraph(const std::shared_ptr& f); static std::shared_ptr getLoweredSubgraph(const std::shared_ptr& f, const ov::PartialShape& master_shape, - ov::pass::Manager target_optimizations = {}, + ov::pass::Manager pre_dialect = {}, + ov::pass::Manager post_dialect = {}, + ov::pass::Manager post_precision = {}, const std::shared_ptr generator = nullptr); static std::shared_ptr getTokenizedSubgraph(const std::shared_ptr& f); ov::PartialShape master_shape{}; diff --git a/src/common/snippets/tests/include/pass/precision_propagation.hpp b/src/common/snippets/tests/include/pass/precision_propagation.hpp new file mode 100644 index 00000000000000..a60b9161ab4fc4 --- /dev/null +++ b/src/common/snippets/tests/include/pass/precision_propagation.hpp @@ -0,0 +1,54 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "lowering_utils.hpp" +#include "snippets_helpers.hpp" + +namespace ov { +namespace test { +namespace snippets { + +class PrecisionPropagationParamsValues { +public: + class Actual { + public: + std::pair convertion_before_op1; + element::Type convertion_before_op2_1; + std::pair convertion_before_op2_2; + std::set> op1_supported_precisions; + std::set> op2_supported_precisions; + }; + + class Expected { + public: + std::pair convertion_before_op1; + element::Type convertion_before_op2_1; + std::pair convertion_before_op2_2; + element::Type convertion_after_op2; + }; + + std::vector input_types; + Actual actual; + Expected expected; +}; + +typedef std::tuple< + std::pair, // input shapes + PrecisionPropagationParamsValues +> PrecisionPropagationParams; + +class PrecisionPropagationTest : public TransformationTestsF, + public testing::WithParamInterface { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + +protected: + std::shared_ptr snippets_function; +}; + +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/common/snippets/tests/src/lowering_utils.cpp b/src/common/snippets/tests/src/lowering_utils.cpp index a536a0317eae12..55480e95dae510 100644 --- a/src/common/snippets/tests/src/lowering_utils.cpp +++ b/src/common/snippets/tests/src/lowering_utils.cpp @@ -11,10 +11,12 @@ namespace ov { namespace test { namespace snippets { -DummyTargetMachine::DummyTargetMachine(const std::vector& custom_opset) { - auto dummy_functor = [](const std::shared_ptr& n) { - return std::make_shared(); +DummyTargetMachine::DummyTargetMachine(const std::vector&custom_opset) { + auto dummy_functor = ngraph::snippets::jitters_value { + [](const std::shared_ptr& n) { return std::make_shared(); }, + [](const std::shared_ptr& n) { return std::set>{};} }; + jitters[op::v0::Parameter::get_type_info_static()] = dummy_functor; jitters[op::v0::Constant::get_type_info_static()] = dummy_functor; jitters[op::v0::Result::get_type_info_static()] = dummy_functor; @@ -97,7 +99,9 @@ std::shared_ptr LoweringTests::getSubgraph(const std::shared_ptr LoweringTests::getLoweredSubgraph(const std::shared_ptr &f, const ov::PartialShape& master_shape, - ov::pass::Manager target_optimizations, + ov::pass::Manager pre_dialect, + ov::pass::Manager post_dialect, + ov::pass::Manager post_precision, const std::shared_ptr generator) { auto subgraph = getTokenizedSubgraph(f); subgraph->set_generator(generator == nullptr ? std::make_shared() : generator); @@ -119,7 +123,7 @@ std::shared_ptr LoweringTests::getLoweredSubgrap } body_rt_info["PluginShapesOverride"] = new_shapes; subgraph->set_tile_rank(2); - subgraph->generate(target_optimizations); + subgraph->generate(pre_dialect, post_precision, post_precision); return subgraph; } diff --git a/src/common/snippets/tests/src/pass/precision_propagation.cpp b/src/common/snippets/tests/src/pass/precision_propagation.cpp new file mode 100644 index 00000000000000..3c7da4d06aa165 --- /dev/null +++ b/src/common/snippets/tests/src/pass/precision_propagation.cpp @@ -0,0 +1,294 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "pass/precision_propagation.hpp" + +#include +#include "ngraph/pass/validate.hpp" +#include "snippets/pass/propagate_precision.hpp" +#include "snippets/op/convert_saturation.hpp" +#include "common_test_utils/common_utils.hpp" +#include "precision_propagation_function.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +class DummyPrecisionPropagationTargetMachine : public DummyTargetMachine { +public: + DummyPrecisionPropagationTargetMachine( + const std::set>& op1_supported_precisions, + const std::set>& op2_supported_precisions) + : DummyTargetMachine() { + jitters[DummyAdd::get_type_info_static()] = ngraph::snippets::jitters_value { + [](const std::shared_ptr& n) { return std::make_shared(); }, + [op1_supported_precisions](const std::shared_ptr& n) { return op1_supported_precisions; }}; + jitters[op::v1::Maximum::get_type_info_static()] = ngraph::snippets::jitters_value{ + [](const std::shared_ptr& n) { return std::make_shared(); }, + [op2_supported_precisions](const std::shared_ptr&n) { return op2_supported_precisions; }}; + + auto default_jitter = ngraph::snippets::jitters_value{ + [](const std::shared_ptr& n) { return std::make_shared(); }, + [](const std::shared_ptr& n) { return std::set>{};} }; + jitters[ngraph::snippets::op::ConvertSaturation::get_type_info_static()] = default_jitter; + } +}; + +} // namespace + +std::string PrecisionPropagationTest::getTestCaseName(testing::TestParamInfo obj) { + std::pair shapes; + PrecisionPropagationParamsValues test_values; + std::tie(shapes, test_values) = obj.param; + + auto to_string = [](const std::set>& precisions_pack) noexcept { + std::ostringstream result; + result << "{"; + for (const auto& precisions : precisions_pack) { + result << CommonTestUtils::vec2str(precisions) << "_"; + } + result << "}"; + return result.str(); + }; + + std::ostringstream result; + result << "IN0_" << shapes.first << "_" << test_values.input_types[0] << "_" + << "IN1_" << shapes.second << "_" << test_values.input_types[1] << "_" + << "IN2_" << test_values.input_types[2] + << to_string(test_values.actual.op1_supported_precisions) << "_" + << to_string(test_values.actual.op2_supported_precisions) << "_" + << test_values.expected.convertion_before_op1.first << "_" << test_values.expected.convertion_before_op1.second << "_" + << test_values.expected.convertion_before_op2_1 << "_" + << test_values.expected.convertion_before_op2_2.first << "_" << test_values.expected.convertion_before_op2_2.second << "_" + << test_values.expected.convertion_after_op2 << "_"; + return result.str(); +} + +TEST_P(PrecisionPropagationTest, CompareFunctions) { + disable_rt_info_check(); + + const auto param = GetParam(); + const auto shapes = std::get<0>(param); + const auto test_values = std::get<1>(param); + + const auto input_shapes = std::vector({ shapes.first, shapes.second }); + PrecisionPropagationAddFunction function_stub( + input_shapes, + test_values.input_types[0], + test_values.input_types[1], + test_values.input_types[2], + { + test_values.actual.convertion_before_op1, + test_values.actual.convertion_before_op2_1, + test_values.actual.convertion_before_op2_2 + }, + { + test_values.expected.convertion_before_op1, + test_values.expected.convertion_before_op2_1, + test_values.expected.convertion_before_op2_2, + test_values.expected.convertion_after_op2 + }); + function = function_stub.getOriginal(); + + const auto target_machine = std::make_shared( + test_values.actual.op1_supported_precisions, + test_values.actual.op2_supported_precisions); + + manager.register_pass(target_machine); + + function_ref = function_stub.getReference(); +} + +namespace PrecisionPropagationTestInstantiation { +// clang-format off + +std::vector> shapes { + {{1, 3, 16, 16}, {1, 3, 16, 16}} +}; + +std::vector test_cases { + { + {element::f32, element::f32, element::f32}, + { + {}, + {}, + {}, + {{element::f32, element::f32}}, + {{element::f32, element::f32}} + }, + {} + }, + // in: Parameter I8 => Op1 I32 => Convert I8 => Op1 I8 => Result + // out: Parameter I8 => Add I32 => Convert I8 => Convert FP32 => Op1 FP32 => Result + { + {element::i8, element::i8, element::i8}, + { + {}, + {}, + {}, + {{element::i8, element::i8}}, + {{element::f32, element::f32}} + }, + { + {}, + element::i8, + {element::f32, element::f32}, + {element::i8} + } + }, + { + {element::i8, element::i8, element::i8}, + { + {}, + {}, + {}, + {{element::i8, element::i8}}, + {{element::i8, element::i8}} + }, + { + {}, + {}, + {element::i8, element::undefined}, + {} + } + }, + { + {element::i8, element::i8, element::i8}, + { + {}, + {}, + {}, + {{element::i8, element::i8}}, + {{element::i32, element::i32}} + }, + { + {}, + {element::i8}, + {element::i32, element::i32}, + {element::i8} + } + }, + { + {element::bf16, element::bf16, element::f32}, + { + {element::f32, element::f32}, + {}, + {}, + { + {element::f32, element::f32}, + {element::i8, element::i8} + }, + { + {element::f32, element::f32}, + {element::i32, element::i32} + } + }, + { + {element::f32, element::f32}, + {}, + {}, + {} + } + }, + // propagate precision via operation #1 + { + {element::bf16, element::bf16, element::f32}, + { + {element::f32, element::f32}, + {}, + {}, + { + {element::f32, element::f32}, + {element::bf16, element::bf16} + }, + { + {element::f32, element::f32} + } + }, + { + {}, + {}, + {element::f32, element::undefined}, + {} + } + }, + // propagate precision via operation #1 + { + {element::bf16, element::bf16, element::bf16}, + { + {element::f32, element::f32}, + {}, + {element::undefined, element::f32}, + { + {element::f32, element::f32}, + {element::bf16, element::bf16} + }, + { + {element::f32, element::f32} + } + }, + { + {}, + {}, + {element::f32, element::f32}, + {} + } + }, + // propagate precision via both operations + { + {element::bf16, element::bf16, element::bf16}, + { + {element::f32, element::f32}, + {}, + {element::undefined, element::f32}, + { + {element::f32, element::f32}, + {element::bf16, element::bf16} + }, + { + {element::f32, element::f32}, + {element::bf16, element::bf16} + } + }, + { + {}, + {}, + {}, + {element::f32} + } + }, + { + {element::bf16, element::bf16, element::bf16}, + { + {}, + {}, + {}, + {{element::f32, element::f32}}, + {{element::f32, element::f32}} + }, + { + {{element::f32}, {element::f32}}, + {element::bf16}, + {{element::f32}, {element::f32}}, + {element::bf16} + } + }, +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_PrecisionPropagationTest, + PrecisionPropagationTest, + ::testing::Combine( + ::testing::ValuesIn(shapes), + ::testing::ValuesIn(test_cases)), + PrecisionPropagationTest::getTestCaseName); + +// clang-format on +} // namespace PrecisionPropagationTestInstantiation + +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/common/snippets/tests/src/pass/precision_propagation_convert_test.cpp b/src/common/snippets/tests/src/pass/precision_propagation_convert_test.cpp new file mode 100644 index 00000000000000..cc6c113cc3f671 --- /dev/null +++ b/src/common/snippets/tests/src/pass/precision_propagation_convert_test.cpp @@ -0,0 +1,153 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "snippets/pass/propagate_precision.hpp" + +namespace ov { +namespace test { +namespace snippets { + +class PrecisionPropagationConvertTest : public testing::Test {}; + +TEST_F(PrecisionPropagationConvertTest, smoke_Snippets_PrecisionPropagation_can_be_fused) { + const std::set> precisions_set = { + {element::u64, element::u64}, + {element::u64, element::u32}, + {element::u64, element::u16}, + {element::u64, element::u8}, + {element::u32, element::u32}, + {element::u32, element::u16}, + {element::u32, element::u8}, + {element::u16, element::u16}, + {element::u16, element::u8}, + {element::u8, element::u8}, + + {element::i64, element::i64}, + {element::i64, element::i32}, + {element::i64, element::i16}, + {element::i64, element::i8}, + {element::i32, element::i32}, + {element::i32, element::i16}, + {element::i32, element::i8}, + {element::i16, element::i16}, + {element::i16, element::i8}, + {element::i8, element::i8}, + + {element::f64, element::f64}, + {element::f64, element::f32}, + {element::f64, element::f16}, + {element::f32, element::f32}, + {element::f32, element::f16}, + {element::f16, element::f16}, + + {element::f32, element::bf16}, + {element::bf16, element::bf16}, + {element::f32, element::i8}, + {element::f16, element::i8}, + {element::bf16, element::i8}, + {element::f32, element::u8}, + {element::f16, element::u8}, + {element::bf16, element::u8} + }; + + for (const auto& precisions : precisions_set) { + ASSERT_TRUE(ngraph::snippets::pass::PropagatePrecision::can_be_fused( + precisions.first, + precisions.second)) << precisions.second << " can replace " << precisions.first; + + if (precisions.first == precisions.second) { + continue; + } + + ASSERT_FALSE(ngraph::snippets::pass::PropagatePrecision::can_be_fused( + precisions.second, + precisions.first)) << precisions.second << " can not replace " << precisions.first; + } +} + +TEST_F(PrecisionPropagationConvertTest, smoke_Snippets_PrecisionPropagation_can_not_be_fused) { + const std::set> precisions_set = { + {element::i64, element::f32}, + {element::i64, element::f16}, + {element::i64, element::bf16}, + + {element::i32, element::f32}, + {element::i32, element::f16}, + {element::i32, element::bf16}, + + {element::i16, element::f16}, + {element::i16, element::bf16}, + + {element::u64, element::f32}, + {element::u64, element::f16}, + {element::u64, element::bf16}, + + {element::u32, element::f32}, + {element::u32, element::f16}, + {element::u32, element::bf16}, + + {element::u16, element::f16}, + {element::u16, element::bf16} + }; + + for (const auto& precisions : precisions_set) { + ASSERT_FALSE(ngraph::snippets::pass::PropagatePrecision::can_be_fused( + precisions.first, + precisions.second)) << precisions.second << " can not replace " << precisions.first; + } +} + +TEST_F(PrecisionPropagationConvertTest, smoke_Snippets_PrecisionPropagation_can_be_removed) { + const std::set> precisions_set = { + {element::u64, element::u64, element::u64}, + {element::u32, element::u64, element::u32}, + {element::u16, element::u64, element::u16}, + {element::u8, element::u64, element::u8}, + {element::u32, element::u32, element::u32}, + {element::u16, element::u32, element::u16}, + {element::u8, element::u32, element::u8}, + {element::u16, element::u16, element::u16}, + {element::u8, element::u16, element::u8}, + {element::u8, element::u8, element::u8}, + + {element::i64, element::i64, element::i64}, + {element::i32, element::i64, element::i32}, + {element::i16, element::i64, element::i16}, + {element::i8, element::i64, element::i8}, + {element::i32, element::i32, element::i32}, + {element::i16, element::i32, element::i16}, + {element::i8, element::i32, element::i8}, + {element::i16, element::i16, element::i16}, + {element::i8, element::i16, element::i8}, + {element::i8, element::i8, element::i8}, + + {element::f64, element::f64, element::f64}, + {element::f32, element::f64, element::f32}, + {element::f16, element::f64, element::f16}, + {element::f32, element::f32, element::f32}, + {element::f16, element::f16, element::f16}, + + {element::bf16, element::f32, element::bf16}, + {element::bf16, element::bf16, element::bf16}, + }; + + for (const auto& precisions : precisions_set) { + const auto actual_before = std::get<0>(precisions); + const auto actual_after = std::get<1>(precisions); + const auto required_after = std::get<2>(precisions); + ASSERT_TRUE(ngraph::snippets::pass::PropagatePrecision::can_be_removed( + actual_before, + actual_after, + required_after)) << "can_be_removed: " << actual_before << " => " << actual_after << " => " << required_after; + + if ((actual_before == actual_after) && (actual_before == required_after)) { + continue; + } + } +} + +} // namespace snippets +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/common/snippets/tests/src/pass/precision_propagation_get_precisions.cpp b/src/common/snippets/tests/src/pass/precision_propagation_get_precisions.cpp new file mode 100644 index 00000000000000..9e97fcc8ad4aa1 --- /dev/null +++ b/src/common/snippets/tests/src/pass/precision_propagation_get_precisions.cpp @@ -0,0 +1,45 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "snippets/pass/propagate_precision.hpp" + +namespace ov { +namespace test { +namespace snippets { + + +class PrecisionPropagationGetPrecisionsTest : public testing::Test {}; + +TEST_F(PrecisionPropagationGetPrecisionsTest, empty) { + ASSERT_EQ(std::vector{}, ngraph::snippets::pass::PropagatePrecision::get_precisions({}, {})); +} + +TEST_F(PrecisionPropagationGetPrecisionsTest, selected) { + ASSERT_EQ( + std::vector({element::f32, element::f32}), + ngraph::snippets::pass::PropagatePrecision::get_precisions( + { element::f32, element::f32 }, + { + {element::bf16, element::bf16}, + {element::f32, element::f32}, + {element::i8, element::i8}, + })); +} + +TEST_F(PrecisionPropagationGetPrecisionsTest, first) { + ASSERT_EQ( + std::vector({ element::bf16, element::bf16 }), + ngraph::snippets::pass::PropagatePrecision::get_precisions( + { element::i32, element::i32 }, + { + {element::bf16, element::bf16}, + {element::f32, element::f32}, + {element::i8, element::i8}, + })); +} + +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/common/snippets/tests/src/pass/set_scalar_count_for_load_and_store.cpp b/src/common/snippets/tests/src/pass/set_scalar_count_for_load_and_store.cpp index 9bb45a81fce7ff..50448be3a5c38f 100644 --- a/src/common/snippets/tests/src/pass/set_scalar_count_for_load_and_store.cpp +++ b/src/common/snippets/tests/src/pass/set_scalar_count_for_load_and_store.cpp @@ -19,18 +19,20 @@ using namespace ngraph; // todo: Rewrite this test using Snippets test infrastructure. See ./include/canonicalization.hpp for example -template -size_t get_count(const std::shared_ptr& f, const std::string& name) { - size_t load_count = std::numeric_limits::max(); +size_t get_count(const std::shared_ptr& f, const std::string& name, bool is_load = true) { + size_t count = std::numeric_limits::max(); for (auto op : f->get_ops()) { if (op->get_friendly_name() == name) { - load_count = ov::as_type_ptr(op)->get_count(); + if (const auto memory_access = std::dynamic_pointer_cast(op)) { + count = is_load ? memory_access->get_input_offset(0) + : memory_access->get_output_offset(0); + } } } - return load_count; + return count; } -TEST(TransformationTests, SetScalarCountForLoad) { +TEST(TransformationTests, SetScalarCountForLoadStore) { std::shared_ptr f(nullptr), f_ref(nullptr); const auto count = 16; { @@ -39,11 +41,13 @@ TEST(TransformationTests, SetScalarCountForLoad) { load->set_friendly_name("load"); auto neg = std::make_shared(load); auto store = std::make_shared(neg, count); + store->set_friendly_name("store"); f = std::make_shared(NodeVector{store}, ParameterVector{data}); pass::Manager m; m.register_pass(); m.register_pass(); + m.register_pass(); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } @@ -52,39 +56,6 @@ TEST(TransformationTests, SetScalarCountForLoad) { auto load = std::make_shared(data, 1lu); load->set_friendly_name("load_ref"); auto neg = std::make_shared(load); - auto store = std::make_shared(neg, count); - f_ref = std::make_shared(NodeVector{store}, ParameterVector{data}); - } - - auto res = compare_functions(f, f_ref); - ASSERT_TRUE(res.first) << res.second; - - auto load_count = get_count(f, "load"); - auto load_count_ref = get_count(f_ref, "load_ref"); - ASSERT_EQ(load_count, load_count_ref); -} - -TEST(TransformationTests, SetScalarCountForStore) { - std::shared_ptr f(nullptr), f_ref(nullptr); - const auto count = 16; - { - auto data = std::make_shared(element::f32, Shape{2, 2}); - auto load = std::make_shared(data, count); - auto neg = std::make_shared(load); - auto store = std::make_shared(neg, count); - store->set_friendly_name("store"); - f = std::make_shared(NodeVector{store}, ParameterVector{data}); - - pass::Manager m; - m.register_pass(); - m.register_pass(); - m.run_passes(f); - ASSERT_NO_THROW(check_rt_info(f)); - } - { - auto data = std::make_shared(element::f32, Shape{2, 2}); - auto load = std::make_shared(data, count); - auto neg = std::make_shared(load); auto store = std::make_shared(neg, 1lu); store->set_friendly_name("store_ref"); f_ref = std::make_shared(NodeVector{store}, ParameterVector{data}); @@ -93,7 +64,11 @@ TEST(TransformationTests, SetScalarCountForStore) { auto res = compare_functions(f, f_ref); ASSERT_TRUE(res.first) << res.second; - int64_t store_count = get_count(f, "store"); - int64_t store_count_ref = get_count(f_ref, "store_ref"); + auto load_count = get_count(f, "load"); + auto load_count_ref = get_count(f_ref, "load_ref"); + ASSERT_EQ(load_count, load_count_ref); + + auto store_count = get_count(f, "store", false); + auto store_count_ref = get_count(f_ref, "store_ref", false); ASSERT_EQ(store_count, store_count_ref); } diff --git a/src/common/snippets/tests/src/registers.cpp b/src/common/snippets/tests/src/registers.cpp index 531190e6048a9e..e9d7c503802142 100644 --- a/src/common/snippets/tests/src/registers.cpp +++ b/src/common/snippets/tests/src/registers.cpp @@ -13,6 +13,7 @@ #include #include "common_test_utils/ngraph_test_utils.hpp" +#include "lowering_utils.hpp" using namespace testing; using namespace ngraph; @@ -20,6 +21,7 @@ using namespace ngraph; // todo: Rewrite this test using Snippets test infrastructure. See ./include/canonicalization.hpp for example TEST(TransformationTests, AssignRegisters) { + const auto generator = std::make_shared(); std::shared_ptr f(nullptr); { auto p0 = std::make_shared(element::f32, Shape(1)); @@ -37,7 +39,12 @@ TEST(TransformationTests, AssignRegisters) { pass::Manager m; m.register_pass(); - m.register_pass(); + std::function& op)> reg_type_mapper = + [=](const std::shared_ptr& op) -> snippets::Generator::opRegType { + return generator->get_op_reg_type(op); + }; + m.register_pass(reg_type_mapper); + m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } @@ -73,6 +80,7 @@ TEST(TransformationTests, AssignRegisters) { } TEST(TransformationTests, AssignRegisters2) { + const auto generator = std::make_shared(); std::shared_ptr f(nullptr); { auto p0 = std::make_shared(ngraph::element::f32, Shape()); @@ -126,7 +134,11 @@ TEST(TransformationTests, AssignRegisters2) { pass::Manager m; m.register_pass(); - m.register_pass(); + std::function& op)> reg_type_mapper = + [=](const std::shared_ptr& op) -> snippets::Generator::opRegType { + return generator->get_op_reg_type(op); + }; + m.register_pass(reg_type_mapper); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); } diff --git a/src/common/transformations/include/ov_ops/nms_static_shape_ie.hpp b/src/common/transformations/include/ov_ops/nms_static_shape_ie.hpp index 0c4f38628dce1b..5da183db15a240 100644 --- a/src/common/transformations/include/ov_ops/nms_static_shape_ie.hpp +++ b/src/common/transformations/include/ov_ops/nms_static_shape_ie.hpp @@ -29,14 +29,9 @@ namespace internal { template class NmsStaticShapeIE : public BaseNmsOp { public: - OPENVINO_SUPPRESS_DEPRECATED_START // TODO: it should be std::string("NmsStaticShapeIE_") + BaseNmsOp::get_type_info_static().name, // but currently it does not pass conversion to Legacy Opset correctly - OPENVINO_RTTI(BaseNmsOp::get_type_info_static().name, - "ie_internal_opset", - BaseNmsOp, - BaseNmsOp::get_type_info_static().version); - OPENVINO_SUPPRESS_DEPRECATED_END + OPENVINO_RTTI(BaseNmsOp::get_type_info_static().name, "ie_internal_opset", BaseNmsOp); NmsStaticShapeIE() = default; diff --git a/src/common/transformations/include/ov_ops/type_relaxed.hpp b/src/common/transformations/include/ov_ops/type_relaxed.hpp index 47a791a4b0be42..fa84ab4eb7c945 100644 --- a/src/common/transformations/include/ov_ops/type_relaxed.hpp +++ b/src/common/transformations/include/ov_ops/type_relaxed.hpp @@ -189,10 +189,7 @@ OPENVINO_SUPPRESS_DEPRECATED_START template class TypeRelaxed : public BaseOp, public TypeRelaxedBase { public: - OPENVINO_OP(BaseOp::get_type_info_static().name, - BaseOp::get_type_info_static().version_id, - BaseOp, - BaseOp::get_type_info_static().version); + OPENVINO_OP(BaseOp::get_type_info_static().name, BaseOp::get_type_info_static().version_id, BaseOp); using BaseOp::BaseOp; diff --git a/src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp b/src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp new file mode 100644 index 00000000000000..aa5409acad635a --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ov { +namespace pass { + +/** + * @ingroup ie_transformation_common_api + * @brief ConvolutionToGroupConvolutionFusion transformation replaces following graph: + * Split (or VariadicSplit) + * / \ + * Conv ... Conv + * \ / + * \ / + * Concat + * + * to GroupConvolution + */ +class TRANSFORMATIONS_API ConvolutionToGroupConvolutionFusion : public MatcherPass { +public: + OPENVINO_RTTI("ConvolutionToGroupConvolutionFusion", "0"); + ConvolutionToGroupConvolutionFusion(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/include/transformations/common_optimizations/nonzero_fusion.hpp b/src/common/transformations/include/transformations/common_optimizations/nonzero_fusion.hpp new file mode 100644 index 00000000000000..de6623f334c36e --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/nonzero_fusion.hpp @@ -0,0 +1,28 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API NonZeroFusion; + +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief NonZeroFusion transformation makes horizontal fusion for equal NonZero layers + */ +class ov::pass::NonZeroFusion : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("NonZeroFusion", "0"); + NonZeroFusion(); +}; diff --git a/src/common/transformations/include/transformations/common_optimizations/prelu_fusion.hpp b/src/common/transformations/include/transformations/common_optimizations/prelu_fusion.hpp index 7e4939a2c5b8b8..e9b090381ec7f8 100644 --- a/src/common/transformations/include/transformations/common_optimizations/prelu_fusion.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/prelu_fusion.hpp @@ -18,6 +18,7 @@ class TRANSFORMATIONS_API PReluFusionNegativeSub; class TRANSFORMATIONS_API PReluFusionMultiplyAdd; class TRANSFORMATIONS_API PReluFusionMultiplySub; class TRANSFORMATIONS_API PReluFusionAbsSubMulMulAdd; +class TRANSFORMATIONS_API PReluFusionNegReluMulAdd; } // namespace pass } // namespace ov @@ -103,11 +104,11 @@ class ov::pass::PReluFusionMultiplySub : public ov::pass::MatcherPass { /** * @ingroup ie_transformation_common_api * @brief PReluFusionAbsSubMulMulAdd transformation replaces a sub-graph - * Op + * Op * / | \ * Relu | Abs * | \ | - * | Sub + * | Subtract * | | * | Multiply * | | @@ -121,6 +122,25 @@ class ov::pass::PReluFusionAbsSubMulMulAdd : public ov::pass::MatcherPass { PReluFusionAbsSubMulMulAdd(); }; +/** + * @ingroup ie_transformation_common_api + * @brief PReluFusionNegReluMulAdd transformation replaces a sub-graph + * Op + * / \ + * Relu Negative + * | | + * | Relu + * | | + * | Multiply + * \ / + * Add + */ +class ov::pass::PReluFusionNegReluMulAdd : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("PReluFusionNegReluMulAdd", "0"); + PReluFusionNegReluMulAdd(); +}; + /** * @ingroup ie_transformation_common_api * @brief PReluFusion transformation replaces various sub-graphs with a PRelu op. @@ -134,5 +154,6 @@ class ov::pass::PReluFusion : public ov::pass::GraphRewrite { add_matcher(); add_matcher(); add_matcher(); + add_matcher(); } }; diff --git a/src/common/transformations/include/transformations/common_optimizations/softmax_fusion.hpp b/src/common/transformations/include/transformations/common_optimizations/softmax_fusion.hpp index 965c6a9747564f..337bb65bd21630 100644 --- a/src/common/transformations/include/transformations/common_optimizations/softmax_fusion.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/softmax_fusion.hpp @@ -17,7 +17,7 @@ class TRANSFORMATIONS_API SoftmaxFusion; /** * @ingroup ie_transformation_common_api - * @brief SoftmaxFusion transformation replaces following graph: + * @brief SoftmaxFusion transformation replaces following graphs: * * +---------------+ * │ │ @@ -63,6 +63,38 @@ class TRANSFORMATIONS_API SoftmaxFusion; * │ │ * +-------------+ * + * and + * +---------------+ + * │ │ + * │ input │ + * │ │ + * +---------------+ + * | + * | + * | + * v + * +---------------+ + * │ │ + * │ Exp │ + * │ │ + * +---------------+ + * │ │ + * │ v + * │ +-----------+ + * │ │ │ + * │ │ ReduceSum │ + * │ │ │ + * │ +-----------+ + * │ │ + * │ │ + * v v + * +-------------+ + * | │ + * | Div │ + * │ │ + * +-------------+ + * + * * to a single Softmax node * * * Restrictions: diff --git a/src/common/transformations/include/transformations/op_conversions/convert_interpolate11_downgrade.hpp b/src/common/transformations/include/transformations/op_conversions/convert_interpolate11_downgrade.hpp new file mode 100644 index 00000000000000..b112c5d8abdf45 --- /dev/null +++ b/src/common/transformations/include/transformations/op_conversions/convert_interpolate11_downgrade.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ov { +namespace pass { +/** + * @ingroup ie_transformation_common_api + * @brief Converts Interpolate version 11 to Interpolate version 4 if the new op uses any of the v4 allowed + * interpolation modes. + */ +class TRANSFORMATIONS_API ConvertInterpolate11ToInterpolate4 : public MatcherPass { +public: + OPENVINO_RTTI("ConvertInterpolate11ToInterpolate4", "0"); + ConvertInterpolate11ToInterpolate4(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/include/transformations/transpose_sinking/ts_general.hpp b/src/common/transformations/include/transformations/transpose_sinking/ts_general.hpp index 8199f4c378dc9a..18b080d4b10e27 100644 --- a/src/common/transformations/include/transformations/transpose_sinking/ts_general.hpp +++ b/src/common/transformations/include/transformations/transpose_sinking/ts_general.hpp @@ -16,6 +16,9 @@ class TRANSFORMATIONS_API TSGeneralBackward; class TRANSFORMATIONS_API TSGeneral; } // namespace transpose_sinking + +using TransposeSinkingGeneral = ov::pass::transpose_sinking::TSGeneral; + } // namespace pass } // namespace ov diff --git a/src/common/transformations/include/transformations/transpose_sinking/ts_reduction.hpp b/src/common/transformations/include/transformations/transpose_sinking/ts_reduction.hpp index 6f462875b7f6ba..7f7eeb9e270699 100644 --- a/src/common/transformations/include/transformations/transpose_sinking/ts_reduction.hpp +++ b/src/common/transformations/include/transformations/transpose_sinking/ts_reduction.hpp @@ -21,7 +21,7 @@ class TRANSFORMATIONS_API TSReductionBackward; /** * @ingroup ie_transformation_common_api - * @brief TransposeReductionForward transformation sinks Transpose through Reduce, Squeeze, Unsqueeze operations + * @brief TransposeReductionForward transformation sinks Transpose through Reduce operations * in the forward direction. */ class ov::pass::transpose_sinking::TSReductionForward : public ov::pass::MatcherPass { @@ -32,7 +32,7 @@ class ov::pass::transpose_sinking::TSReductionForward : public ov::pass::Matcher /** * @ingroup ie_transformation_common_api - * @brief TransposeReductionBackward transformation sinks Transpose through Reduce, Squeeze, Unsqueeze operations + * @brief TransposeReductionBackward transformation sinks Transpose through Reduce operations * in the backward direction. */ class ov::pass::transpose_sinking::TSReductionBackward : public ov::pass::MatcherPass { diff --git a/src/common/transformations/include/transformations/transpose_sinking/ts_slice.hpp b/src/common/transformations/include/transformations/transpose_sinking/ts_slice.hpp new file mode 100644 index 00000000000000..a5a135d44b39e2 --- /dev/null +++ b/src/common/transformations/include/transformations/transpose_sinking/ts_slice.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2022-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { +namespace transpose_sinking { + +class TRANSFORMATIONS_API TSSliceForward; +class TRANSFORMATIONS_API TSSliceBackward; + +} // namespace transpose_sinking +} // namespace pass +} // namespace ov + +class ov::pass::transpose_sinking::TSSliceForward : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ov::pass::TSSliceForward", "0"); + TSSliceForward(); +}; + +class ov::pass::transpose_sinking::TSSliceBackward : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ov::pass::TSSliceBackward", "0"); + TSSliceBackward(); +}; diff --git a/src/common/transformations/include/transformations/transpose_sinking/ts_squeeze.hpp b/src/common/transformations/include/transformations/transpose_sinking/ts_squeeze.hpp new file mode 100644 index 00000000000000..c7aa6f2aa0fbc5 --- /dev/null +++ b/src/common/transformations/include/transformations/transpose_sinking/ts_squeeze.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2022-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { +namespace transpose_sinking { + +class TRANSFORMATIONS_API TSSqueezeForward; +class TRANSFORMATIONS_API TSSqueezeBackward; + +} // namespace transpose_sinking +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief TSSqueezeForward transformation sinks Transpose through Reshape, Squeeze operations + * in the forward direction. + */ +class ov::pass::transpose_sinking::TSSqueezeForward : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ov::pass::TSSqueezeForward", "0"); + TSSqueezeForward(); +}; + +/** + * @ingroup ie_transformation_common_api + * @brief TSSqueezeBackward transformation sinks Transpose through Reshape, Squeeze operations + * in the backward direction. + */ +class ov::pass::transpose_sinking::TSSqueezeBackward : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ov::pass::TSSqueezeBackward", "0"); + TSSqueezeBackward(); +}; diff --git a/src/common/transformations/include/transformations/transpose_sinking/ts_unsqueeze.hpp b/src/common/transformations/include/transformations/transpose_sinking/ts_unsqueeze.hpp new file mode 100644 index 00000000000000..05150bfe1fb58d --- /dev/null +++ b/src/common/transformations/include/transformations/transpose_sinking/ts_unsqueeze.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2022-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { +namespace transpose_sinking { + +class TRANSFORMATIONS_API TSUnsqueezeForward; +class TRANSFORMATIONS_API TSUnsqueezeBackward; + +} // namespace transpose_sinking +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief TSUnsqueezeForward transformation sinks Transpose through Unsqueeze, Reshape operations + * in the forward direction. + */ +class ov::pass::transpose_sinking::TSUnsqueezeForward : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ov::pass::TSUnsqueezeForward", "0"); + TSUnsqueezeForward(); +}; + +/** + * @ingroup ie_transformation_common_api + * @brief TSUnsqueezeBackward transformation sinks Transpose through Unsqueeze, Reshape operations + * in the backward direction. + */ +class ov::pass::transpose_sinking::TSUnsqueezeBackward : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ov::pass::TSUnsqueezeBackward", "0"); + TSUnsqueezeBackward(); +}; diff --git a/src/common/transformations/include/transformations/transpose_sinking/ts_utils.hpp b/src/common/transformations/include/transformations/transpose_sinking/ts_utils.hpp index 7c14886f893cd2..78e03d7af13364 100644 --- a/src/common/transformations/include/transformations/transpose_sinking/ts_utils.hpp +++ b/src/common/transformations/include/transformations/transpose_sinking/ts_utils.hpp @@ -98,17 +98,43 @@ void UpdateForwardSinkingAbility(const std::shared_ptr&); bool HasSameOutputTransposeNodes(const ov::Output&); /** - * Removes all direct node consumers that have one output + * @brief Removes all direct node consumers that have one output */ void RemoveSingleOutputConsumers(const std::shared_ptr&); /** - * Changes the order of values in @arg input according to @arg transpose_axis_order along @arg axis + * @brief Changes the order of values in @arg input according to @arg transpose_axis_order along @arg axis */ ov::Output ChangeValuesOrder(const ov::Output& input, const ov::AxisVector& transpose_axis_order, const std::shared_ptr& axis); +/** + * @brief Returns the updated axes order for case when the initial axes order has more elements + * than after TransposeSinking, e.g.: + * + * before: Transpose(the initial axes order) -> ReduceMax + * after : ReduceMax -> Transpose (the updated axes order) + * + * before: Unsqueeze -> Transpose (the initial axes order) + * after : Transpose (the updated axes order) -> Unsqueeze + */ +std::vector GetOrderAfterReduction(const std::vector& axes_values, + const std::vector& order_values); + +/** + * @brief Returns the updated axes order for case when the initial axes order has less elements + * than after TransposeSinking, e.g.: + * + * before : ReduceMax -> Transpose (the updated axes order) + * after: Transpose(the initial axes order) -> ReduceMax + * + * before: Transpose (the updated axes order) -> Unsqueeze + * after : Unsqueeze -> Transpose (the initial axes order) + */ +std::vector GetOrderBeforeReduction(const std::vector& axes_values, + const std::vector& order_values); + } // namespace utils } // namespace transpose_sinking } // namespace pass diff --git a/src/common/transformations/src/ov_ops/augru_cell.cpp b/src/common/transformations/src/ov_ops/augru_cell.cpp index ba9179b60317bf..c98d9f77b6b78a 100644 --- a/src/common/transformations/src/ov_ops/augru_cell.cpp +++ b/src/common/transformations/src/ov_ops/augru_cell.cpp @@ -63,7 +63,9 @@ void ov::op::internal::AUGRUCell::validate_and_infer_types() { "Element types for inputs do not match."); // Get input partial shape for all inputs + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes = {ov::PartialShape::dynamic(2)}; shape_infer(this, input_shapes, output_shapes); diff --git a/src/common/transformations/src/ov_ops/augru_sequence.cpp b/src/common/transformations/src/ov_ops/augru_sequence.cpp index 1ff7f41b3e8216..f83b8028474536 100644 --- a/src/common/transformations/src/ov_ops/augru_sequence.cpp +++ b/src/common/transformations/src/ov_ops/augru_sequence.cpp @@ -61,7 +61,9 @@ void ov::op::internal::AUGRUSequence::validate_and_infer_types() { element::Type::merge(result_et, result_et, get_input_element_type(6)), "Element types for inputs do not match."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes = {ov::PartialShape::dynamic(4), ov::PartialShape::dynamic(3)}; shape_infer(this, input_shapes, output_shapes); diff --git a/src/common/transformations/src/transformations/common_optimizations/add_fake_quantize_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/add_fake_quantize_fusion.cpp index 38d4e20e360f25..4f18d4a8d33707 100644 --- a/src/common/transformations/src/transformations/common_optimizations/add_fake_quantize_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/add_fake_quantize_fusion.cpp @@ -105,11 +105,15 @@ ov::pass::AddFakeQuantizeFusion::AddFakeQuantizeFusion() { } auto input_low_sub = std::make_shared(fq->input_value(1), new_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_input_low = get_constant_from_source(input_low_sub); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_input_low) new_input_low = input_low_sub; auto input_high_sub = std::make_shared(fq->input_value(2), new_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_input_high = get_constant_from_source(input_high_sub); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_input_high) new_input_high = input_high_sub; auto new_fq = diff --git a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp index 8b43dcfc8d2b29..6064effe880c4b 100644 --- a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -76,6 +76,7 @@ #include "transformations/op_conversions/convert_gather_downgrade.hpp" #include "transformations/op_conversions/convert_gather_upgrade.hpp" #include "transformations/op_conversions/convert_gelu.hpp" +#include "transformations/op_conversions/convert_interpolate11_downgrade.hpp" #include "transformations/op_conversions/convert_interpolate1_to_interpolate4.hpp" #include "transformations/op_conversions/convert_maxpool_downgrade.hpp" #include "transformations/op_conversions/convert_maxpool_upgrade.hpp" @@ -211,6 +212,7 @@ bool ov::pass::CommonOptimizations::run_on_model(const std::shared_ptr(); ADD_MATCHER(fq_fusions, FakeQuantizeMulFusion) diff --git a/src/common/transformations/src/transformations/common_optimizations/conv_to_binary_conv.cpp b/src/common/transformations/src/transformations/common_optimizations/conv_to_binary_conv.cpp index 613f31f60328d4..4fe14a0df81346 100644 --- a/src/common/transformations/src/transformations/common_optimizations/conv_to_binary_conv.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/conv_to_binary_conv.cpp @@ -112,7 +112,9 @@ ov::pass::ConvToBinaryConv::ConvToBinaryConv() { weights_reduced, opset5::Constant::create(element::i64, Shape{weights_reduced_shape.size()}, weights_reduced_shape), false); + OPENVINO_SUPPRESS_DEPRECATED_START weights_reduced_reshaped = ngraph::get_constant_from_source(weights_reduced_reshaped); + OPENVINO_SUPPRESS_DEPRECATED_END auto add = std::make_shared(new_conv, weights_reduced_reshaped); auto mul = std::make_shared(add, opset5::Constant::create(element::f32, Shape{}, {0.5})); copy_runtime_info(conv, {new_conv, add, mul}); diff --git a/src/common/transformations/src/transformations/common_optimizations/convert_quantize_dequantize.cpp b/src/common/transformations/src/transformations/common_optimizations/convert_quantize_dequantize.cpp index 49aa008ce1718b..65be131c1f36c0 100644 --- a/src/common/transformations/src/transformations/common_optimizations/convert_quantize_dequantize.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/convert_quantize_dequantize.cpp @@ -151,10 +151,14 @@ ov::pass::ConvertQuantizeDequantize::ConvertQuantizeDequantize() { if (out_high_shape.rank().is_dynamic() || out_high_shape.rank().get_length() > data_shape.rank().get_length()) return false; + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr const_out_low = get_constant_from_source(new_out_low); + OPENVINO_SUPPRESS_DEPRECATED_END if (const_out_low) new_out_low = const_out_low; + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr const_out_high = get_constant_from_source(new_out_high); + OPENVINO_SUPPRESS_DEPRECATED_END if (const_out_high) new_out_high = const_out_high; diff --git a/src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp b/src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp new file mode 100644 index 00000000000000..835d62cc56f42c --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/convolution_to_group_convolution.cpp @@ -0,0 +1,164 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include + +#include "itt.hpp" +#include "transformations/common_optimizations/convolution_to_group_convolution_fusion.hpp" + +static bool compare_convolutions(const ov::opset10::Convolution* conv1, ov::Node* node) { + const auto conv2 = ov::as_type(node); + if (!conv2) + return false; + return conv1->get_strides() == conv2->get_strides() && conv1->get_pads_begin() == conv2->get_pads_begin() && + conv1->get_pads_end() == conv2->get_pads_end() && conv1->get_dilations() == conv2->get_dilations() && + conv1->get_auto_pad() == conv2->get_auto_pad(); +} + +static int64_t get_split_axis(const std::shared_ptr& split) { + const auto axis = ov::as_type(split->get_input_node_ptr(1)); + if (!axis) + return -1; + auto axis_value = axis->cast_vector()[0]; + if (axis_value < 0) { + const auto& input_rank = split->get_input_partial_shape(0).rank(); + if (input_rank.is_dynamic()) + return -1; + axis_value += input_rank.get_length(); + } + + return axis_value; +} + +static std::shared_ptr create_new_weights(ov::pass::NodeRegistry& node_registry, + const std::shared_ptr& concat) { + const auto concat_input = concat->get_input_node_ptr(0); + if (concat_input->get_input_partial_shape(1).is_dynamic()) + return nullptr; + + // unsqueeze weights shape from (O, I, X, Y) to (1, O, I, X, Y) + const auto& weights_shape = concat_input->get_input_shape(1); + ov::Shape new_shape = weights_shape; + new_shape.insert(new_shape.begin(), 1); + + const size_t num_inputs = concat->get_input_size(); + ov::OutputVector weights_to_concat; + weights_to_concat.reserve(num_inputs); + + for (size_t i = 0; i < num_inputs; i++) { + const auto conv = concat->get_input_node_shared_ptr(i); + const auto weights = conv->get_input_node_shared_ptr(1); + const auto& shape = weights->get_output_partial_shape(0); + if (shape.is_dynamic() || weights->get_output_shape(0) != weights_shape) + return nullptr; + if (auto constant = ov::as_type_ptr(weights)) { + weights_to_concat.push_back(node_registry.make(*constant, new_shape)); + } else { + weights_to_concat.push_back(node_registry.make( + weights, + ov::opset10::Constant::create(ov::element::i32, ov::Shape{}, {0}))); + } + weights_to_concat.back().get_node()->set_friendly_name(weights->get_friendly_name()); + } + + return node_registry.make(weights_to_concat, 0); +} + +ov::pass::ConvolutionToGroupConvolutionFusion::ConvolutionToGroupConvolutionFusion() { + MATCHER_SCOPE(ConvolutionToGroupConvolutionFusion); + + auto has_conv_inputs = [](const Output& node) -> bool { + const auto concat = node.get_node(); + size_t num_inputs = concat->get_input_size(); + if (num_inputs == 0) + return false; + + const auto first_conv = as_type(concat->get_input_node_ptr(0)); + if (!first_conv) + return false; + + const auto split = first_conv->get_input_node_ptr(0); + if (!is_type(split) && !is_type(split)) + return false; + + // go through Concat inputs and check + // - if all of them are Convolutions + // - if those Convolutions have the same Split input + for (size_t i = 1; i < concat->get_input_size(); i++) { + const auto conv = concat->get_input_node_ptr(i); + if (conv->get_input_node_ptr(0) != split) + return false; + if (!compare_convolutions(first_conv, conv)) + return false; + } + return true; + }; + auto concat_label = pattern::wrap_type(has_conv_inputs); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& pattern_value_map = m.get_pattern_value_map(); + const auto& concat = pattern_value_map.at(concat_label).get_node_shared_ptr(); + + const auto first_conv = as_type_ptr(concat->get_input_node_shared_ptr(0)); + const auto split = first_conv->get_input_node_shared_ptr(0); + const bool is_split = is_type(split); + const bool is_variadic_split = is_type(split); + if (!is_split && !is_variadic_split) + return false; + + if (get_split_axis(split) != 1) + return false; + + if (is_variadic_split) { + // split_lengths in VariadicSplit must have the same values + if (auto split_lengths = as_type(split->get_input_node_ptr(1))) { + const auto split_lengths_values = split_lengths->cast_vector(); + const auto first_length = split_lengths_values[0]; + if (!std::all_of(split_lengths_values.begin() + 1, + split_lengths_values.end(), + [first_length](int split_length) { + return split_length == first_length; + })) + return false; + } else { + return false; + } + } + + NodeRegistry node_registry; + const auto weights = create_new_weights(node_registry, concat); + if (!weights) + return false; + + const auto conv = node_registry.make(split->get_input_node_shared_ptr(0), + weights, + first_conv->get_strides(), + first_conv->get_pads_begin(), + first_conv->get_pads_end(), + first_conv->get_dilations(), + first_conv->get_auto_pad()); + conv->set_friendly_name(concat->get_friendly_name()); + register_new_node(conv); + + const size_t concat_num_inputs = concat->get_input_size(); + NodeVector from; + from.reserve(concat_num_inputs + 2); + from.push_back(split); + from.push_back(first_conv); + for (size_t i = 1; i < concat_num_inputs; i++) { + from.push_back(concat->get_input_node_shared_ptr(i)); + } + from.push_back(concat); + + copy_runtime_info(from, node_registry.get()); + replace_node(concat, conv); + + return true; + }; + + auto m = std::make_shared(concat_label, matcher_name); + this->register_matcher(m, callback); +} diff --git a/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp index 3d00d41c0ea2c5..69e1f023b1ff6b 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fq_mul_fusion.cpp @@ -102,7 +102,9 @@ ov::pass::FakeQuantizeMulFusion::FakeQuantizeMulFusion() { auto get_adjusted_output_range = [&](const Output& node) -> std::shared_ptr { auto ret = std::make_shared(node, mul_constant); copy_runtime_info(node.get_node_shared_ptr(), ret); + OPENVINO_SUPPRESS_DEPRECATED_START auto constant = get_constant_from_source(ret); + OPENVINO_SUPPRESS_DEPRECATED_END if (constant) return constant; return ret; @@ -113,7 +115,9 @@ ov::pass::FakeQuantizeMulFusion::FakeQuantizeMulFusion() { fq_node->input_value(2), get_adjusted_output_range(original_output_low), get_adjusted_output_range(original_output_high)}); + OPENVINO_SUPPRESS_DEPRECATED_START bool fq_on_weights = is_type(data.get_node()) || get_constant_from_source(data) != nullptr; + OPENVINO_SUPPRESS_DEPRECATED_END if (!fq_on_weights && transformation_callback(new_fq_node)) return false; diff --git a/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp b/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp index 1cf7f2fdee8576..157bce8df83a4a 100644 --- a/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp @@ -337,18 +337,18 @@ bool MarkSugraphsToKeepInMixedPrecision::run_on_model(const shared_ptr(); ADD_MATCHER(propagate_up, PropagateUpMarkToKeepInMixedPrecision) // Mark nodes in ShapeOf subgraphs to keep in FP32 REGISTER_PASS(manager, MarkPrecisionSensitiveShapeOfSubgraphs) + REGISTER_PASS(manager, MarkNormalizationOps) manager.run_passes(m); for (auto& node : m->get_ops()) { diff --git a/src/common/transformations/src/transformations/common_optimizations/matmul_const_transposes_extraction.cpp b/src/common/transformations/src/transformations/common_optimizations/matmul_const_transposes_extraction.cpp index 8d8b8cdbb18156..fd77e9b90ff75d 100644 --- a/src/common/transformations/src/transformations/common_optimizations/matmul_const_transposes_extraction.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/matmul_const_transposes_extraction.cpp @@ -34,8 +34,11 @@ ov::pass::MatMulConstTransposesExtraction::MatMulConstTransposesExtraction() { weights, opset8::Constant::create(element::i32, {transpose_order.size()}, transpose_order)); if (ov::is_type(weights.get_node())) { - if (auto constant = get_constant_from_source(transpose)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = get_constant_from_source(transpose)) { + OPENVINO_SUPPRESS_DEPRECATED_END transpose = constant; + } } auto new_matmul = std::make_shared(pattern_value_map.at(data_pattern), transpose, diff --git a/src/common/transformations/src/transformations/common_optimizations/matmul_multiply_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/matmul_multiply_fusion.cpp index 7208ec88d31c64..8772f5d00f0b1c 100644 --- a/src/common/transformations/src/transformations/common_optimizations/matmul_multiply_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/matmul_multiply_fusion.cpp @@ -130,7 +130,9 @@ static std::shared_ptr fuse_const_to_weights(const std::shared_ptr& auto transpose = std::make_shared(new_const, opset8::Constant::create(element::i64, Shape{perm.size()}, perm)); + OPENVINO_SUPPRESS_DEPRECATED_START return get_constant_from_source(transpose); + OPENVINO_SUPPRESS_DEPRECATED_END }; // If weights meant to be transposed - we need to also transpose constant @@ -168,7 +170,9 @@ pass::MatMulMultiplyFusion::MatMulMultiplyFusion() { // Constantfold new weights, only if old weights is a constant node. // To make sure that subgraphs with e.g. FakeQuantize don't get constant folded here. if (ov::is_type(weights.get_node())) { + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = get_constant_from_source(new_weights)) { + OPENVINO_SUPPRESS_DEPRECATED_END new_weights = constant; } } diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index f394f7d037d26c..20f2ede4454786 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -188,6 +190,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptrset_name("ov::pass::CommonFusions"); REGISTER_PASS(manager, BinarizeWeights) @@ -236,7 +240,6 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr(weights, mul_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_weights = get_constant_from_source(weights_multiply); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_weights) new_weights = weights_multiply; @@ -112,7 +114,9 @@ ov::pass::MultiplyGroupConvolutionFusion::MultiplyGroupConvolutionFusion() { } auto weights_multiply = std::make_shared(weights, mul_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_weights = get_constant_from_source(weights_multiply); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_weights) new_weights = weights_multiply; @@ -179,7 +183,9 @@ ov::pass::MultiplyConvolutionBackpropDataFusion::MultiplyConvolutionBackpropData } auto weights_multiply = std::make_shared(weights, mul_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_weights = get_constant_from_source(weights_multiply); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_weights) new_weights = weights_multiply; @@ -249,7 +255,9 @@ ov::pass::MultiplyGroupConvolutionBackpropDataFusion::MultiplyGroupConvolutionBa } auto weights_multiply = std::make_shared(weights, mul_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_weights = get_constant_from_source(weights_multiply); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_weights) new_weights = weights_multiply; diff --git a/src/common/transformations/src/transformations/common_optimizations/mul_fake_quantize_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/mul_fake_quantize_fusion.cpp index 605888a3da8236..9538d1a3d7f01c 100644 --- a/src/common/transformations/src/transformations/common_optimizations/mul_fake_quantize_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/mul_fake_quantize_fusion.cpp @@ -99,11 +99,15 @@ ov::pass::MulFakeQuantizeFusion::MulFakeQuantizeFusion() { } auto input_low_div = std::make_shared(fq->input_value(1), new_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_input_low = get_constant_from_source(input_low_div); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_input_low) new_input_low = input_low_div; auto input_high_div = std::make_shared(fq->input_value(2), new_const); + OPENVINO_SUPPRESS_DEPRECATED_START std::shared_ptr new_input_high = get_constant_from_source(input_high_div); + OPENVINO_SUPPRESS_DEPRECATED_END if (!new_input_high) new_input_high = input_high_div; diff --git a/src/common/transformations/src/transformations/common_optimizations/nonzero_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/nonzero_fusion.cpp new file mode 100644 index 00000000000000..5eaebe4aac584c --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/nonzero_fusion.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/nonzero_fusion.hpp" + +#include +#include +#include +#include + +#include "itt.hpp" +#include "transformations/utils/utils.hpp" + +ov::pass::NonZeroFusion::NonZeroFusion() { + MATCHER_SCOPE(NonZeroFusion); + auto input_m = pass::pattern::any_input(ov::pass::pattern::consumers_more_than(1)); + auto nonzero_m = pass::pattern::wrap_type({input_m}); + + ov::matcher_pass_callback callback = [=](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + const auto nonzero = ov::as_type_ptr(pattern_map.at(nonzero_m).get_node_shared_ptr()); + const auto out_prc = nonzero->get_output_type(); + + bool status = false; + auto replace_if_nodes_match = [&](const ov::Input& in) { + auto cur_nonzero = ov::as_type_ptr(in.get_node()->shared_from_this()); + if (cur_nonzero && cur_nonzero->get_output_type() == out_prc) { + status |= ov::replace_output_update_name(cur_nonzero->output(0), nonzero->output(0)); + } + }; + + const auto consumers = pattern_map.at(input_m).get_target_inputs(); + std::for_each(consumers.begin(), consumers.end(), replace_if_nodes_match); + return status; + }; + + auto m = std::make_shared(nonzero_m, matcher_name); + register_matcher(m, callback); +} diff --git a/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp b/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp index 8ac258993f2085..52acd6c4628d20 100644 --- a/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,7 @@ using namespace ov; //`simplify_gather`, optimizes gather if Gather is gathering the // whole input tensor static bool simplify_gather(shared_ptr node) { - if (auto gather = ov::as_type_ptr(node)) { + if (auto gather = ov::as_type_ptr(node)) { // check if we are gathering the whole input auto data = gather->input_value(0); auto indices = gather->input_value(1); @@ -34,10 +35,6 @@ static bool simplify_gather(shared_ptr node) { if (data.get_partial_shape().is_dynamic() || indices.get_partial_shape().is_dynamic()) { return false; } - // if rank of data and gather output dont match, we will skip - if (data.get_shape().size() != node->get_shape().size()) { - return false; - } auto axis = gather->get_axis(); if (axis == opset3::Gather::AXIS_NOT_SET_VALUE) { @@ -45,6 +42,22 @@ static bool simplify_gather(shared_ptr node) { return false; } + if (data.get_shape().size() != node->get_shape().size()) { + auto constant_indices = ov::as_type_ptr(gather->input_value(1).get_node_shared_ptr()); + if (!constant_indices) + return false; + // case_3: if input_shape is (1,3,5,5) and axis = 0, indices = 0, then gather is just a Squeeze + const auto const_indices = constant_indices->cast_vector(); + if (data.get_shape()[axis] == 1 && const_indices.size() == 1 && const_indices[0] == 0) { + auto squeeze = std::make_shared(gather->input_value(0), gather->input_value(2)); + squeeze->set_friendly_name(gather->get_friendly_name()); + ov::copy_runtime_info(gather, squeeze); + ov::replace_node(gather, squeeze); + return true; + } + return false; + } + // case_1 : if the input tensor is of shape (4, 1, 4) // and axis = 1, then the gather would be simply // gathering the whole input tensor, so we can optimize this @@ -297,7 +310,7 @@ static bool eliminate_unsqueeze(const shared_ptr& node) { SIMPLE_MATCHER_PASS_DEFINITION(EliminateReshape, eliminate_reshape_v1, opset3::Reshape); SIMPLE_MATCHER_PASS_DEFINITION(EliminateUnsqueeze, eliminate_unsqueeze, opset3::Unsqueeze); SIMPLE_MATCHER_PASS_DEFINITION(EliminateBroadcast, eliminate_nop, op::v1::Broadcast, op::v3::Broadcast); -SIMPLE_MATCHER_PASS_DEFINITION(EliminateGather, simplify_gather, opset3::Gather); +SIMPLE_MATCHER_PASS_DEFINITION(EliminateGather, simplify_gather, opset3::Gather, opset7::Gather, opset8::Gather); pass::EliminatePad::EliminatePad() { MATCHER_SCOPE(EliminatePad); @@ -306,8 +319,10 @@ pass::EliminatePad::EliminatePad() { matcher_pass_callback callback = [=](pattern::Matcher& m) { auto pad = m.get_match_root(); + OPENVINO_SUPPRESS_DEPRECATED_START auto pad_begin_const = get_constant_from_source(pad->input_value(1)); auto pad_end_const = get_constant_from_source(pad->input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END if (!pad_begin_const || !pad_end_const) { return false; diff --git a/src/common/transformations/src/transformations/common_optimizations/pad_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/pad_fusion.cpp index f9b59e44de8a64..dd71c408907af5 100644 --- a/src/common/transformations/src/transformations/common_optimizations/pad_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/pad_fusion.cpp @@ -27,7 +27,9 @@ static bool can_be_fused(const std::shared_ptr& pad, if (!node) return false; + OPENVINO_SUPPRESS_DEPRECATED_START auto pad_value_const = ov::get_constant_from_source(pad_value_node); + OPENVINO_SUPPRESS_DEPRECATED_END if (!pad_value_const) return false; auto pad_value = pad_value_const->cast_vector()[0]; diff --git a/src/common/transformations/src/transformations/common_optimizations/prelu_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/prelu_fusion.cpp index 98474c2ad9ad18..4a2ed9729ec53d 100644 --- a/src/common/transformations/src/transformations/common_optimizations/prelu_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/prelu_fusion.cpp @@ -201,3 +201,38 @@ ov::pass::PReluFusionAbsSubMulMulAdd::PReluFusionAbsSubMulMulAdd() { auto m = make_shared(add, matcher_name); register_matcher(m, callback); } + +ov::pass::PReluFusionNegReluMulAdd::PReluFusionNegReluMulAdd() { + MATCHER_SCOPE(PReluFusionNegReluMulAdd); + + using namespace std; + using namespace ov; + using namespace ov::opset10; + + const auto input = pass::pattern::any_input(); + const auto relu_pos = pattern::wrap_type({input}); + const auto neg1 = pattern::wrap_type({input}); + const auto relu_neg = pattern::wrap_type({neg1}); + const auto mul_constant = pattern::wrap_type(); + const auto mul = pattern::wrap_type({relu_neg, mul_constant}); + const auto add = pattern::wrap_type({relu_pos, mul}); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& pattern_to_output = m.get_pattern_value_map(); + const auto input_output = pattern_to_output.at(input); + const auto add_node = pattern_to_output.at(add).get_node_shared_ptr(); + const auto slope = op::util::make_try_fold(pattern_to_output.at(mul_constant)); + const auto prelu = make_shared(input_output, slope); + prelu->set_friendly_name(m.get_match_root()->get_friendly_name()); + NodeVector copy_from = {pattern_to_output.at(relu_pos).get_node_shared_ptr(), + pattern_to_output.at(neg1).get_node_shared_ptr(), + pattern_to_output.at(relu_neg).get_node_shared_ptr(), + pattern_to_output.at(mul).get_node_shared_ptr(), + pattern_to_output.at(add).get_node_shared_ptr()}; + copy_runtime_info(copy_from, prelu); + replace_node(add_node, prelu); + return true; + }; + auto matcher = make_shared(add, matcher_name); + register_matcher(matcher, callback); +} diff --git a/src/common/transformations/src/transformations/common_optimizations/pull_through_reduce.cpp b/src/common/transformations/src/transformations/common_optimizations/pull_through_reduce.cpp index 5a88b007341d1d..db0c3cf3852574 100644 --- a/src/common/transformations/src/transformations/common_optimizations/pull_through_reduce.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/pull_through_reduce.cpp @@ -128,9 +128,11 @@ ov::pass::PullUnsqueezeThroughReduce::PullUnsqueezeThroughReduce() { } auto unsqueeze_axes_val = unsqueeze_axes_input->cast_vector(); + OPENVINO_SUPPRESS_DEPRECATED_START normalize_axes(unsqueeze_node.get(), unsqueeze_node->get_output_partial_shape(0).rank().get_length(), unsqueeze_axes_val); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduce_axes_val = reduce_node->get_reduction_axes().to_vector(); if (have_same_axes(unsqueeze_axes_val, reduce_axes_val)) { diff --git a/src/common/transformations/src/transformations/common_optimizations/pull_transpose_through_fq.cpp b/src/common/transformations/src/transformations/common_optimizations/pull_transpose_through_fq.cpp index 699444a81af7b0..804a07f9e6e6e7 100644 --- a/src/common/transformations/src/transformations/common_optimizations/pull_transpose_through_fq.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/pull_transpose_through_fq.cpp @@ -62,7 +62,9 @@ ov::pass::PullTransposeThroughFQUp::PullTransposeThroughFQUp() { new_ops.push_back(fq_input.get_node_shared_ptr()); } fq_input = std::make_shared(fq_input, transpose->input_value(1)); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = get_constant_from_source(fq_input)) { + OPENVINO_SUPPRESS_DEPRECATED_END fq_input = constant; } ngraph::copy_runtime_info(transpose, fq_input.get_node_shared_ptr()); diff --git a/src/common/transformations/src/transformations/common_optimizations/push_constant_to_subgraph.cpp b/src/common/transformations/src/transformations/common_optimizations/push_constant_to_subgraph.cpp index 21e636992067d5..2289ce74bc73c2 100644 --- a/src/common/transformations/src/transformations/common_optimizations/push_constant_to_subgraph.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/push_constant_to_subgraph.cpp @@ -4,10 +4,10 @@ #include "transformations/common_optimizations/push_constant_to_subgraph.hpp" -#include -#include - #include "itt.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/util/multi_subgraph_base.hpp" +#include "validation_util.hpp" using MultiSubGraphOp = ov::op::util::MultiSubGraphOp; @@ -21,7 +21,7 @@ static std::shared_ptr try_constantfold_input( const auto input_index = input_desc->m_input_index; auto it = cache.find(input_index); if (it == cache.end()) { - auto constant = constantfold_subgraph(op->input_value(input_index)); + auto constant = ov::util::constantfold_subgraph(op->input_value(input_index)); if (constant) { cache.insert({input_index, constant}); } diff --git a/src/common/transformations/src/transformations/common_optimizations/random_uniform_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/random_uniform_fusion.cpp index 4137f87a9c8176..9ef43710558580 100644 --- a/src/common/transformations/src/transformations/common_optimizations/random_uniform_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/random_uniform_fusion.cpp @@ -59,8 +59,10 @@ ov::pass::RandomUniformFusion::RandomUniformFusion() { const auto new_mul_add1 = mul_add_ptr->clone_with_new_inputs({ru->input_value(1), new_const}); const auto new_mul_add2 = mul_add_ptr->clone_with_new_inputs({ru->input_value(2), new_const}); + OPENVINO_SUPPRESS_DEPRECATED_START const auto& folded_const1 = ngraph::get_constant_from_source(new_mul_add1); const auto& folded_const2 = ngraph::get_constant_from_source(new_mul_add2); + OPENVINO_SUPPRESS_DEPRECATED_END const auto new_ru = ru->clone_with_new_inputs( {data, folded_const1 ? folded_const1 : new_mul_add1, folded_const2 ? folded_const2 : new_mul_add2}); diff --git a/src/common/transformations/src/transformations/common_optimizations/reduce_merge.cpp b/src/common/transformations/src/transformations/common_optimizations/reduce_merge.cpp index 2d7f189e9b458f..55951b75503671 100644 --- a/src/common/transformations/src/transformations/common_optimizations/reduce_merge.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/reduce_merge.cpp @@ -58,7 +58,9 @@ bool fuse_reduce_operations(const std::shared_ptr& node) { std::shared_ptr axes = std::make_shared(OutputVector{top_reduce->input_value(1), bottom_reduce->input_value(1)}, int64_t(0)); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = ov::get_constant_from_source(axes)) { + OPENVINO_SUPPRESS_DEPRECATED_END axes = constant; } axes->set_friendly_name(bottom_reduce->get_friendly_name() + "/Axes"); diff --git a/src/common/transformations/src/transformations/common_optimizations/reduce_reshape_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/reduce_reshape_fusion.cpp index 726789c34f0c2a..daaa22f37739d3 100644 --- a/src/common/transformations/src/transformations/common_optimizations/reduce_reshape_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/reduce_reshape_fusion.cpp @@ -60,6 +60,7 @@ ov::pass::ReduceReshapeFusion::ReduceReshapeFusion() { std::dynamic_pointer_cast(reduce_node)) { logical_reduce_node->set_keep_dims(true); } + reduce_node->validate_and_infer_types(); reduce_node->set_friendly_name(reshape_node->get_friendly_name()); copy_runtime_info(reshape_node, reduce_node); replace_node(m.get_match_root(), reduce_node); diff --git a/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp b/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp index 91e24ab2dacc0f..5ef4ebfeea5f4c 100644 --- a/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp @@ -251,7 +251,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptr(op)) { + OPENVINO_SUPPRESS_DEPRECATED_START auto transpose_order = get_constant_from_source(op->input_value(1)); + OPENVINO_SUPPRESS_DEPRECATED_END if (output_shape.rank().is_static()) { if (transpose_order) { // set more precise dimensions during reverse infer diff --git a/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp index ecfd67225788c0..261d714016ed85 100644 --- a/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/ric_fusion.cpp @@ -279,7 +279,9 @@ class Gather : public ov::pass::MatcherPass { const auto& pattern_map = m.get_pattern_value_map(); const auto& output = pattern_map.at(pattern_root); + OPENVINO_SUPPRESS_DEPRECATED_START auto axis = ov::get_constant_from_source(pattern_map.at(axis_p)); + OPENVINO_SUPPRESS_DEPRECATED_END if (!axis) return false; @@ -291,7 +293,9 @@ class Gather : public ov::pass::MatcherPass { return true; } + OPENVINO_SUPPRESS_DEPRECATED_START auto order = ov::get_constant_from_source(pattern_map.at(indices_p)); + OPENVINO_SUPPRESS_DEPRECATED_END if (!order) return false; diff --git a/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp b/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp index 7fee95fddff78d..20ee9cf4072ced 100644 --- a/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp @@ -147,7 +147,9 @@ ov::pass::GatherNopElimination::GatherNopElimination() { return false; std::vector expected_vector(number_of_indices); std::iota(expected_vector.begin(), expected_vector.end(), 0); + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& indices = get_constant_from_source(gather->input_value(1))) { + OPENVINO_SUPPRESS_DEPRECATED_END const auto& indices_values = indices->cast_vector(); if (indices_values != expected_vector) return false; diff --git a/src/common/transformations/src/transformations/common_optimizations/softmax_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/softmax_fusion.cpp index 41fed3d350420d..accbec8b42d87e 100644 --- a/src/common/transformations/src/transformations/common_optimizations/softmax_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/softmax_fusion.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "itt.hpp" @@ -15,56 +16,66 @@ ov::pass::SoftmaxFusion::SoftmaxFusion() { MATCHER_SCOPE(SoftmaxFusion); - auto data_pattern = pass::pattern::any_input(pattern::has_static_rank()); + + auto data_pattern = pass::pattern::any_input(pass::pattern::has_static_rank()); auto reduce_max_axes_pattern = ngraph::pattern::wrap_type(); auto reduce_max_pattern = ngraph::pattern::wrap_type({data_pattern, reduce_max_axes_pattern}); auto sub_pattern = ngraph::pattern::wrap_type({data_pattern, reduce_max_pattern}); - auto exp_pattern = ngraph::pattern::wrap_type({sub_pattern}); + + auto exp_input = std::make_shared(OutputVector{sub_pattern, data_pattern}); + auto exp_pattern = ngraph::pattern::wrap_type({exp_input}); + auto reduce_sum_axes_pattern = ngraph::pattern::wrap_type(); auto reduce_sum_pattern = ngraph::pattern::wrap_type({exp_pattern, reduce_sum_axes_pattern}); auto div_pattern = ngraph::pattern::wrap_type({exp_pattern, reduce_sum_pattern}); - ov::matcher_pass_callback callback = [=](pattern::Matcher& m) { + ov::matcher_pass_callback callback = [=](pass::pattern::Matcher& m) { if (transformation_callback(m.get_match_root())) return false; const auto& pattern_map = m.get_pattern_value_map(); - auto reduce_max_axes = - std::dynamic_pointer_cast(pattern_map.at(reduce_max_axes_pattern).get_node_shared_ptr()); - if (!reduce_max_axes || shape_size(reduce_max_axes->get_shape()) != 1) - return false; auto reduce_sum_axes = std::dynamic_pointer_cast(pattern_map.at(reduce_sum_axes_pattern).get_node_shared_ptr()); if (!reduce_sum_axes || shape_size(reduce_sum_axes->get_shape()) != 1) return false; - - int64_t reduce_max_axis = reduce_max_axes->cast_vector()[0]; int64_t reduce_sum_axis = reduce_sum_axes->cast_vector()[0]; - if (reduce_max_axis < 0 || reduce_sum_axis < 0) { - const auto& pshape = pattern_map.at(data_pattern).get_partial_shape(); - auto rank = pshape.rank().get_length(); - if (reduce_max_axis < 0) + if (reduce_sum_axis < 0) { + const auto& rank = pattern_map.at(data_pattern).get_partial_shape().rank().get_length(); + reduce_sum_axis += rank; + } + + auto exp_input_is_subtract = pattern_map.count(sub_pattern) != 0; + if (exp_input_is_subtract) { + auto reduce_max_axes = std::dynamic_pointer_cast( + pattern_map.at(reduce_max_axes_pattern).get_node_shared_ptr()); + if (!reduce_max_axes || shape_size(reduce_max_axes->get_shape()) != 1) + return false; + int64_t reduce_max_axis = reduce_max_axes->cast_vector()[0]; + + if (reduce_max_axis < 0) { + const auto& rank = pattern_map.at(data_pattern).get_partial_shape().rank().get_length(); reduce_max_axis += rank; - if (reduce_sum_axis < 0) - reduce_sum_axis += rank; + } + + if (reduce_max_axis != reduce_sum_axis) + return false; } - if (reduce_max_axis != reduce_sum_axis) - return false; auto softmax = register_new_node(pattern_map.at(data_pattern), reduce_sum_axis); auto div = pattern_map.at(div_pattern).get_node_shared_ptr(); softmax->set_friendly_name(div->get_friendly_name()); - copy_runtime_info( - { - pattern_map.at(reduce_max_pattern).get_node_shared_ptr(), - pattern_map.at(sub_pattern).get_node_shared_ptr(), - pattern_map.at(exp_pattern).get_node_shared_ptr(), - pattern_map.at(reduce_sum_pattern).get_node_shared_ptr(), - div, - }, - softmax); + NodeVector fused_nodes; + fused_nodes.push_back(pattern_map.at(exp_pattern).get_node_shared_ptr()); + fused_nodes.push_back(pattern_map.at(reduce_sum_pattern).get_node_shared_ptr()); + fused_nodes.push_back(div); + if (exp_input_is_subtract) { + fused_nodes.push_back(pattern_map.at(reduce_max_pattern).get_node_shared_ptr()); + fused_nodes.push_back(pattern_map.at(sub_pattern).get_node_shared_ptr()); + } + copy_runtime_info(fused_nodes, softmax); + replace_node(div, softmax); return true; diff --git a/src/common/transformations/src/transformations/common_optimizations/split_concat_pair_to_interpolate_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/split_concat_pair_to_interpolate_fusion.cpp index af17b6eafa26db..69494e13013bcd 100644 --- a/src/common/transformations/src/transformations/common_optimizations/split_concat_pair_to_interpolate_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/split_concat_pair_to_interpolate_fusion.cpp @@ -206,7 +206,9 @@ ov::pass::SplitConcatPairToInterpolateFusion::SplitConcatPairToInterpolateFusion std::shared_ptr sizes_node; if (use_shape_for_elimination) { + OPENVINO_SUPPRESS_DEPRECATED_START sizes_node = get_constant_from_source(cast_mul_result_to_int); + OPENVINO_SUPPRESS_DEPRECATED_END } else { disable_constant_folding(shape_node); } diff --git a/src/common/transformations/src/transformations/common_optimizations/strides_optimization.cpp b/src/common/transformations/src/transformations/common_optimizations/strides_optimization.cpp index 25dc8300f674e9..9add3b590b2830 100644 --- a/src/common/transformations/src/transformations/common_optimizations/strides_optimization.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/strides_optimization.cpp @@ -51,7 +51,9 @@ static void insert_pooling(const Output& first, Input& second, const const auto ones = rg.make(element::i64, Shape{diff}, vector(diff, 1)); const auto current_shape = rg.make(first); shared_ptr new_shape = rg.make(OutputVector{ones, current_shape}, 0); + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto constant_new_shape = get_constant_from_source(new_shape)) { + OPENVINO_SUPPRESS_DEPRECATED_END rg.add(constant_new_shape); new_shape = constant_new_shape; } @@ -65,7 +67,9 @@ static void insert_pooling(const Output& first, Input& second, const iota(axes.begin(), axes.end(), 0); new_node = rg.make(new_node, rg.make(element::u64, Shape{diff}, axes)); } + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto constant_new_node = get_constant_from_source(new_node)) { + OPENVINO_SUPPRESS_DEPRECATED_END rg.add(constant_new_node); new_node = constant_new_node; } diff --git a/src/common/transformations/src/transformations/common_optimizations/transpose_sinking.cpp b/src/common/transformations/src/transformations/common_optimizations/transpose_sinking.cpp index 6fb9a12cc6e095..62f794270e1fba 100644 --- a/src/common/transformations/src/transformations/common_optimizations/transpose_sinking.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/transpose_sinking.cpp @@ -89,7 +89,9 @@ ov::pass::TransposeEltwise::TransposeEltwise() { if (ov::shape_size(shape) != 1) { eltwise_const_input = std::make_shared(eltwise_const_input, transpose->input_value(1)); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto const_node = ov::get_constant_from_source(eltwise_const_input)) { + OPENVINO_SUPPRESS_DEPRECATED_END eltwise_const_input = const_node; } } @@ -167,9 +169,11 @@ ov::pass::TransposeReduction::TransposeReduction() { if (!transpose_order || !reduction_axes) return false; + OPENVINO_SUPPRESS_DEPRECATED_START const auto& non_negative_axes = normalize_axes(reduction->get_friendly_name(), reduction_axes->cast_vector(), reduction->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END reduction_axes = opset6::Constant::create(ngraph::element::i64, {non_negative_axes.size()}, non_negative_axes); ngraph::NodeVector new_ops; diff --git a/src/common/transformations/src/transformations/control_flow/unroll_if.cpp b/src/common/transformations/src/transformations/control_flow/unroll_if.cpp index 05c97fb719a32f..b8507c7842b2b1 100644 --- a/src/common/transformations/src/transformations/control_flow/unroll_if.cpp +++ b/src/common/transformations/src/transformations/control_flow/unroll_if.cpp @@ -30,7 +30,9 @@ bool ov::pass::UnrollIf::run_on_model(const std::shared_ptr& f continue; } Output cond = if_node->input_value(0); + OPENVINO_SUPPRESS_DEPRECATED_START const auto cond_is_const = ngraph::get_constant_from_source(cond); + OPENVINO_SUPPRESS_DEPRECATED_END if (!cond_is_const) { continue; } diff --git a/src/common/transformations/src/transformations/convert_precision.cpp b/src/common/transformations/src/transformations/convert_precision.cpp index e7369c4a23066a..67132082857f3d 100644 --- a/src/common/transformations/src/transformations/convert_precision.cpp +++ b/src/common/transformations/src/transformations/convert_precision.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -334,7 +335,9 @@ bool ov::pass::ConvertPrecision::run_on_model(const std::shared_ptr& node, } bool fuse_type_to_topk(const std::shared_ptr& node, const precisions_map& precisions) { - if (auto topk = ov::as_type_ptr(node)) { + if (auto topk = ov::as_type_ptr(node)) { return update_type(1, node, precisions, [&](const element::Type& to) { topk->set_index_element_type(to); }); @@ -823,6 +826,26 @@ std::shared_ptr change_constant_precision +std::shared_ptr change_constant_precision( + std::shared_ptr& constant) { + using src_type = typename element_type_traits::value_type; + using dst_type = typename element_type_traits::value_type; + + const auto* src_data = constant->get_data_ptr(); + const auto size = shape_size(constant->get_shape()); + + auto new_constant = std::make_shared(ov::element::Type_t::f16, constant->get_shape()); + new_constant->output(0).set_names(constant->output(0).get_names()); + auto* dst_data = const_cast(reinterpret_cast(new_constant->get_data_ptr())); + if (dst_data == nullptr) + throw Exception("Can't get destination data pointer"); + + ngraph::runtime::reference::convert(src_data, dst_data, size); + + return new_constant; +} + /** * @brief Method converts low precision integer types * The method uses the next logic for conversion: diff --git a/src/common/transformations/src/transformations/flush_fp32_subnormals_to_zero.cpp b/src/common/transformations/src/transformations/flush_fp32_subnormals_to_zero.cpp index f01c60b1c0389b..ca03c288092260 100644 --- a/src/common/transformations/src/transformations/flush_fp32_subnormals_to_zero.cpp +++ b/src/common/transformations/src/transformations/flush_fp32_subnormals_to_zero.cpp @@ -36,14 +36,28 @@ ov::pass::FlushFP32SubnormalsToZero::FlushFP32SubnormalsToZero() { bool has_subnormals = false; for (size_t i = 0; i < size; ++i) { if (fpclassify(std::abs(data[i])) == FP_SUBNORMAL) { - data[i] = 0.0f; has_subnormals = true; + break; } } - if (has_subnormals) - return true; + if (!has_subnormals) + return false; + + auto new_constant = std::make_shared(ov::element::f32, node->get_shape()); + auto* dst_data = const_cast(new_constant->get_data_ptr()); + + for (size_t i = 0; i < size; ++i) { + if (fpclassify(std::abs(data[i])) != FP_SUBNORMAL) + dst_data[i] = data[i]; + else + dst_data[i] = 0.0f; + } + + new_constant->set_friendly_name(node->get_friendly_name()); + ov::copy_runtime_info(node, new_constant); + ov::replace_node(node, new_constant); - return false; + return true; }; auto m = make_shared(node_pattern, matcher_name); diff --git a/src/common/transformations/src/transformations/op_conversions/batch_norm_decomposition.cpp b/src/common/transformations/src/transformations/op_conversions/batch_norm_decomposition.cpp index 39bc2786ae7077..52b936f42e54ff 100644 --- a/src/common/transformations/src/transformations/op_conversions/batch_norm_decomposition.cpp +++ b/src/common/transformations/src/transformations/op_conversions/batch_norm_decomposition.cpp @@ -78,12 +78,14 @@ ov::pass::BatchNormDecomposition::BatchNormDecomposition() { mean_aligned, opset5::Constant::create(mean_aligned->get_output_element_type(0), Shape{}, {-1})); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = ov::get_constant_from_source(beta_aligned)) beta_aligned = constant; if (auto constant = ov::get_constant_from_source(mean_negative)) mean_negative = constant; if (auto constant = ov::get_constant_from_source(gamma_div_scale_aligned)) gamma_div_scale_aligned = constant; + OPENVINO_SUPPRESS_DEPRECATED_END // input_sub_mean = input + mean * -1 auto input_sub_mean = register_new_node(m_input, mean_negative); diff --git a/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp b/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp index 66bf50691b6172..f6702755edeb63 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp @@ -29,7 +29,9 @@ bool convert_divide(std::shared_ptr node) { ngraph::op::Constant::create(div->get_input_element_type(1), ngraph::Shape{}, {-1})); if (std::dynamic_pointer_cast(div->get_input_node_shared_ptr(1))) { + OPENVINO_SUPPRESS_DEPRECATED_START if (auto const_pow = ngraph::get_constant_from_source(pow)) { + OPENVINO_SUPPRESS_DEPRECATED_END pow = const_pow; } else { NGRAPH_DEBUG << "ConvertDivide has failed due to unsupported evaluate type in " << pow.get(); diff --git a/src/common/transformations/src/transformations/op_conversions/convert_gather_0d.cpp b/src/common/transformations/src/transformations/op_conversions/convert_gather_0d.cpp index eca3823bf9b867..86f10208c0524b 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_gather_0d.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_gather_0d.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include "itt.hpp" diff --git a/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp new file mode 100644 index 00000000000000..c9b2e15dd4cfaf --- /dev/null +++ b/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/op_conversions/convert_interpolate11_downgrade.hpp" + +#include +#include +#include +#include +#include + +#include "itt.hpp" + +ov::pass::ConvertInterpolate11ToInterpolate4::ConvertInterpolate11ToInterpolate4() { + MATCHER_SCOPE(ConvertInterpolate11ToInterpolate4); + + const auto interpolate_v11_pattern = pattern::wrap_type(); + + const matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto v4_compatible_interpolation_mode = [](const op::util::InterpolateBase::InterpolateMode mode) { + constexpr std::array allowed_modes = { + op::util::InterpolateBase::InterpolateMode::NEAREST, + op::util::InterpolateBase::InterpolateMode::LINEAR, + op::util::InterpolateBase::InterpolateMode::LINEAR_ONNX, + op::util::InterpolateBase::InterpolateMode::CUBIC}; + + return std::find(std::begin(allowed_modes), std::end(allowed_modes), mode) != std::end(allowed_modes); + }; + + const auto interpolate_v11 = std::dynamic_pointer_cast(m.get_match_root()); + if (!interpolate_v11 || !v4_compatible_interpolation_mode(interpolate_v11->get_attrs().mode) || + transformation_callback(interpolate_v11)) { + return false; + } + + // downgrade only if the interpolation mode used to create v11 is supported by v4 + std::shared_ptr interpolate_v4; + ov::Output v4_input_output_shape; + ov::Output v4_input_scales; + + if (interpolate_v11->get_attrs().shape_calculation_mode == + ov::op::util::InterpolateBase::ShapeCalcMode::SCALES) { + v4_input_scales = interpolate_v11->input_value(1); + v4_input_output_shape = opset4::Constant::create(element::i32, Shape{1}, {1}); + copy_runtime_info(interpolate_v11, v4_input_output_shape.get_node_shared_ptr()); + } else { + v4_input_output_shape = interpolate_v11->input_value(1); + v4_input_scales = opset4::Constant::create(element::f32, Shape{1}, {1.0f}); + copy_runtime_info(interpolate_v11, v4_input_scales.get_node_shared_ptr()); + } + + if (interpolate_v11->get_input_size() == 3) { // with axes input + interpolate_v4 = std::make_shared(interpolate_v11->input_value(0), + v4_input_output_shape, + v4_input_scales, + interpolate_v11->input_value(2), + interpolate_v11->get_attrs()); + } else { + interpolate_v4 = std::make_shared(interpolate_v11->input_value(0), + v4_input_output_shape, + v4_input_scales, + interpolate_v11->get_attrs()); + } + + interpolate_v4->set_friendly_name(interpolate_v11->get_friendly_name()); + copy_runtime_info(interpolate_v11, interpolate_v4); + replace_node(interpolate_v11, interpolate_v4); + + return true; + }; + + auto m = std::make_shared(interpolate_v11_pattern, matcher_name); + register_matcher(m, callback); +} diff --git a/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp b/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp index f7fe1905b102cb..48c506f0586975 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp @@ -34,8 +34,11 @@ ov::pass::ConvertInterpolate1ToInterpolate4::ConvertInterpolate1ToInterpolate4() element::f32); std::shared_ptr scales = std::make_shared(out_dims, in_dims); - if (const auto& constant = ov::get_constant_from_source(scales)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (const auto& constant = ov::get_constant_from_source(scales)) { + OPENVINO_SUPPRESS_DEPRECATED_END scales = constant; + } auto axisConstant = opset1::Constant::create(ngraph::element::i64, {axes.size()}, axes); ov::opset4::Interpolate::InterpolateAttrs attrsV4; diff --git a/src/common/transformations/src/transformations/op_conversions/convert_scatter_elements_to_scatter.cpp b/src/common/transformations/src/transformations/op_conversions/convert_scatter_elements_to_scatter.cpp index b17b761ab690ae..9def4bde880ba1 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_scatter_elements_to_scatter.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_scatter_elements_to_scatter.cpp @@ -56,9 +56,11 @@ ov::pass::ConvertScatterElementsToScatter::ConvertScatterElementsToScatter() { return false; } + OPENVINO_SUPPRESS_DEPRECATED_START const size_t axis = ngraph::normalize_axes(scatter->get_friendly_name(), axis_const->cast_vector(), data_pshape.rank())[0]; + OPENVINO_SUPPRESS_DEPRECATED_END struct Range { uint64_t l, r; diff --git a/src/common/transformations/src/transformations/op_conversions/convert_slice_to_strided_slice.cpp b/src/common/transformations/src/transformations/op_conversions/convert_slice_to_strided_slice.cpp index bc6ee2c6022b66..f85781f97b2b67 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_slice_to_strided_slice.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_slice_to_strided_slice.cpp @@ -78,9 +78,11 @@ ov::pass::SliceToStridedSlice::SliceToStridedSlice(bool use_shapes) { std::shared_ptr step_const; if (use_shapes) { + OPENVINO_SUPPRESS_DEPRECATED_START start_const = get_constant_from_source(slice_node->input_value(1)); stop_const = get_constant_from_source(slice_node->input_value(2)); step_const = get_constant_from_source(slice_node->input_value(3)); + OPENVINO_SUPPRESS_DEPRECATED_END } else { start_const = std::dynamic_pointer_cast(slice_node->input_value(1).get_node_shared_ptr()); stop_const = std::dynamic_pointer_cast(slice_node->input_value(2).get_node_shared_ptr()); @@ -93,10 +95,12 @@ ov::pass::SliceToStridedSlice::SliceToStridedSlice(bool use_shapes) { std::shared_ptr axes_const; if (slice_node->get_input_size() > 4) { + OPENVINO_SUPPRESS_DEPRECATED_START axes_const = use_shapes ? get_constant_from_source(slice_node->input_value(4)) : std::dynamic_pointer_cast(slice_node->input_value(4).get_node_shared_ptr()); + OPENVINO_SUPPRESS_DEPRECATED_END } else { axes_const = slice_node->get_default_const_axes(start_input); } @@ -106,7 +110,9 @@ ov::pass::SliceToStridedSlice::SliceToStridedSlice(bool use_shapes) { const auto& data_shape = slice_node->get_input_partial_shape(0); auto axes_vec = axes_const->cast_vector(); if (data_shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START auto norm_axes_vec = normalize_axes(slice_node->get_friendly_name(), axes_vec, data_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END axes_vec = std::vector(norm_axes_vec.begin(), norm_axes_vec.end()); } else { const bool need_normalization = std::any_of(axes_vec.begin(), axes_vec.end(), [](int64_t axis) { diff --git a/src/common/transformations/src/transformations/op_conversions/convert_softmax_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_softmax_downgrade.cpp index 493b098b0d43d0..ecf49f70a0d813 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_softmax_downgrade.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_softmax_downgrade.cpp @@ -25,7 +25,9 @@ ov::pass::ConvertSoftMax8ToSoftMax1::ConvertSoftMax8ToSoftMax1() { auto v8_axis = softmax_v8_node->get_axis(); auto rank = softmax_v8_node->get_input_partial_shape(0).rank().get_length(); + OPENVINO_SUPPRESS_DEPRECATED_START auto v1_axis = static_cast(ov::normalize_axis(softmax_v8_node->description(), v8_axis, rank)); + OPENVINO_SUPPRESS_DEPRECATED_END auto softmax_v1_node = std::make_shared(softmax_v8_node->input_value(0), v1_axis); softmax_v1_node->set_friendly_name(softmax_v8_node->get_friendly_name()); diff --git a/src/common/transformations/src/transformations/op_conversions/convert_topk11_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_topk11_downgrade.cpp index e54b9dacd785d4..5eff705bbc4b0c 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_topk11_downgrade.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_topk11_downgrade.cpp @@ -18,11 +18,13 @@ ov::pass::ConvertTopK11ToTopK3::ConvertTopK11ToTopK3() { const matcher_pass_callback callback = [=](pattern::Matcher& m) { const auto topk_v11 = std::dynamic_pointer_cast(m.get_match_root()); - if (!topk_v11 || topk_v11->get_stable() || transformation_callback(topk_v11)) { + if (!topk_v11 || transformation_callback(topk_v11)) { return false; } - // downgrade only if the stable sort is NOT required + // downgrade even if stable attribute is True + // this is needed to provide backward-compatibility + // and operation working in the plugins that have not yet added stable mode const auto topk_v3 = std::make_shared(topk_v11->input_value(0), topk_v11->input_value(1), diff --git a/src/common/transformations/src/transformations/op_conversions/einsum_decomposition.cpp b/src/common/transformations/src/transformations/op_conversions/einsum_decomposition.cpp index 06c5699b94ab4e..5186ac7bde8804 100644 --- a/src/common/transformations/src/transformations/op_conversions/einsum_decomposition.cpp +++ b/src/common/transformations/src/transformations/op_conversions/einsum_decomposition.cpp @@ -295,7 +295,9 @@ ngraph::Output reshape_input_for_matmul(const ngraph::Output reshaped_input_op; + OPENVINO_SUPPRESS_DEPRECATED_START if (auto new_shape_const = ngraph::get_constant_from_source(new_shape_op)) { + OPENVINO_SUPPRESS_DEPRECATED_END reshaped_input_op = std::make_shared(input_node, new_shape_const, false); subgraph_nodes.insert(subgraph_nodes.end(), {new_shape_const}); } else { @@ -660,7 +662,9 @@ void contract_two_inputs(ov::pass::EinsumDecomposition* einsum_decompose_ptr, // if new shape is possible to compute on the shape infer stage, insert Constant node immediatelly // in order to prevent repeated computing during constant-folding pass std::shared_ptr result_op; + OPENVINO_SUPPRESS_DEPRECATED_START if (auto new_shape_const = ngraph::get_constant_from_source(result_shape_op)) { + OPENVINO_SUPPRESS_DEPRECATED_END result_op = std::make_shared(matmul->output(0), new_shape_const, false); subgraph_nodes.insert(subgraph_nodes.end(), {new_shape_const}); } else { diff --git a/src/common/transformations/src/transformations/op_conversions/gather_normalize_negative_indices.cpp b/src/common/transformations/src/transformations/op_conversions/gather_normalize_negative_indices.cpp index 2e41b627b87ad6..bef1d5f14d8be3 100644 --- a/src/common/transformations/src/transformations/op_conversions/gather_normalize_negative_indices.cpp +++ b/src/common/transformations/src/transformations/op_conversions/gather_normalize_negative_indices.cpp @@ -66,8 +66,11 @@ ov::pass::GatherNegativeConstIndicesNormalize::GatherNegativeConstIndicesNormali ov::opset7::Constant::create(input_type, Shape{}, {0})); std::shared_ptr add = std::make_shared(input_gather, indices_constant); - if (auto folded_const = ngraph::get_constant_from_source(add)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto folded_const = ngraph::get_constant_from_source(add)) { + OPENVINO_SUPPRESS_DEPRECATED_END add = folded_const; + } gather->input(1).replace_source_output(add); ngraph::copy_runtime_info(gather, {shape_of, input_gather, add}); diff --git a/src/common/transformations/src/transformations/smart_reshape/matmul_sr.cpp b/src/common/transformations/src/transformations/smart_reshape/matmul_sr.cpp index 405f825a4fa6ca..aa8c4fa0d3f3b5 100644 --- a/src/common/transformations/src/transformations/smart_reshape/matmul_sr.cpp +++ b/src/common/transformations/src/transformations/smart_reshape/matmul_sr.cpp @@ -35,7 +35,9 @@ bool relax_hc_reshape_followed_by_matmul(const ngraph::pattern::PatternValueMap& const auto& raw_idx = reshape_is_A_input ? (matmul->get_transpose_b() ? -1 : -2) : (matmul->get_transpose_a() ? -2 : -1); + OPENVINO_SUPPRESS_DEPRECATED_START const auto& idx = ngraph::normalize_axes(matmul->description(), {raw_idx}, reshape_rank); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& C = std::make_shared(std::make_shared(shape_source), ov::opset4::Constant::create(ngraph::element::i64, {idx.size()}, idx), diff --git a/src/common/transformations/src/transformations/smart_reshape/shape_of_const_folding.cpp b/src/common/transformations/src/transformations/smart_reshape/shape_of_const_folding.cpp index fa343c42fc4356..13166d94ae3ae2 100644 --- a/src/common/transformations/src/transformations/smart_reshape/shape_of_const_folding.cpp +++ b/src/common/transformations/src/transformations/smart_reshape/shape_of_const_folding.cpp @@ -19,7 +19,9 @@ ov::pass::ShapeOfConstFolding::ShapeOfConstFolding() { matcher_pass_callback callback = [=](pattern::Matcher& m) -> bool { auto node = m.get_match_root(); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = get_constant_from_source(node)) { + OPENVINO_SUPPRESS_DEPRECATED_END constant->set_friendly_name(node->get_friendly_name()); copy_runtime_info(node, constant); replace_node(node, constant); diff --git a/src/common/transformations/src/transformations/smart_reshape/strided_slice_squeeze.cpp b/src/common/transformations/src/transformations/smart_reshape/strided_slice_squeeze.cpp index a0e142b7ceed55..0a26eb7cd6d96c 100644 --- a/src/common/transformations/src/transformations/smart_reshape/strided_slice_squeeze.cpp +++ b/src/common/transformations/src/transformations/smart_reshape/strided_slice_squeeze.cpp @@ -55,9 +55,11 @@ ov::pass::StridedSliceSqueeze::StridedSliceSqueeze() { })) return false; + OPENVINO_SUPPRESS_DEPRECATED_START const auto& axes = normalize_axes(squeeze->description(), const_axes->cast_vector(), squeeze->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END // Here squeeze input shape is equal to stridedslice input shape, // since new_axis_mask, shrink_axis_mask and ellipsis_mask are all zeros. @@ -158,9 +160,11 @@ ov::pass::SqueezeStridedSlice::SqueezeStridedSlice() { })) return false; + OPENVINO_SUPPRESS_DEPRECATED_START auto axes = normalize_axes(squeeze->description(), const_axes->cast_vector(), squeeze->get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END std::sort(axes.begin(), axes.end()); for (const auto& axis : axes) { begin_vec.insert(begin_vec.begin() + axis, 0); @@ -206,9 +210,12 @@ bool squeezes_perform_the_same(std::shared_ptr lhs, std::sh return false; const auto l_axes = std::dynamic_pointer_cast(lhs->get_input_node_shared_ptr(1)); const auto r_axes = std::dynamic_pointer_cast(rhs->get_input_node_shared_ptr(1)); - if (l_axes && r_axes) + if (l_axes && r_axes) { + OPENVINO_SUPPRESS_DEPRECATED_START return ngraph::normalize_axes(lhs->description(), l_axes->cast_vector(), rank) == ngraph::normalize_axes(rhs->description(), r_axes->cast_vector(), rank); + } + OPENVINO_SUPPRESS_DEPRECATED_END return false; } diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_binary.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_binary.cpp index ef46b4143c2305..f4fbeaefb1f96a 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_binary.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_binary.cpp @@ -32,6 +32,10 @@ TSBinaryForward::TSBinaryForward() { const auto& pattern_to_output = m.get_pattern_value_map(); auto& main_node_output = pattern_to_output.at(main_node_label); auto main_node = main_node_output.get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } + TransposeInputsInfo transpose_input_info = GetFirstTransposeInput(main_node); // todo: support dynamic rank case @@ -73,6 +77,9 @@ TSBinaryBackward::TSBinaryBackward() { auto transpose_const = as_type_ptr(pattern_to_output.at(transpose_const_label).get_node_shared_ptr()); auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); auto main_node = pattern_to_output.at(main_node_label).get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, transpose_const)) { register_new_node(new_node); diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_concat.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_concat.cpp index 4694b94565009a..726f40f216c9a9 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_concat.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_concat.cpp @@ -27,6 +27,9 @@ TSConcatForward::TSConcatForward() { auto& main_node_output = pattern_to_output.at(main_node_label); auto main_node = main_node_output.get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } TransposeInputsInfo transpose_input_info = GetFirstTransposeInput(main_node); auto concat_node = as_type_ptr(main_node); @@ -77,6 +80,10 @@ TSConcatBackward::TSConcatBackward() { auto transpose_const = as_type_ptr(pattern_to_output.at(transpose_const_label).get_node_shared_ptr()); auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); auto main_node = pattern_to_output.at(main_node_label).get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } + auto concat_node = as_type_ptr(main_node); auto concat_axis = concat_node->get_concatenation_axis(); if (concat_axis < 0) { diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_data_movement.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_data_movement.cpp index 482841d2c6532f..c59fb887be5a0b 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_data_movement.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_data_movement.cpp @@ -29,6 +29,10 @@ TSDataMovementForward::TSDataMovementForward() { const auto& pattern_to_node = m.get_pattern_map(); auto& main_node = pattern_to_node.at(main_node_label); + if (transformation_callback(main_node)) { + return false; + } + auto transpose = std::dynamic_pointer_cast(pattern_to_node.at(transpose_label)); if (!transpose) { return false; @@ -92,6 +96,9 @@ TSDataMovementBackward::TSDataMovementBackward() { auto transpose_const = as_type_ptr(pattern_to_output.at(transpose_const_label).get_node_shared_ptr()); auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); auto main_node = pattern_to_output.at(main_node_label).get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, transpose_const, diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_general.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_general.cpp index 507b874d78b3a9..c3ba55647720f8 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_general.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_general.cpp @@ -9,14 +9,18 @@ #include "openvino/pass/graph_rewrite.hpp" #include "openvino/pass/manager.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/common_optimizations/disable_shapeof_constant_folding.hpp" #include "transformations/transpose_sinking/ts_binary.hpp" #include "transformations/transpose_sinking/ts_concat.hpp" #include "transformations/transpose_sinking/ts_data_movement.hpp" #include "transformations/transpose_sinking/ts_fuse.hpp" #include "transformations/transpose_sinking/ts_interpolate.hpp" #include "transformations/transpose_sinking/ts_reduction.hpp" +#include "transformations/transpose_sinking/ts_slice.hpp" #include "transformations/transpose_sinking/ts_split.hpp" +#include "transformations/transpose_sinking/ts_squeeze.hpp" #include "transformations/transpose_sinking/ts_unary.hpp" +#include "transformations/transpose_sinking/ts_unsqueeze.hpp" #include "transformations/utils/utils.hpp" using namespace ov::pass::transpose_sinking; @@ -29,7 +33,10 @@ TSGeneralForward::TSGeneralForward() { add_matcher(); add_matcher(); add_matcher(); + add_matcher(); + add_matcher(); add_matcher(); + add_matcher(); add_matcher(); } @@ -41,7 +48,10 @@ TSGeneralBackward::TSGeneralBackward() { add_matcher(); add_matcher(); add_matcher(); + add_matcher(); + add_matcher(); add_matcher(); + add_matcher(); add_matcher(); } @@ -49,6 +59,7 @@ bool TSGeneral::run_on_model(const std::shared_ptr& f) { RUN_ON_FUNCTION_SCOPE(TSGeneral); { Manager manager(get_pass_config()); + manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.run_passes(f); @@ -56,6 +67,7 @@ bool TSGeneral::run_on_model(const std::shared_ptr& f) { { Manager manager(get_pass_config()); + manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.run_passes(f); diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_interpolate.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_interpolate.cpp index 0a9c2b7458f019..88d603aba01d43 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_interpolate.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_interpolate.cpp @@ -29,6 +29,10 @@ TSInterpolateForward::TSInterpolateForward() { const auto& pattern_to_node = m.get_pattern_map(); auto& main_node = pattern_to_node.at(main_node_label); + if (transformation_callback(main_node)) { + return false; + } + auto transpose = std::dynamic_pointer_cast(pattern_to_node.at(transpose_label)); if (!transpose) { return false; @@ -102,6 +106,9 @@ TSInterpolateBackward::TSInterpolateBackward() { auto transpose_const = as_type_ptr(pattern_to_output.at(transpose_const_label).get_node_shared_ptr()); auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); auto main_node = pattern_to_output.at(main_node_label).get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, transpose_const, diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_reduction.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_reduction.cpp index ab4e76994ece7f..1b10c01e6967b0 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_reduction.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_reduction.cpp @@ -18,60 +18,15 @@ using namespace ov; using namespace opset10; +using namespace ov::pass::pattern; using namespace ov::pass::transpose_sinking; using namespace ov::pass::transpose_sinking::utils; namespace { -std::vector get_updated_order_forward(const std::vector& axes_values, - const std::vector& order_values) { - size_t buffer_size = order_values.size() - axes_values.size(); - std::vector aligned_order(buffer_size, 0); - std::vector values_to_reduce(axes_values); - for (size_t i = 0; i < values_to_reduce.size(); ++i) { - values_to_reduce[i] = order_values[axes_values[i]]; - } - std::sort(values_to_reduce.begin(), values_to_reduce.end()); - for (size_t i = 0, j = 0; i < order_values.size(); ++i) { - if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) { - continue; - } - - auto lb = std::lower_bound(values_to_reduce.begin(), values_to_reduce.end(), order_values[i]); - aligned_order[j] = order_values[i] - (lb - values_to_reduce.begin()); - ++j; - } - return aligned_order; -} - -std::vector get_updated_order_backward(const std::vector& axes_values, - const std::vector& order_values) { - size_t buffer_size = order_values.size() + axes_values.size(); - std::vector aligned_order(buffer_size); - - std::vector cnt_deleted(buffer_size); - int64_t cnt = 0; - for (int64_t i = 0; i < static_cast(cnt_deleted.size()); ++i) { - if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) { - cnt++; - } - cnt_deleted[i] = i - cnt; - } - - for (size_t i = 0, j = 0; i < aligned_order.size(); ++i) { - if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) { - aligned_order[i] = i; - continue; - } - - aligned_order[i] = std::find(cnt_deleted.begin(), cnt_deleted.end(), order_values[j]) - cnt_deleted.begin(); - ++j; - } - return aligned_order; -} bool get_keep_dims(const std::shared_ptr& reduction) { - auto arithmetic_reduce = std::dynamic_pointer_cast(reduction); - auto logical_reduce = std::dynamic_pointer_cast(reduction); + auto arithmetic_reduce = as_type_ptr(reduction); + auto logical_reduce = as_type_ptr(reduction); bool keep_dims = false; // squeeze/unsqueeze always reduces number of output dimensions if (logical_reduce) @@ -80,34 +35,37 @@ bool get_keep_dims(const std::shared_ptr& reduction) { keep_dims = arithmetic_reduce->get_keep_dims(); return keep_dims; } + } // namespace TSReductionForward::TSReductionForward() { MATCHER_SCOPE(TSReductionForward); - auto transpose_label = pattern::wrap_type({pattern::any_input(), pattern::wrap_type()}, - pattern::consumers_count(1)); - auto reduce_or_squeeze_label = pattern:: - wrap_type( - {transpose_label, pattern::wrap_type()}); + auto transpose_label = wrap_type({any_input(), wrap_type()}); + auto reduce_label = wrap_type( + {transpose_label, wrap_type()}); ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { - const auto& pattern_to_output = m.get_pattern_value_map(); + const auto& pattern_to_output = m.get_pattern_map(); + + auto transpose = pattern_to_output.at(transpose_label); + auto reduction = pattern_to_output.at(reduce_label); + if (transformation_callback(reduction)) { + return false; + } - auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); - auto reduction = pattern_to_output.at(reduce_or_squeeze_label).get_node_shared_ptr(); auto keep_dims = get_keep_dims(reduction); - auto transpose_order = std::dynamic_pointer_cast(transpose->get_input_node_shared_ptr(1)); - auto reduction_axes = std::dynamic_pointer_cast(reduction->get_input_node_shared_ptr(1)); + auto transpose_order = as_type_ptr(transpose->get_input_node_shared_ptr(1)); + auto reduction_axes = as_type_ptr(reduction->get_input_node_shared_ptr(1)); if (!transpose_order || !reduction_axes) return false; - auto unsqueeze = std::dynamic_pointer_cast(reduction); - auto rank = - unsqueeze ? reduction->get_output_partial_shape(0).rank() : reduction->get_input_partial_shape(0).rank(); + auto rank = reduction->get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START auto non_negative_axes = normalize_axes(reduction->get_friendly_name(), reduction_axes->cast_vector(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END auto transpose_order_values = transpose_order->cast_vector(); std::vector new_values; @@ -117,38 +75,17 @@ TSReductionForward::TSReductionForward() { } if (!keep_dims) { - if (non_negative_axes.empty()) { - auto input_pshape = transpose->input_value(0).get_partial_shape(); - - if (input_pshape.is_static()) { - for (size_t i = 0; i < input_pshape.size(); ++i) { - if (input_pshape[i] == 1) { - non_negative_axes.push_back(i); - } - } - } else { - return false; - } - } - if (unsqueeze) { - transpose_order_values = get_updated_order_backward(non_negative_axes, transpose_order_values); - } else { - transpose_order_values = get_updated_order_forward(non_negative_axes, transpose_order_values); - } + transpose_order_values = GetOrderAfterReduction(non_negative_axes, transpose_order_values); } - auto new_transpose_order = std::make_shared(transpose_order->get_element_type(), - Shape{transpose_order_values.size()}, - transpose_order_values); - std::shared_ptr new_reduction; - if (!unsqueeze) { - auto new_const = - std::make_shared(reduction_axes->get_element_type(), reduction_axes->get_shape(), new_values); - new_reduction = reduction->clone_with_new_inputs({transpose->input_value(0), new_const}); - } else { - new_reduction = reduction->clone_with_new_inputs({transpose->input_value(0), reduction->input_value(1)}); - } + auto new_transpose_order = Constant::create(transpose_order->get_element_type(), + {transpose_order_values.size()}, + transpose_order_values); + + auto new_const = Constant::create(reduction_axes->get_element_type(), reduction_axes->get_shape(), new_values); + auto new_reduction = reduction->clone_with_new_inputs({transpose->input_value(0), new_const}); auto new_transpose = transpose->clone_with_new_inputs({new_reduction, new_transpose_order}); + replace_node(reduction, new_transpose); new_reduction->set_friendly_name(transpose->get_friendly_name()); new_transpose->set_friendly_name(reduction->get_friendly_name()); @@ -158,98 +95,61 @@ TSReductionForward::TSReductionForward() { return true; }; - auto m = std::make_shared(reduce_or_squeeze_label, matcher_name); + auto m = std::make_shared(reduce_label, matcher_name); register_matcher(m, matcher_pass_callback); } TSReductionBackward::TSReductionBackward() { MATCHER_SCOPE(TSReductionBackward); - auto reduce_or_squeeze_label = pattern:: - wrap_type( - {pattern::any_input(), pattern::wrap_type()}, - HasSameOutputTransposeNodes); - auto transpose_label = pattern::wrap_type({reduce_or_squeeze_label, pattern::wrap_type()}); + auto reduce_label = wrap_type( + {any_input(), wrap_type()}, + HasSameOutputTransposeNodes); + auto transpose_label = wrap_type({reduce_label, wrap_type()}); + ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { - const auto& pattern_to_output = m.get_pattern_value_map(); + const auto& pattern_to_output = m.get_pattern_map(); + auto transpose = pattern_to_output.at(transpose_label); + auto reduction = pattern_to_output.at(reduce_label); + if (transformation_callback(reduction)) { + return false; + } - auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); - auto reduction = pattern_to_output.at(reduce_or_squeeze_label).get_node_shared_ptr(); auto keep_dims = get_keep_dims(reduction); - auto transpose_order = std::dynamic_pointer_cast(transpose->get_input_node_shared_ptr(1)); - auto reduction_axes = std::dynamic_pointer_cast(reduction->get_input_node_shared_ptr(1)); + + auto transpose_order = as_type_ptr(transpose->get_input_node_shared_ptr(1)); + auto reduction_axes = as_type_ptr(reduction->get_input_node_shared_ptr(1)); if (!transpose_order || !reduction_axes) return false; - auto unsqueeze = std::dynamic_pointer_cast(reduction); - auto rank = - unsqueeze ? reduction->get_output_partial_shape(0).rank() : reduction->get_input_partial_shape(0).rank(); + auto rank = reduction->get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START auto non_negative_axes = normalize_axes(reduction->get_friendly_name(), reduction_axes->cast_vector(), rank); - + OPENVINO_SUPPRESS_DEPRECATED_END auto transpose_order_values = transpose_order->cast_vector(); - auto old_transpose_order_values = transpose_order_values; - std::vector new_values; - if (unsqueeze) { - if (non_negative_axes.size() == transpose_order_values.size()) { - // input is a scalar, we unsqueeze all dims - // it's enough to eliminate such Transpose - transpose->output(0).replace(reduction); - return true; - } - for (const auto& axis : non_negative_axes) { - auto it = std::find(old_transpose_order_values.begin(), old_transpose_order_values.end(), axis); - if (it != old_transpose_order_values.end()) { - new_values.push_back(it - old_transpose_order_values.begin()); - } - } - } - bool squeeze_all_dims = false; if (!keep_dims) { - if (non_negative_axes.empty()) { - auto input_pshape = reduction->input_value(0).get_partial_shape(); - if (input_pshape.is_static()) { - for (size_t i = 0; i < input_pshape.size(); ++i) { - if (input_pshape[i] == 1) { - non_negative_axes.push_back(i); - } - } - squeeze_all_dims = true; - } else { - return false; - } - } - if (unsqueeze) { - transpose_order_values = get_updated_order_forward(new_values, transpose_order_values); - } else { - transpose_order_values = get_updated_order_backward(non_negative_axes, transpose_order_values); - } + transpose_order_values = GetOrderBeforeReduction(non_negative_axes, transpose_order_values); } + auto reversed_order_values = ReverseTransposeOrder(transpose_order_values); + auto new_transpose_order = Constant::create(transpose_order->get_element_type(), + {transpose_order_values.size()}, + transpose_order_values); - if (!unsqueeze) { - auto reversed_order_values = ReverseTransposeOrder(transpose_order_values); - for (const auto& axis : non_negative_axes) { - new_values.push_back(reversed_order_values[axis]); - } + std::vector new_values; + for (const auto& axis : non_negative_axes) { + new_values.push_back(reversed_order_values[axis]); } - auto new_transpose_order = std::make_shared(transpose_order->get_element_type(), - Shape{transpose_order_values.size()}, - transpose_order_values); - std::shared_ptr new_transpose, new_reduction; - if (squeeze_all_dims) { - new_transpose = transpose->clone_with_new_inputs({reduction->input_value(0), new_transpose_order}); - new_reduction = reduction->clone_with_new_inputs({new_transpose, reduction->input_value(1)}); - } else { - auto new_const = - std::make_shared(reduction_axes->get_element_type(), reduction_axes->get_shape(), new_values); - new_transpose = transpose->clone_with_new_inputs({reduction->input_value(0), new_transpose_order}); - new_reduction = reduction->clone_with_new_inputs({new_transpose, new_const}); - } + auto new_const = Constant::create(reduction_axes->get_element_type(), reduction_axes->get_shape(), new_values); + auto new_transpose = transpose->clone_with_new_inputs({reduction->input_value(0), new_transpose_order}); + auto new_reduction = reduction->clone_with_new_inputs({new_transpose, new_const}); + replace_node(transpose, new_reduction); copy_runtime_info({transpose, reduction}, {new_transpose, new_reduction}); UpdateForwardSinkingAbility(new_transpose); new_reduction->set_friendly_name(transpose->get_friendly_name()); + new_transpose->set_friendly_name(reduction->get_friendly_name()); register_new_node(new_transpose); return true; }; diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_slice.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_slice.cpp new file mode 100644 index 00000000000000..e8b0f5c663d698 --- /dev/null +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_slice.cpp @@ -0,0 +1,123 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/transpose_sinking/ts_slice.hpp" + +#include "itt.hpp" +#include "openvino/op/util/op_types.hpp" +#include "openvino/opsets/opset10.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/util/common_util.hpp" +#include "transformations/rt_info/transpose_sinking_attr.hpp" +#include "transformations/transpose_sinking/ts_utils.hpp" + +using namespace ov; +using namespace ov::opset10; +using namespace ov::pass::pattern; +using namespace ov::pass::transpose_sinking; +using namespace ov::pass::transpose_sinking::utils; + +TSSliceForward::TSSliceForward() { + MATCHER_SCOPE(TSSliceForward); + auto const_label = wrap_type(); + auto transpose_label = wrap_type({any_input(), const_label}); + auto main_node_label = wrap_type({transpose_label, any_input(), any_input(), any_input(), any_input()}); + + matcher_pass_callback matcher_pass_callback = [=](Matcher& m) { + const auto& pattern_to_node = m.get_pattern_map(); + + auto& main_node = pattern_to_node.at(main_node_label); + if (transformation_callback(main_node)) { + return false; + } + + auto transpose = std::dynamic_pointer_cast(pattern_to_node.at(transpose_label)); + if (!transpose || main_node->get_input_size() < 5) { + return false; + } + + auto transpose_const = as_type_ptr(pattern_to_node.at(const_label)); + if (!transpose_const) { + return false; + } + + // remove Transpose on 1st input: + auto transpose_parent = transpose->input_value(0); + main_node->input(0).replace_source_output(transpose_parent); + + const auto transpose_axis_order = transpose_const->get_axis_vector_val(); + auto axis = std::make_shared(element::i32, Shape{}, std::vector{0}); + + auto data = std::make_shared(element::i32, Shape{transpose_axis_order.size()}, transpose_axis_order); + const auto& indices = main_node->input_value(4); + auto new_axis = std::make_shared(data, indices, axis); + + main_node->input(4).replace_source_output(new_axis); + + main_node->validate_and_infer_types(); + TransposeInputsInfo transpose_input_info = {transpose, transpose_const, 0}; + for (auto& new_node : sink_forward::InsertOutputTransposes(main_node, transpose_input_info)) { + register_new_node(new_node); + UpdateForwardSinkingAbility(new_node); + } + return true; + }; + + auto m = std::make_shared(main_node_label, matcher_name); + register_matcher(m, matcher_pass_callback); +} + +TSSliceBackward::TSSliceBackward() { + MATCHER_SCOPE(TSSliceBackward); + + auto main_node_label = wrap_type([](const Output& output) -> bool { + return has_static_rank()(output) && HasSameOutputTransposeNodes(output); + }); + + auto transpose_const_label = wrap_type(); + + auto transpose_label = + wrap_type({main_node_label, transpose_const_label}, [](const Output& output) -> bool { + return has_static_rank()(output) && is_sinking_node(output); + }); + + matcher_pass_callback matcher_pass_callback = [=](Matcher& m) { + const auto& pattern_to_output = m.get_pattern_value_map(); + auto transpose_const = as_type_ptr(pattern_to_output.at(transpose_const_label).get_node_shared_ptr()); + auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); + auto main_node = pattern_to_output.at(main_node_label).get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } + + if (main_node->get_input_size() < 5) { + return false; + } + + for (auto& new_node : sink_backward::InsertTransposeBeforeNode(main_node, + transpose_const, + /* input_indexes= */ {0})) { + register_new_node(new_node); + } + + // remove output transposes + RemoveSingleOutputConsumers(main_node); + SwapNames(main_node, transpose); + const auto transpose_axis_order = transpose_const->get_axis_vector_val(); + const auto reversed_transpose_order = ReverseTransposeOrder(transpose_axis_order); + auto axis = std::make_shared(element::i32, Shape{}, std::vector{0}); + auto data = + std::make_shared(element::i32, Shape{reversed_transpose_order.size()}, reversed_transpose_order); + const auto& indices = main_node->input_value(4); + auto new_axis = std::make_shared(data, indices, axis); + main_node->input(4).replace_source_output(new_axis); + + main_node->validate_and_infer_types(); + return true; + }; + + auto m = std::make_shared(transpose_label, matcher_name); + register_matcher(m, matcher_pass_callback); +} diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_split.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_split.cpp index 3aeb74436e7783..38c5e5c9c2806b 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_split.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_split.cpp @@ -134,7 +134,7 @@ TSSplitBackward::TSSplitBackward() { split = FindInputNode(transpose_label_node); } - if (!split) { + if (!split || transformation_callback(split)) { return false; } auto split_axis_constant = as_type_ptr(split->input_value(1).get_node_shared_ptr()); @@ -200,6 +200,10 @@ TSSplitForward::TSSplitForward() { auto& main_node_output = pattern_to_output.at(main_node_label); auto main_node = main_node_output.get_node_shared_ptr(); + if (transformation_callback(main_node)) { + return false; + } + auto split_axis_constant = as_type_ptr(main_node->input_value(1).get_node_shared_ptr()); if (!split_axis_constant) { return false; diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_squeeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_squeeze.cpp new file mode 100644 index 00000000000000..f2954701e347cf --- /dev/null +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_squeeze.cpp @@ -0,0 +1,281 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/transpose_sinking/ts_squeeze.hpp" + +#include +#include + +#include "itt.hpp" +#include "openvino/core/validation_util.hpp" +#include "openvino/opsets/opset10.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/rt_info/transpose_sinking_attr.hpp" +#include "transformations/transpose_sinking/ts_utils.hpp" +#include "transformations/utils/utils.hpp" + +using namespace ov; +using namespace opset10; +using namespace ov::pass::pattern; +using namespace ov::pass::transpose_sinking; +using namespace ov::pass::transpose_sinking::utils; + +namespace { + +/** + * @brief Checks that Reshape operation is equal to Squeeze: + * Only 1 dims are deleted, all other dims must be the same. + * Converts these 1 dims to axes format. + * @arg reshape Reshape operation. + * @arg reshape_to_shape 2nd input to Reshape op as a constant. + * @arg result_axes Contains axes which will be squeezed. + */ +bool shape_to_squeeze_axes(const std::shared_ptr& reshape, + const std::shared_ptr& reshape_to_shape, + std::vector& result_axes) { + result_axes.clear(); + auto reduction_axes_values = reshape_to_shape->cast_vector(); + // supported the case if Reshape is equal to Squeeze + const auto& new_shape = reduction_axes_values; + const auto& input_pshape = reshape->get_input_partial_shape(0); + // todo: support dynamic case + if (input_pshape.is_dynamic()) { + return false; + } + + const auto input_shape = input_pshape.to_shape(); + if (new_shape.size() < input_shape.size()) { + size_t j = 0; + for (size_t i = 0; i < input_shape.size(); i++) { + const auto input_dim = static_cast(input_shape[i]); + if (j < new_shape.size() && new_shape[j] == input_dim) { + j++; + } else if (input_dim != 1) { + return false; + } else { + result_axes.push_back(i); + } + } + if (j != new_shape.size()) { + // not all new_shape values are in input_shape + return false; + } + } else { + // another reshape type, not Squeeze + // todo: move this checks in the pattern + return false; + } + return true; +} + +/** + * @brief Converts squeezed_axes to actual shape (2nd input) for Reshape operation + * using the shape of the 1st input to Reshape. + * @arg input_node 1st input to Reshape op. + * @arg squeeze_axes In case of Reshape op is equal to squeeze, these axes indicate the places where 1 dims have + * to be deleted. + */ +bool squeeze_axes_to_shape(const Output& input_node, + std::vector squeeze_axes, + std::vector& to_shape) { + to_shape.clear(); + std::sort(squeeze_axes.begin(), squeeze_axes.end()); + const auto& input_pshape = input_node.get_partial_shape(); + if (input_pshape.is_dynamic()) { + return false; + } + const auto& input_shape = input_pshape.get_shape(); + for (size_t i = 0, j = 0; i < input_shape.size(); ++i) { + if (j < squeeze_axes.size() && i == squeeze_axes[j]) { + ++j; + continue; + } + to_shape.push_back(input_shape[i]); + } + return true; +} + +} // namespace + +TSSqueezeForward::TSSqueezeForward() { + MATCHER_SCOPE(TSSqueezeForward); + + auto transpose_label = wrap_type({any_input(), wrap_type()}); + auto squeeze_label = wrap_type({transpose_label, wrap_type()}); + + ov::matcher_pass_callback matcher_pass_callback = [=](Matcher& m) { + const auto& pattern_to_output = m.get_pattern_map(); + + auto transpose = pattern_to_output.at(transpose_label); + auto squeeze = pattern_to_output.at(squeeze_label); + if (transformation_callback(squeeze)) { + return false; + } + + auto transpose_order = as_type_ptr(transpose->get_input_node_shared_ptr(1)); + auto squeeze_axes = as_type_ptr(squeeze->get_input_node_shared_ptr(1)); + if (!transpose_order || !squeeze_axes) { + return false; + } + + std::vector non_negative_axes; + if (as_type_ptr(squeeze)) { + auto success = shape_to_squeeze_axes(squeeze, squeeze_axes, non_negative_axes); + if (!success) { + return false; + } + } else { + auto rank = squeeze->get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START + non_negative_axes = + normalize_axes(squeeze->get_friendly_name(), squeeze_axes->cast_vector(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END + } + + // if 2nd input to squeeze is empty then all '1' dims will be deleted. + if (non_negative_axes.empty()) { + auto input_pshape = transpose->output(0).get_partial_shape(); + if (input_pshape.is_dynamic()) { + return false; + } + for (size_t i = 0; i < input_pshape.size(); ++i) { + if (input_pshape[i].get_length() == 1) { + non_negative_axes.push_back(i); + } + } + } + + auto transpose_order_values = transpose_order->cast_vector(); + std::vector new_values; + new_values.reserve(non_negative_axes.size()); + for (const auto& axis : non_negative_axes) { + new_values.push_back(transpose_order_values[axis]); + } + + transpose_order_values = GetOrderAfterReduction(non_negative_axes, transpose_order_values); + auto new_transpose_order = Constant::create(transpose_order->get_element_type(), + {transpose_order_values.size()}, + transpose_order_values); + + if (as_type_ptr(squeeze)) { + std::vector to_shape; + auto success = squeeze_axes_to_shape(transpose->input_value(0), new_values, to_shape); + if (!success) { + return false; + } + new_values = to_shape; + } + + auto new_const = Constant::create(squeeze_axes->get_element_type(), {new_values.size()}, new_values); + auto new_squeeze = squeeze->clone_with_new_inputs({transpose->input_value(0), new_const}); + auto new_transpose = transpose->clone_with_new_inputs({new_squeeze, new_transpose_order}); + + replace_node(squeeze, new_transpose); + new_squeeze->set_friendly_name(transpose->get_friendly_name()); + new_transpose->set_friendly_name(squeeze->get_friendly_name()); + UpdateForwardSinkingAbility(new_transpose); + register_new_node(new_transpose); + copy_runtime_info({transpose, squeeze}, {new_transpose, new_squeeze}); + + return true; + }; + + auto m = std::make_shared(squeeze_label, matcher_name); + register_matcher(m, matcher_pass_callback); +} + +TSSqueezeBackward::TSSqueezeBackward() { + MATCHER_SCOPE(TSSqueezeBackward); + + auto squeeze_label = wrap_type({any_input(), wrap_type()}, HasSameOutputTransposeNodes); + auto transpose_label = + wrap_type({squeeze_label, wrap_type()}, [](const Output& output) -> bool { + return has_static_rank()(output) && is_sinking_node(output); + }); + + ov::matcher_pass_callback matcher_pass_callback = [=](Matcher& m) { + const auto& pattern_to_output = m.get_pattern_map(); + + auto transpose = pattern_to_output.at(transpose_label); + auto squeeze = pattern_to_output.at(squeeze_label); + if (transformation_callback(squeeze)) { + return false; + } + + auto transpose_order = as_type_ptr(transpose->get_input_node_shared_ptr(1)); + auto squeeze_axes = as_type_ptr(squeeze->get_input_node_shared_ptr(1)); + if (!transpose_order || !squeeze_axes) { + return false; + } + + std::vector non_negative_axes; + if (as_type_ptr(squeeze)) { + auto success = shape_to_squeeze_axes(squeeze, squeeze_axes, non_negative_axes); + if (!success) { + return false; + } + } else { + auto rank = squeeze->get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START + non_negative_axes = + normalize_axes(squeeze->get_friendly_name(), squeeze_axes->cast_vector(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END + } + + bool squeeze_all_dims = false; + if (non_negative_axes.empty()) { + auto input_pshape = squeeze->input_value(0).get_partial_shape(); + if (input_pshape.is_dynamic()) { + return false; + } + for (size_t i = 0; i < input_pshape.size(); ++i) { + if (input_pshape[i] == 1) { + non_negative_axes.push_back(i); + } + } + squeeze_all_dims = true; + } + + auto transpose_order_values = transpose_order->cast_vector(); + transpose_order_values = GetOrderBeforeReduction(non_negative_axes, transpose_order_values); + auto reversed_order_values = ReverseTransposeOrder(transpose_order_values); + + std::vector new_values; + for (const auto& axis : non_negative_axes) { + new_values.push_back(reversed_order_values[axis]); + } + + auto new_transpose_order = Constant::create(transpose_order->get_element_type(), + {transpose_order_values.size()}, + transpose_order_values); + auto new_transpose = transpose->clone_with_new_inputs({squeeze->input_value(0), new_transpose_order}); + if (as_type_ptr(squeeze)) { + std::vector to_shape; + auto success = squeeze_axes_to_shape(new_transpose->output(0), new_values, to_shape); + if (!success) { + return false; + } + new_values = to_shape; + } + + std::shared_ptr new_squeeze; + if (squeeze_all_dims) { + new_squeeze = squeeze->clone_with_new_inputs({new_transpose, squeeze->input_value(1)}); + } else { + auto new_const = + std::make_shared(squeeze_axes->get_element_type(), squeeze_axes->get_shape(), new_values); + new_squeeze = squeeze->clone_with_new_inputs({new_transpose, new_const}); + } + + replace_node(transpose, new_squeeze); + copy_runtime_info({transpose, squeeze}, {new_transpose, new_squeeze}); + new_squeeze->set_friendly_name(transpose->get_friendly_name()); + new_transpose->set_friendly_name(squeeze->get_friendly_name()); + register_new_node(new_transpose); + return true; + }; + + auto m = std::make_shared(transpose_label, matcher_name); + register_matcher(m, matcher_pass_callback); +} \ No newline at end of file diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unary.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unary.cpp index ed543ca9d92139..e3147d5cf7a190 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_unary.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unary.cpp @@ -64,6 +64,9 @@ TSUnaryForward::TSUnaryForward() { const auto& pattern_to_output = m.get_pattern_value_map(); auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); auto unary = pattern_to_output.at(unary_label).get_node_shared_ptr(); + if (transformation_callback(unary)) { + return false; + } const NodePair new_nodes = SwapNodes(transpose, unary); @@ -105,6 +108,9 @@ TSUnaryBackward::TSUnaryBackward() { auto transpose_const = as_type_ptr(pattern_to_output.at(transpose_const_label).get_node_shared_ptr()); auto transpose = pattern_to_output.at(transpose_label).get_node_shared_ptr(); auto unary = pattern_to_output.at(unary_label).get_node_shared_ptr(); + if (transformation_callback(unary)) { + return false; + } for (auto& new_node : sink_backward::InsertTransposeBeforeNode(unary, transpose_const)) { register_new_node(new_node); diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp new file mode 100644 index 00000000000000..3b92a2072a7099 --- /dev/null +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_unsqueeze.cpp @@ -0,0 +1,253 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/transpose_sinking/ts_unsqueeze.hpp" + +#include +#include + +#include "itt.hpp" +#include "openvino/core/validation_util.hpp" +#include "openvino/opsets/opset10.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/rt_info/transpose_sinking_attr.hpp" +#include "transformations/transpose_sinking/ts_utils.hpp" +#include "transformations/utils/utils.hpp" + +using namespace ov; +using namespace opset10; +using namespace ov::pass::pattern; +using namespace ov::pass::transpose_sinking; +using namespace ov::pass::transpose_sinking::utils; + +namespace { + +/** + * @brief Checks that Reshape operation is equal to Unsqueeze: + * Only 1 dims are inserted, all other dims must be the same. + * Converts these 1 dims to axes format. + * @arg reshape Reshape operation. + * @arg reshape_to_shape 2nd input to Reshape op as a constant. + * @arg result_axes contains axes which will be unsqueezed. + */ +bool shape_to_unsqueeze_axes(const std::shared_ptr& reshape, + const std::shared_ptr& reshape_to_shape, + std::vector& result_axes) { + result_axes.clear(); + auto reduction_axes_values = reshape_to_shape->cast_vector(); + // supported the case if Reshape is equal to Unsqueeze + const auto& new_shape = reduction_axes_values; + const auto& input_pshape = reshape->get_input_partial_shape(0); + // todo: support dynamic case + if (input_pshape.is_dynamic()) { + return false; + } + + const auto input_shape = input_pshape.to_shape(); + if (new_shape.size() > input_shape.size()) { + size_t j = 0; + for (size_t i = 0; i < new_shape.size(); ++i) { + if (j < input_shape.size() && static_cast(input_shape[j]) == new_shape[i]) { + j++; + } else if (new_shape[i] != 1) { + return false; + } else { + result_axes.push_back(i); + } + } + if (j != input_shape.size()) { + // not all input_shape values are in new_shape + return false; + } + } else { + // another reshape type, not Unsqueeze + // todo: move this checks in the pattern + return false; + } + return true; +} + +/** + * @brief Converts unsqueeze_axes to actual shape (2nd input) for Reshape operation + * using the shape of the 1st input to Reshape. + * @arg input_node 1st input to Reshape op. + * @arg unsqueeze_axes In case of Reshape op is equal to Unsqueeze, these axes indicate the places where 1 dims have + * to be inserted. + */ +bool unsqueeze_axes_to_shape(const Output& input_node, + std::vector unsqueeze_axes, + std::vector& to_shape) { + to_shape.clear(); + const auto& input_pshape = input_node.get_partial_shape(); + if (input_pshape.is_dynamic()) { + return false; + } + const auto& input_shape = input_pshape.get_shape(); + to_shape.resize(input_shape.size() + unsqueeze_axes.size()); + std::sort(unsqueeze_axes.begin(), unsqueeze_axes.end()); + for (size_t i = 0, j = 0, k = 0; i < to_shape.size(); ++i) { + if (j < unsqueeze_axes.size() && i == unsqueeze_axes[j]) { + to_shape[i] = 1; + j++; + } else if (k < input_shape.size()) { + to_shape[i] = input_shape[k]; + k++; + } + } + return true; +} +} // namespace + +TSUnsqueezeForward::TSUnsqueezeForward() { + MATCHER_SCOPE(TSUnsqueezeForward); + + auto transpose_label = wrap_type({any_input(), wrap_type()}); + auto unsqueeze_label = wrap_type({transpose_label, wrap_type()}); + + ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { + const auto& pattern_to_output = m.get_pattern_map(); + + auto transpose = pattern_to_output.at(transpose_label); + auto unsqueeze = pattern_to_output.at(unsqueeze_label); + if (transformation_callback(unsqueeze)) { + return false; + } + + auto transpose_order = as_type_ptr(transpose->get_input_node_shared_ptr(1)); + auto unsqueeze_axes = as_type_ptr(unsqueeze->get_input_node_shared_ptr(1)); + if (!transpose_order || !unsqueeze_axes) { + return false; + } + + std::vector non_negative_axes; + if (as_type_ptr(unsqueeze)) { + auto success = shape_to_unsqueeze_axes(unsqueeze, unsqueeze_axes, non_negative_axes); + if (!success) { + return false; + } + } else { + auto rank = unsqueeze->get_output_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START + non_negative_axes = + normalize_axes(unsqueeze->get_friendly_name(), unsqueeze_axes->cast_vector(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END + } + auto ts_order_values = transpose_order->cast_vector(); + + ts_order_values = GetOrderBeforeReduction(non_negative_axes, ts_order_values); + auto new_transpose_order = + Constant::create(transpose_order->get_element_type(), {ts_order_values.size()}, ts_order_values); + + std::shared_ptr new_unsqueeze; + if (as_type_ptr(unsqueeze)) { + std::vector new_values; + auto success = unsqueeze_axes_to_shape(transpose->input_value(0), non_negative_axes, new_values); + if (!success) { + return false; + } + auto new_const = Constant::create(unsqueeze_axes->get_element_type(), {new_values.size()}, new_values); + new_unsqueeze = unsqueeze->clone_with_new_inputs({transpose->input_value(0), new_const}); + } else { + new_unsqueeze = unsqueeze->clone_with_new_inputs({transpose->input_value(0), unsqueeze->input_value(1)}); + } + auto new_transpose = transpose->clone_with_new_inputs({new_unsqueeze, new_transpose_order}); + + replace_node(unsqueeze, new_transpose); + new_unsqueeze->set_friendly_name(transpose->get_friendly_name()); + new_transpose->set_friendly_name(unsqueeze->get_friendly_name()); + UpdateForwardSinkingAbility(new_transpose); + register_new_node(new_transpose); + copy_runtime_info({transpose, unsqueeze}, {new_transpose, new_unsqueeze}); + + return true; + }; + + auto m = std::make_shared(unsqueeze_label, matcher_name); + register_matcher(m, matcher_pass_callback); +} + +TSUnsqueezeBackward::TSUnsqueezeBackward() { + MATCHER_SCOPE(TSUnsqueezeBackward); + + auto unsqueeze_label = + wrap_type({any_input(), wrap_type()}, HasSameOutputTransposeNodes); + auto transpose_label = + wrap_type({unsqueeze_label, wrap_type()}, [](const Output& output) -> bool { + return has_static_rank()(output) && is_sinking_node(output); + }); + + ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { + const auto& pattern_to_output = m.get_pattern_map(); + + auto transpose = pattern_to_output.at(transpose_label); + auto unsqueeze = pattern_to_output.at(unsqueeze_label); + if (transformation_callback(unsqueeze)) { + return false; + } + + auto transpose_order = std::dynamic_pointer_cast(transpose->get_input_node_shared_ptr(1)); + auto unsqueeze_axes = std::dynamic_pointer_cast(unsqueeze->get_input_node_shared_ptr(1)); + if (!transpose_order || !unsqueeze_axes) + return false; + + std::vector non_negative_axes; + if (as_type_ptr(unsqueeze)) { + auto success = shape_to_unsqueeze_axes(unsqueeze, unsqueeze_axes, non_negative_axes); + if (!success) { + return false; + } + } else { + auto rank = unsqueeze->get_output_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START + non_negative_axes = + normalize_axes(unsqueeze->get_friendly_name(), unsqueeze_axes->cast_vector(), rank); + OPENVINO_SUPPRESS_DEPRECATED_END + } + + auto transpose_order_values = transpose_order->cast_vector(); + auto old_transpose_order_values = transpose_order_values; + std::vector new_values; + + if (non_negative_axes.size() == transpose_order_values.size()) { + // input is a scalar, we unsqueeze all dims + // it's enough to eliminate such Transpose + transpose->output(0).replace(unsqueeze); + return true; + } + + for (const auto& axis : non_negative_axes) { + auto it = std::find(old_transpose_order_values.begin(), old_transpose_order_values.end(), axis); + if (it != old_transpose_order_values.end()) { + new_values.push_back(it - old_transpose_order_values.begin()); + } + } + + transpose_order_values = GetOrderAfterReduction(new_values, transpose_order_values); + auto new_transpose_order = std::make_shared(transpose_order->get_element_type(), + Shape{transpose_order_values.size()}, + transpose_order_values); + + auto new_transpose = transpose->clone_with_new_inputs({unsqueeze->input_value(0), new_transpose_order}); + if (as_type_ptr(unsqueeze)) { + std::vector to_shape; + auto success = unsqueeze_axes_to_shape(new_transpose->output(0), new_values, to_shape); + if (!success) { + return false; + } + new_values = to_shape; + } + auto new_const = Constant::create(unsqueeze_axes->get_element_type(), unsqueeze_axes->get_shape(), new_values); + auto new_unsqueeze = unsqueeze->clone_with_new_inputs({new_transpose, new_const}); + + replace_node(transpose, new_unsqueeze); + copy_runtime_info({transpose, unsqueeze}, {new_transpose, new_unsqueeze}); + new_unsqueeze->set_friendly_name(transpose->get_friendly_name()); + new_transpose->set_friendly_name(unsqueeze->get_friendly_name()); + register_new_node(new_transpose); + return true; + }; + + auto m = std::make_shared(transpose_label, matcher_name); + register_matcher(m, matcher_pass_callback); +} diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp index 9925087be7d9ce..2f56e4d40c4b22 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp @@ -379,6 +379,53 @@ void RemoveSingleOutputConsumers(const NodePtr& node) { } } +std::vector GetOrderAfterReduction(const std::vector& axes_values, + const std::vector& order_values) { + size_t buffer_size = order_values.size() - axes_values.size(); + std::vector aligned_order(buffer_size, 0); + std::vector values_to_reduce(axes_values); + for (size_t i = 0; i < values_to_reduce.size(); ++i) { + values_to_reduce[i] = order_values[axes_values[i]]; + } + std::sort(values_to_reduce.begin(), values_to_reduce.end()); + for (size_t i = 0, j = 0; i < order_values.size(); ++i) { + if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) { + continue; + } + + auto lb = std::lower_bound(values_to_reduce.begin(), values_to_reduce.end(), order_values[i]); + aligned_order[j] = order_values[i] - (lb - values_to_reduce.begin()); + ++j; + } + return aligned_order; +} + +std::vector GetOrderBeforeReduction(const std::vector& axes_values, + const std::vector& order_values) { + size_t buffer_size = order_values.size() + axes_values.size(); + std::vector aligned_order(buffer_size); + + std::vector cnt_deleted(buffer_size); + int64_t cnt = 0; + for (int64_t i = 0; i < static_cast(cnt_deleted.size()); ++i) { + if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) { + cnt++; + } + cnt_deleted[i] = i - cnt; + } + + for (size_t i = 0, j = 0; i < aligned_order.size(); ++i) { + if (std::find(axes_values.begin(), axes_values.end(), i) != axes_values.end()) { + aligned_order[i] = i; + continue; + } + + aligned_order[i] = std::find(cnt_deleted.begin(), cnt_deleted.end(), order_values[j]) - cnt_deleted.begin(); + ++j; + } + return aligned_order; +} + } // namespace utils } // namespace transpose_sinking } // namespace pass diff --git a/src/common/transformations/src/transformations/utils/utils.cpp b/src/common/transformations/src/transformations/utils/utils.cpp index 695e058fe921e4..95728c63c0f27b 100644 --- a/src/common/transformations/src/transformations/utils/utils.cpp +++ b/src/common/transformations/src/transformations/utils/utils.cpp @@ -271,6 +271,9 @@ bool can_eliminate_eltwise_node(const std::shared_ptr& eltwise, case element::f32: actual_const = reinterpret_cast(data_ptr)[0]; break; + case element::f16: + actual_const = reinterpret_cast(data_ptr)[0]; + break; case element::i32: actual_const = static_cast(reinterpret_cast(data_ptr)[0]); break; diff --git a/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp b/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp index 6127453b84274d..80635215c8aef1 100644 --- a/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp +++ b/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp @@ -171,10 +171,10 @@ TEST_F(TransformationTestsF, align_mixed_fp16_fp32_4) { auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); auto convert_to_f32_1 = make_shared(input_1, element::f32); auto mvn_1 = make_shared(convert_to_f32_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto convert_to_f16_1 = make_shared(mvn_1, element::f32); auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); - auto add_1 = make_shared(mvn_1, addition_const); - auto convert_to_f16_1 = make_shared(add_1, element::f32); - auto matmul_1 = make_shared(convert_to_f16_1, input_2); + auto add_1 = make_shared(convert_to_f16_1, addition_const); + auto matmul_1 = make_shared(add_1, input_2); model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); } @@ -209,17 +209,16 @@ TEST_F(TransformationTestsF, align_mixed_fp16_fp32_mnv_with_split) { auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); auto input_2 = make_shared(element::f32, Shape{1, 3, 56, 224}); - auto convert_to_f32_1 = make_shared(input_1, element::f32); - auto split_axis = Constant::create(element::i64, Shape{}, {3}); - auto split = make_shared(convert_to_f32_1, split_axis, 4); + auto split = make_shared(input_1, split_axis, 4); auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); - auto mvn_1 = make_shared(split->output(0), reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto convert_to_f32_1 = make_shared(split->output(0), element::f32); + auto mvn_1 = make_shared(convert_to_f32_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto convert_to_f16_1 = make_shared(mvn_1, element::f32); auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); - auto add_1 = make_shared(mvn_1, addition_const); - auto convert_to_f16_1 = make_shared(add_1, element::f32); - auto matmul_1 = make_shared(convert_to_f16_1, input_2); + auto add_1 = make_shared(convert_to_f16_1, addition_const); + auto matmul_1 = make_shared(add_1, input_2); // todo: without Converts to fp16 because of GPU auto result_1 = make_shared(matmul_1); diff --git a/src/common/transformations/tests/common_optimizations/convolution_to_group_convolution_fusion.cpp b/src/common/transformations/tests/common_optimizations/convolution_to_group_convolution_fusion.cpp new file mode 100644 index 00000000000000..78acf1e27d5b21 --- /dev/null +++ b/src/common/transformations/tests/common_optimizations/convolution_to_group_convolution_fusion.cpp @@ -0,0 +1,235 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" + +using namespace testing; +using namespace ov; + +TEST_F(TransformationTestsF, ConvToGroupConvFusionSplit) { + Shape input_shape{2, 10, 14, 14}; + size_t num_splits = 5; + int axis = 1; + Shape weights_shape{3, input_shape[1] / num_splits, 1, 1}; + const auto spatial_dim_size = weights_shape.size() - 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split = std::make_shared(data, axis_node, num_splits); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1}); + concat_inputs.push_back(std::make_shared(split->output(i), + weights, + strides, + pads_begin, + pads_end, + dilations)); + } + const auto concat = std::make_shared(concat_inputs, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + { + const auto data = std::make_shared(element::f32, input_shape); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + Shape new_weights_shape = weights_shape; + new_weights_shape.insert(new_weights_shape.begin(), 1); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1}); + concat_inputs.push_back(weights); + } + const auto concat = std::make_shared(concat_inputs, 0); + const auto conv = + std::make_shared(data, concat, strides, pads_begin, pads_end, dilations); + function_ref = std::make_shared(conv, ParameterVector{data}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, ConvToGroupConvFusionVariadicSplit) { + Shape input_shape{2, 10, 14, 14}; + size_t num_splits = 5; + int axis = 1; + Shape weights_shape{3, input_shape[1] / num_splits, 1, 1}; + const auto spatial_dim_size = weights_shape.size() - 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split_lengths = + opset10::Constant::create(element::i32, Shape{num_splits}, std::vector(num_splits, 2)); + const auto split = std::make_shared(data, axis_node, split_lengths); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1}); + concat_inputs.push_back(std::make_shared(split->output(i), + weights, + strides, + pads_begin, + pads_end, + dilations)); + } + const auto concat = std::make_shared(concat_inputs, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + { + const auto data = std::make_shared(element::f32, input_shape); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + Shape new_weights_shape = weights_shape; + new_weights_shape.insert(new_weights_shape.begin(), 1); + for (size_t i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, new_weights_shape, {i + 1}); + concat_inputs.push_back(weights); + } + const auto concat = std::make_shared(concat_inputs, 0); + const auto conv = + std::make_shared(data, concat, strides, pads_begin, pads_end, dilations); + function_ref = std::make_shared(conv, ParameterVector{data}); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitInvalidAxis) { + Shape input_shape{2, 10, 14, 14}; + int num_splits = 2; + int axis = 2; + Shape weights_shape{3, input_shape[1], 1, 1}; + const auto spatial_dim_size = weights_shape.size() - 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split = std::make_shared(data, axis_node, num_splits); + OutputVector concat_inputs; + concat_inputs.reserve(num_splits); + for (int i = 0; i < num_splits; i++) { + const auto weights = opset10::Constant::create(element::f32, weights_shape, {i + 1}); + concat_inputs.push_back(std::make_shared(split->output(i), + weights, + strides, + pads_begin, + pads_end, + dilations)); + } + const auto concat = std::make_shared(concat_inputs, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionSplitNotMatchingConvAttributes) { + Shape input_shape{2, 10, 14, 14}; + size_t num_splits = 2; + int axis = 1; + const auto spatial_dim_size = 2; + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split = std::make_shared(data, axis_node, num_splits); + + const auto weights1 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 2, 2}, {1}); + Strides strides1(spatial_dim_size, 1); + CoordinateDiff pads_begin1(spatial_dim_size, 1); + CoordinateDiff pads_end1(spatial_dim_size, 1); + Strides dilations1(spatial_dim_size, 1); + const auto conv1 = std::make_shared(split->output(0), + weights1, + strides1, + pads_begin1, + pads_end1, + dilations1); + + const auto weights2 = opset10::Constant::create(element::f32, Shape{3, input_shape[1] / num_splits, 4, 4}, {1}); + Strides strides2(spatial_dim_size, 1); + CoordinateDiff pads_begin2(spatial_dim_size, 2); + CoordinateDiff pads_end2(spatial_dim_size, 2); + Strides dilations2(spatial_dim_size, 1); + const auto conv2 = std::make_shared(split->output(1), + weights2, + strides2, + pads_begin2, + pads_end2, + dilations2); + const auto concat = std::make_shared(OutputVector{conv1, conv2}, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + +TEST_F(TransformationTestsF, NegativeConvToGroupConvFusionVariadicSplitUnevenSplitLengths) { + Shape input_shape{2, 10, 14, 14}; + int axis = 1; + const auto spatial_dim_size = 2; + Strides strides(spatial_dim_size, 1); + CoordinateDiff pads_begin(spatial_dim_size, 0); + CoordinateDiff pads_end(spatial_dim_size, 0); + Strides dilations(spatial_dim_size, 1); + + { + const auto data = std::make_shared(element::f32, input_shape); + const auto axis_node = opset10::Constant::create(element::i32, Shape{}, {axis}); + const auto split_lengths = opset10::Constant::create(element::i32, + Shape{2}, + std::vector{3, static_cast(input_shape[1]) - 3}); + const auto split = std::make_shared(data, axis_node, split_lengths); + const auto weights1 = opset10::Constant::create(element::f32, Shape{3, 3, 1, 1}, {1}); + const auto conv1 = std::make_shared(split->output(0), + weights1, + strides, + pads_begin, + pads_end, + dilations); + const auto weights2 = opset10::Constant::create(element::f32, Shape{3, 7, 1, 1}, {2}); + const auto conv2 = std::make_shared(split->output(1), + weights2, + strides, + pads_begin, + pads_end, + dilations); + const auto concat = std::make_shared(OutputVector{conv1, conv2}, axis); + function = std::make_shared(concat, ParameterVector{data}); + manager.register_pass(); + } + + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} diff --git a/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp b/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp index 3a8bca535b9cef..1cf4214d77fef3 100644 --- a/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp +++ b/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp @@ -442,8 +442,6 @@ TEST(TransformationTests, keep_precission_sensitive_fp32_4) { auto matmul_1 = make_shared(unsqueeze_1, input_2); // marking nodes to be kept in fp32 for mixed precision - disable_fp16_compression(addition_const); - disable_fp16_compression(unsqueeze_1); disable_fp16_compression(mvn_1); model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); @@ -487,8 +485,6 @@ TEST(TransformationTests, keep_precission_sensitive_fp32_5) { auto matmul_1 = make_shared(unsqueeze_1, input_2); // marking nodes to be kept in fp32 for mixed precision - disable_fp16_compression(addition_const); - disable_fp16_compression(unsqueeze_1); disable_fp16_compression(normalizel2_1); model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); diff --git a/src/common/transformations/tests/common_optimizations/nonzero_fusion_test.cpp b/src/common/transformations/tests/common_optimizations/nonzero_fusion_test.cpp new file mode 100644 index 00000000000000..4e96a976e02c13 --- /dev/null +++ b/src/common/transformations/tests/common_optimizations/nonzero_fusion_test.cpp @@ -0,0 +1,130 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" + +using namespace testing; + +enum NonZeroType { I32, I64, NONE }; + +struct NonZeroFusionBuilder { + NonZeroFusionBuilder() = default; + NonZeroFusionBuilder(const std::vector& props) : branch_props(props) {} + + std::shared_ptr getOriginal() { + const auto input = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); + ov::NodeVector results; + for (size_t i = 0; i < branch_props.size(); ++i) { + std::shared_ptr nonzero; + switch (branch_props[i]) { + case NonZeroType::I32: + nonzero = std::make_shared(input, ov::element::i32); + break; + case NonZeroType::I64: + nonzero = std::make_shared(input, ov::element::i64); + break; + default: + nonzero = input; + break; + } + auto last_node = std::make_shared(nonzero); + last_node->set_friendly_name("last_node_" + std::to_string(i)); + results.push_back(last_node); + } + return std::make_shared(results, ov::ParameterVector{input}); + }; + + std::shared_ptr getReference() { + const auto input = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); + + std::shared_ptr i32_node; + std::shared_ptr i64_node; + ov::NodeVector results; + for (size_t i = 0; i < branch_props.size(); ++i) { + std::shared_ptr nonzero; + if (branch_props[i] == NonZeroType::I32) { + nonzero = i32_node ? i32_node : std::make_shared(input, ov::element::i32); + if (!i32_node) + i32_node = nonzero; + } else if (branch_props[i] == NonZeroType::I64) { + nonzero = i64_node ? i64_node : std::make_shared(input, ov::element::i64); + if (!i64_node) + i64_node = nonzero; + } else { + nonzero = input; + } + auto last_node = std::make_shared(nonzero); + last_node->set_friendly_name("last_node_" + std::to_string(i)); + results.push_back(last_node); + } + return std::make_shared(results, ov::ParameterVector{input}); + } + + std::vector branch_props; +}; + +class NonZeroFusionTests : public testing::WithParamInterface>, public TransformationTestsF { +public: + NonZeroFusionTests() : TransformationTestsF() { + comparator.enable(FunctionsComparator::CONSUMERS_COUNT); + } + + static std::string getTestCaseName(testing::TestParamInfo> obj) { + const std::vector testValues = obj.param; + std::ostringstream result; + result << "branch_props_{"; + for (const auto& value : testValues) { + switch (value) { + case NonZeroType::I32: + result << "nonzero_i32,"; + break; + case NonZeroType::I64: + result << "nonzero_i64,"; + break; + default: + result << "wo_nonzero,"; + break; + } + } + result << "}"; + return result.str(); + } + +protected: + void SetUp() override { + TransformationTestsF::SetUp(); + const auto branch_props = GetParam(); + builder = NonZeroFusionBuilder(branch_props); + manager.register_pass(); + } + + NonZeroFusionBuilder builder; +}; + +TEST_P(NonZeroFusionTests, NonZeroFusion) { + model = builder.getOriginal(); + model_ref = builder.getReference(); +} + +namespace NonZeroFusionTestsInstantiation { +std::vector> test_params{std::vector(5, I32), + std::vector(5, I64), + std::vector(2, NONE), + {I32, I64, I32, I64, I32}, + {I32, I64, NONE, I64, I32}, + {NONE, I64, NONE, I64, I32}}; + +INSTANTIATE_TEST_SUITE_P(TransformationTestsF, + NonZeroFusionTests, + ::testing::ValuesIn(test_params), + NonZeroFusionTests::getTestCaseName); + +} // namespace NonZeroFusionTestsInstantiation diff --git a/src/common/transformations/tests/common_optimizations/nop_elimination.cpp b/src/common/transformations/tests/common_optimizations/nop_elimination.cpp index 77d595e621f812..0932709116a6bd 100644 --- a/src/common/transformations/tests/common_optimizations/nop_elimination.cpp +++ b/src/common/transformations/tests/common_optimizations/nop_elimination.cpp @@ -971,6 +971,7 @@ class EliminateEltwiseTests : public testing::WithParamInterface types{ element::f32, + element::f16, element::f64, element::i32, element::u32, @@ -1310,3 +1311,31 @@ TEST(SplitConcatElimination, no_sequence_found) { EXPECT_EQ(count_ops_of_type(model), 1) << "SplitConcatElimination transformation has failed. " "The number of Split ops is not 1"; } + +TEST(nop_elimination, gather_to_squeeze) { + auto generate_func = [](int64_t gather_axis) { + ov::Shape shape{3, 3, 4, 4}; + shape[gather_axis] = 1; + auto arg = std::make_shared(element::f32, shape); + auto indices = op::Constant::create(element::i64, Shape{}, vector{0}); + auto axis = op::Constant::create(element::i64, Shape{}, vector{gather_axis}); + auto gather = std::make_shared(arg, indices, axis); + return std::make_shared(NodeVector{gather}, ParameterVector{arg}); + }; + + auto func_axis_0 = generate_func(0); + auto func_axis_1 = generate_func(1); + auto func_axis_2 = generate_func(2); + auto func_axis_3 = generate_func(3); + pass::Manager pass_manager; + pass_manager.register_pass(); + auto run_and_check = [&](std::shared_ptr& func) { + pass_manager.run_passes(func); + EXPECT_EQ(count_ops_of_type(func), 0); + EXPECT_EQ(count_ops_of_type(func), 1); + }; + run_and_check(func_axis_0); + run_and_check(func_axis_1); + run_and_check(func_axis_2); + run_and_check(func_axis_3); +} diff --git a/src/common/transformations/tests/common_optimizations/reduce_reshape_fusion_tests.cpp b/src/common/transformations/tests/common_optimizations/reduce_reshape_fusion_tests.cpp index 8234c2030ba773..106d7838e880bc 100644 --- a/src/common/transformations/tests/common_optimizations/reduce_reshape_fusion_tests.cpp +++ b/src/common/transformations/tests/common_optimizations/reduce_reshape_fusion_tests.cpp @@ -10,11 +10,13 @@ #include #include #include +#include #include #include #include "common_test_utils/ngraph_test_utils.hpp" +using namespace std; using namespace testing; using namespace ov; using namespace opset9; @@ -154,3 +156,21 @@ TEST_F(TransformationTestsF, ReduceMeanReshapeFusionSkipIfMoreThanOneReduceConsu model = std::make_shared(NodeVector{reshape, add}, ParameterVector{input}); manager.register_pass(); } + +TEST(TransformationTests, ReduceMeanReshapeFusionAssertValidOutputShape) { + const auto input = make_shared(element::f32, PartialShape{1, 16, 16, 24}); + const auto reduce_axes = Constant::create(element::i64, Shape{2}, {1, 2}); + const auto reduce_mean = make_shared(input, reduce_axes, false); + const auto target_shape = Constant::create(element::i64, Shape{4}, {1, 1, 1, 24}); + const auto reshape = make_shared(reduce_mean, target_shape, false); + const auto order = Constant::create(element::i64, Shape{4}, {0, 3, 1, 2}); + const auto transpose = make_shared(reshape, order); + + auto model = make_shared(NodeVector{transpose}, ParameterVector{input}); + + pass::Manager manager; + manager.set_per_pass_validation(false); + manager.register_pass(); + manager.register_pass(); + ASSERT_NO_THROW(manager.run_passes(model)); +} diff --git a/src/common/transformations/tests/common_optimizations/softmax_fusion.cpp b/src/common/transformations/tests/common_optimizations/softmax_fusion.cpp index 4027c8500fe51f..e64d8b3daac478 100644 --- a/src/common/transformations/tests/common_optimizations/softmax_fusion.cpp +++ b/src/common/transformations/tests/common_optimizations/softmax_fusion.cpp @@ -17,6 +17,7 @@ using namespace testing; using namespace ngraph; class SoftmaxFusionFixture : public ::testing::TestWithParam> {}; +class SoftmaxFusionSimplePatternFixture : public ::testing::TestWithParam> {}; TEST_P(SoftmaxFusionFixture, SoftmaxFusion) { Shape shape{1, 1, 256}; @@ -67,25 +68,16 @@ INSTANTIATE_TEST_SUITE_P(SoftmaxFusionTests, std::make_tuple(2, -1), std::make_tuple(2, 2))); -class NegativeSoftmaxFusionFixture - : public ::testing::TestWithParam, std::vector>> {}; - -TEST_P(NegativeSoftmaxFusionFixture, NegativeSoftmaxFusion) { - Shape shape{1, 1, 256}; +TEST_P(SoftmaxFusionSimplePatternFixture, SoftmaxFusionSimplePatternTest) { + Shape shape{1, 3, 256, 256}; auto params = GetParam(); - auto reduce_max_axes_val = std::get<0>(params); - auto reduce_sum_axes_val = std::get<1>(params); + auto reduce_axis_val = std::get<0>(params); std::shared_ptr f(nullptr), f_ref(nullptr); { auto data = std::make_shared(element::f32, shape); - auto reduce_max_axes = - opset6::Constant::create(element::i64, Shape{reduce_max_axes_val.size()}, reduce_max_axes_val); - auto reduce_max = std::make_shared(data, reduce_max_axes); - auto sub = std::make_shared(data, reduce_max); - auto exp = std::make_shared(sub); - auto reduce_sum_axes = - opset6::Constant::create(element::i64, Shape{reduce_sum_axes_val.size()}, reduce_sum_axes_val); - auto reduce_sum = std::make_shared(exp, reduce_sum_axes); + auto exp = std::make_shared(data); + auto reduce_axis = opset6::Constant::create(element::i64, Shape{}, {reduce_axis_val}); + auto reduce_sum = std::make_shared(exp, reduce_axis, true); auto div = std::make_shared(exp, reduce_sum); f = std::make_shared(NodeVector{div}, ParameterVector{data}); @@ -100,16 +92,10 @@ TEST_P(NegativeSoftmaxFusionFixture, NegativeSoftmaxFusion) { } { auto data = std::make_shared(element::f32, shape); - auto reduce_max_axes = - opset6::Constant::create(element::i64, Shape{reduce_max_axes_val.size()}, reduce_max_axes_val); - auto reduce_max = std::make_shared(data, reduce_max_axes); - auto sub = std::make_shared(data, reduce_max); - auto exp = std::make_shared(sub); - auto reduce_sum_axes = - opset6::Constant::create(element::i64, Shape{reduce_sum_axes_val.size()}, reduce_sum_axes_val); - auto reduce_sum = std::make_shared(exp, reduce_sum_axes); - auto div = std::make_shared(exp, reduce_sum); - f_ref = std::make_shared(NodeVector{div}, ParameterVector{data}); + if (reduce_axis_val < 0) + reduce_axis_val += shape.size(); + auto softmax = std::make_shared(data, reduce_axis_val); + f_ref = std::make_shared(NodeVector{softmax}, ParameterVector{data}); } auto fc = @@ -118,6 +104,50 @@ TEST_P(NegativeSoftmaxFusionFixture, NegativeSoftmaxFusion) { ASSERT_TRUE(res.valid) << res.message; } +INSTANTIATE_TEST_SUITE_P(SoftmaxFusionSimplePatternTests, + SoftmaxFusionSimplePatternFixture, + ::testing::Values(std::make_tuple(0), + std::make_tuple(1), + std::make_tuple(2), + std::make_tuple(-1), + std::make_tuple(-2))); + +class NegativeSoftmaxFusionFixture + : public ::testing::TestWithParam, std::vector>> {}; + +TEST_P(NegativeSoftmaxFusionFixture, NegativeSoftmaxFusion) { + // ReduceMax arguments do not match conditions, therefore these nodes + // are not included into final SoftMax node + Shape shape{1, 1, 256}; + auto params = GetParam(); + auto reduce_max_axes_val = std::get<0>(params); + auto reduce_sum_axes_val = std::get<1>(params); + std::shared_ptr f(nullptr); + + auto data = std::make_shared(element::f32, shape); + auto reduce_max_axes = + opset6::Constant::create(element::i64, Shape{reduce_max_axes_val.size()}, reduce_max_axes_val); + auto reduce_max = std::make_shared(data, reduce_max_axes); + auto sub = std::make_shared(data, reduce_max); + auto exp = std::make_shared(sub); + auto reduce_sum_axes = + opset6::Constant::create(element::i64, Shape{reduce_sum_axes_val.size()}, reduce_sum_axes_val); + auto reduce_sum = std::make_shared(exp, reduce_sum_axes); + auto div = std::make_shared(exp, reduce_sum); + f = std::make_shared(NodeVector{div}, ParameterVector{data}); + + auto unh = std::make_shared(); + pass::Manager m; + m.register_pass(unh); + m.register_pass(); + m.register_pass(); + m.register_pass(unh); + m.run_passes(f); + ASSERT_NO_THROW(check_rt_info(f)); + ASSERT_EQ(count_ops_of_type(f), 1); + ASSERT_EQ(count_ops_of_type(f), 1); +} + INSTANTIATE_TEST_SUITE_P(NegativeSoftmaxFusionTests, NegativeSoftmaxFusionFixture, ::testing::ValuesIn(std::vector, std::vector>>{ diff --git a/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp b/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp new file mode 100644 index 00000000000000..7504cd378ebba6 --- /dev/null +++ b/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp @@ -0,0 +1,147 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" + +using namespace testing; + +namespace { +constexpr bool WITH_AXES = true; +constexpr bool WITHOUT_AXES = false; + +std::shared_ptr create_v11_model(const bool with_axes, + const ov::opset11::Interpolate::ShapeCalcMode shape_calc_mode) { + auto attributes = ov::opset11::Interpolate::InterpolateAttrs{}; + attributes.shape_calculation_mode = shape_calc_mode; + attributes.pads_begin = {0, 0}; + attributes.pads_end = {0, 0}; + + const auto input = std::make_shared(ov::element::i32, ov::Shape{1, 2, 10, 10}); + std::shared_ptr scales_or_sizes; + std::shared_ptr interpolate; + + const size_t num_scales_or_sizes = with_axes ? 2 : 4; + if (shape_calc_mode == ov::opset11::Interpolate::ShapeCalcMode::SCALES) { + scales_or_sizes = std::make_shared(ov::element::f32, ov::Shape{num_scales_or_sizes}); + } else { + scales_or_sizes = std::make_shared(ov::element::i32, ov::Shape{num_scales_or_sizes}); + } + + ov::ParameterVector model_params; + model_params.push_back(input); + model_params.push_back(scales_or_sizes); + if (with_axes) { + const auto axes = std::make_shared(ov::element::i32, ov::Shape{2}); + model_params.push_back(axes); + interpolate = std::make_shared(input, scales_or_sizes, axes, attributes); + } else { + interpolate = std::make_shared(input, scales_or_sizes, attributes); + } + interpolate->set_friendly_name("interpolate11"); + + return std::make_shared(interpolate->outputs(), model_params); +} + +std::shared_ptr create_v4_model(const bool with_axes, + const ov::opset4::Interpolate::ShapeCalcMode shape_calc_mode) { + auto attributes = ov::opset4::Interpolate::InterpolateAttrs{}; + attributes.shape_calculation_mode = shape_calc_mode; + attributes.pads_begin = {0, 0}; + attributes.pads_end = {0, 0}; + + const auto input = std::make_shared(ov::element::i32, ov::Shape{1, 2, 10, 10}); + std::shared_ptr output_shape; + std::shared_ptr scales; + std::shared_ptr interpolate; + + ov::ParameterVector model_params; + model_params.push_back(input); + + const size_t num_scales_or_sizes = with_axes ? 2 : 4; + if (shape_calc_mode == ov::opset4::Interpolate::ShapeCalcMode::SCALES) { + scales = std::make_shared(ov::element::f32, ov::Shape{num_scales_or_sizes}); + model_params.push_back(std::dynamic_pointer_cast(scales)); + output_shape = ov::opset4::Constant::create(ov::element::i32, ov::Shape{1}, {1}); + + } else { + output_shape = std::make_shared(ov::element::i32, ov::Shape{num_scales_or_sizes}); + model_params.push_back(std::dynamic_pointer_cast(output_shape)); + scales = ov::opset4::Constant::create(ov::element::f32, ov::Shape{1}, {1.0f}); + } + + if (with_axes) { + const auto axes = std::make_shared(ov::element::i32, ov::Shape{2}); + model_params.push_back(axes); + interpolate = std::make_shared(input, output_shape, scales, axes, attributes); + } else { + interpolate = std::make_shared(input, output_shape, scales, attributes); + } + interpolate->set_friendly_name("interpolate11"); + + return std::make_shared(interpolate->outputs(), model_params); +} + +} // namespace + +TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_scales) { + manager.register_pass(); + function = create_v11_model(WITH_AXES, ov::opset11::Interpolate::ShapeCalcMode::SCALES); + function_ref = create_v4_model(WITH_AXES, ov::opset4::Interpolate::ShapeCalcMode::SCALES); +} + +TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_sizes) { + manager.register_pass(); + function = create_v11_model(WITH_AXES, ov::opset11::Interpolate::ShapeCalcMode::SIZES); + function_ref = create_v4_model(WITH_AXES, ov::opset4::Interpolate::ShapeCalcMode::SIZES); +} + +TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_scales_no_axes) { + manager.register_pass(); + function = create_v11_model(WITHOUT_AXES, ov::opset11::Interpolate::ShapeCalcMode::SCALES); + function_ref = create_v4_model(WITHOUT_AXES, ov::opset4::Interpolate::ShapeCalcMode::SCALES); +} + +TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_sizes_no_axes) { + manager.register_pass(); + function = create_v11_model(WITHOUT_AXES, ov::opset11::Interpolate::ShapeCalcMode::SIZES); + function_ref = create_v4_model(WITHOUT_AXES, ov::opset4::Interpolate::ShapeCalcMode::SIZES); +} + +namespace { +std::shared_ptr create_non_downgradeable_model(const ov::opset11::Interpolate::InterpolateMode mode) { + auto attributes = ov::opset11::Interpolate::InterpolateAttrs{}; + attributes.mode = mode; + attributes.shape_calculation_mode = ov::opset11::Interpolate::ShapeCalcMode::SCALES; + attributes.pads_begin = {0, 0}; + attributes.pads_end = {0, 0}; + + const auto input = std::make_shared(ov::element::i32, ov::Shape{1, 2, 10, 10}); + const auto scales = std::make_shared(ov::element::f32, ov::Shape{2}); + const auto axes = std::make_shared(ov::element::i32, ov::Shape{2}); + + const auto interpolate = std::make_shared(input, scales, axes, attributes); + interpolate->set_friendly_name("interpolate11"); + + return std::make_shared(interpolate->outputs(), ov::ParameterVector{input, scales, axes}); +} +} // namespace + +TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_bicubic_pillow) { + function = create_non_downgradeable_model(ov::opset11::Interpolate::InterpolateMode::BICUBIC_PILLOW); + manager.register_pass(); +} + +TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_bilinear_pillow) { + function = create_non_downgradeable_model(ov::opset11::Interpolate::InterpolateMode::BILINEAR_PILLOW); + manager.register_pass(); +} diff --git a/src/common/transformations/tests/op_conversions/convert_topk11_downgrade_test.cpp b/src/common/transformations/tests/op_conversions/convert_topk11_downgrade_test.cpp index 15bde748c5b5a3..65fe2db35a78cc 100644 --- a/src/common/transformations/tests/op_conversions/convert_topk11_downgrade_test.cpp +++ b/src/common/transformations/tests/op_conversions/convert_topk11_downgrade_test.cpp @@ -47,18 +47,34 @@ TEST_F(TransformationTestsF, ConvertTopK11ToTopK3) { } } -TEST_F(TransformationTestsF, ConvertTopK11ToTopK3_fail) { - const auto input = std::make_shared(ov::element::i32, ov::Shape{2, 3, 4}); - const auto k = std::make_shared(ov::element::i8, ov::Shape{}); - const auto topk = std::make_shared(input, - k, - -2, - ov::op::TopKMode::MAX, - ov::op::TopKSortType::SORT_VALUES, - ov::element::i64, - true); // stable sort on - topk->set_friendly_name("topk11"); +TEST_F(TransformationTestsF, ConvertTopK11ToTopK3StableMode) { + { + const auto input = std::make_shared(ov::element::i32, ov::Shape{2, 3, 4}); + const auto k = std::make_shared(ov::element::i8, ov::Shape{}); + const auto topk = std::make_shared(input, + k, + -2, + ov::op::TopKMode::MAX, + ov::op::TopKSortType::SORT_VALUES, + ov::element::i64, + true); + topk->set_friendly_name("topk11"); + + function = std::make_shared(topk->outputs(), ov::ParameterVector{input, k}); + manager.register_pass(); + } - function = std::make_shared(topk->outputs(), ov::ParameterVector{input, k}); - manager.register_pass(); + { + const auto input = std::make_shared(ov::element::i32, ov::Shape{2, 3, 4}); + const auto k = std::make_shared(ov::element::i8, ov::Shape{}); + const auto topk = std::make_shared(input, + k, + -2, + ov::op::TopKMode::MAX, + ov::op::TopKSortType::SORT_VALUES, + ov::element::i64); + topk->set_friendly_name("topk11"); + + function_ref = std::make_shared(topk->outputs(), ov::ParameterVector{input, k}); + } } diff --git a/src/common/transformations/tests/op_conversions/gather_normalize_negative_indices_test.cpp b/src/common/transformations/tests/op_conversions/gather_normalize_negative_indices_test.cpp index dc4b42efa97b2d..6c27102627f9e2 100644 --- a/src/common/transformations/tests/op_conversions/gather_normalize_negative_indices_test.cpp +++ b/src/common/transformations/tests/op_conversions/gather_normalize_negative_indices_test.cpp @@ -43,7 +43,9 @@ TEST_F(TransformationTestsF, GatherNegativeIndicesNormalize) { ngraph::opset7::Constant::create(indices_type, ngraph::Shape{}, {1}), ngraph::opset7::Constant::create(indices_type, ngraph::Shape{}, {0})); auto add = std::make_shared(input_gather, indices); + OPENVINO_SUPPRESS_DEPRECATED_START auto const_add = ngraph::get_constant_from_source(add); + OPENVINO_SUPPRESS_DEPRECATED_END if (const_add == nullptr) throw ngraph::ngraph_error("indices should've been constant folded"); auto gather = std::make_shared(data, const_add, axis); @@ -78,7 +80,9 @@ TEST_F(TransformationTestsF, GatherNegativeIndicesNormalize_neg_axis) { ngraph::opset7::Constant::create(indices_type, ngraph::Shape{}, {1}), ngraph::opset7::Constant::create(indices_type, ngraph::Shape{}, {0})); auto add = std::make_shared(input_gather, indices); + OPENVINO_SUPPRESS_DEPRECATED_START auto const_add = ngraph::get_constant_from_source(add); + OPENVINO_SUPPRESS_DEPRECATED_END if (const_add == nullptr) throw ngraph::ngraph_error("indices should've been constant folded"); auto gather = std::make_shared(data, const_add, axis); @@ -113,7 +117,9 @@ TEST_F(TransformationTestsF, GatherNegativeIndicesNormalize_dif_input_types) { ngraph::opset7::Constant::create(indices_type, ngraph::Shape{}, {1}), ngraph::opset7::Constant::create(indices_type, ngraph::Shape{}, {0})); auto add = std::make_shared(input_gather, indices); + OPENVINO_SUPPRESS_DEPRECATED_START auto const_add = ngraph::get_constant_from_source(add); + OPENVINO_SUPPRESS_DEPRECATED_END if (const_add == nullptr) throw ngraph::ngraph_error("indices should've been constant folded"); auto gather = std::make_shared(data, const_add, axis); diff --git a/src/common/transformations/tests/prelu_fusion.cpp b/src/common/transformations/tests/prelu_fusion.cpp index 3c2df005f3d0a2..ba878750cddff4 100644 --- a/src/common/transformations/tests/prelu_fusion.cpp +++ b/src/common/transformations/tests/prelu_fusion.cpp @@ -156,4 +156,29 @@ TEST_F(TransformationTestsF, PReluFusionAbsSubMulMulAdd) { const auto prelu = make_shared(data, prelu_const); function_ref = make_shared(NodeVector{prelu}, ParameterVector{data}); } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); +} + +TEST_F(TransformationTestsF, PReluFusionNegReluMulAdd) { + using namespace std; + using namespace ov::opset10; + { + const auto data = make_shared(element::f32, Shape{2, 12}); + const auto relu_pos = make_shared(data); + const auto neg = make_shared(data); + const auto relu_neg = make_shared(neg); + const auto mul_const = Constant::create(element::f32, Shape{1}, {0.235}); + const auto mul = make_shared(relu_neg, mul_const); + const auto add = make_shared(relu_pos, mul); + function = make_shared(NodeVector{add}, ParameterVector{data}); + + manager.register_pass(); + } + { + const auto data = make_shared(element::f32, Shape{2, 12}); + const auto prelu_const = Constant::create(element::f32, Shape{1}, {-0.235}); + const auto prelu = make_shared(data, prelu_const); + function_ref = make_shared(NodeVector{prelu}, ParameterVector{data}); + } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); } diff --git a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp index 10de11c2893071..4cec6952975aff 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_common_test.cpp @@ -11,8 +11,11 @@ #include "transformations/transpose_sinking/ts_data_movement.hpp" #include "transformations/transpose_sinking/ts_interpolate.hpp" #include "transformations/transpose_sinking/ts_reduction.hpp" +#include "transformations/transpose_sinking/ts_slice.hpp" #include "transformations/transpose_sinking/ts_split.hpp" +#include "transformations/transpose_sinking/ts_squeeze.hpp" #include "transformations/transpose_sinking/ts_unary.hpp" +#include "transformations/transpose_sinking/ts_unsqueeze.hpp" #include "ts_test_utils.hpp" using namespace std; @@ -183,6 +186,34 @@ class InterpolateFactory : public IFactory { FactoryPtr CreateInterpolateFactory(const std::string& type_name, bool is_reference) { return std::make_shared(type_name, is_reference); } + +class SliceFactory : public IFactory { +public: + explicit SliceFactory(const std::string& type_name) : IFactory(type_name) {} + NodePtr create(const OutputVector& parent_nodes) const override { + return std::make_shared(parent_nodes[0], + parent_nodes[1], + parent_nodes[2], + parent_nodes[3], + parent_nodes[4]); + } +}; + +FactoryPtr CreateSliceFactory(const std::string& type_name) { + return std::make_shared(type_name); +} + +class ReshapeFactory : public IFactory { +public: + explicit ReshapeFactory(const std::string& type_name) : IFactory(type_name) {} + NodePtr create(const OutputVector& parent_nodes) const override { + return std::make_shared(parent_nodes[0], parent_nodes[1], false); + } +}; + +FactoryPtr CreateReshapeFactory(const std::string& type_name) { + return std::make_shared(type_name); +} // ---------------------------------------------------------------------------- #undef CREATE_UNARY_FACTORY @@ -214,6 +245,12 @@ FactoryPtr CreateInterpolateFactory(const std::string& type_name, bool is_refere #undef CREATE_INTERPOLATE_FACTORY #define CREATE_INTERPOLATE_FACTORY(type_name, reference_flag) CreateInterpolateFactory(#type_name, reference_flag) + +#undef CREATE_SLICE_FACTORY +#define CREATE_SLICE_FACTORY(type_name) CreateSliceFactory(#type_name) + +#undef CREATE_RESHAPE_FACTORY +#define CREATE_RESHAPE_FACTORY(type_name) CreateReshapeFactory(#type_name) // ---------------------------------------------------------------------------- struct Preprocessing { @@ -651,7 +688,7 @@ auto test_forward_squeeze = []() { TestCase test_case; // Initialize common attributes - test_case.transformation = CREATE_PASS_FACTORY(TSReductionForward); + test_case.transformation = CREATE_PASS_FACTORY(TSSqueezeForward); test_case.num_main_ops = {1}; test_case.inputs_to_main = { parameter(element::f32, {32, 1, 2, 1}), @@ -685,7 +722,7 @@ auto test_forward_unsqueeze = []() { TestCase test_case; // Initialize common attributes - test_case.transformation = CREATE_PASS_FACTORY(TSReductionForward); + test_case.transformation = CREATE_PASS_FACTORY(TSUnsqueezeForward); test_case.num_main_ops = {1}; test_case.inputs_to_main = { parameter(element::f32, {32, 3, 2, 1}), @@ -722,6 +759,128 @@ auto test_forward_unsqueeze = []() { INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonUnsqueezeForward, TransposeSinkingTestFixture, test_forward_unsqueeze()); +auto test_forward_slice = []() { + TestCase test_case; + + // Initialize common attributes + test_case.transformation = CREATE_PASS_FACTORY(TSSliceForward); + test_case.num_main_ops = {1}; + test_case.inputs_to_main = { + parameter(element::f32, {6, 4, 5, 3}), + constant(element::i32, {3}, {1, 2, 3}), + constant(element::i32, {3}, {0, 4, 11}), + constant(element::i32, {3}, {1, 2, -1}), + constant(element::i32, {3}, {0, 1, 2}), + }; + + // Test model description: + test_case.model.preprocess_inputs_to_main = {{set_transpose_for}, {{0}}}; + test_case.model.main_op = {CREATE_SLICE_FACTORY(SliceFactory)}; + test_case.model.model_template = create_model; + + // Reference model description: + auto set_specific_gather_for = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector result = out_vec; + for (const auto& idx : idxs) { + const auto& out = out_vec[idx]; + vector transpose_order(out_vec[0].get_shape().size()); + iota(transpose_order.begin(), transpose_order.end(), 0); + reverse(transpose_order.begin(), transpose_order.end()); + auto data = make_shared(element::i32, Shape{transpose_order.size()}, transpose_order); + auto axis = make_shared(element::i32, Shape{}, 0); + auto transpose = make_shared(data, out, axis); + result[idx] = transpose; + } + return result; + }; + test_case.model_ref.preprocess_inputs_to_main = {{set_specific_gather_for}, {{4}}}; + test_case.model_ref.main_op = {CREATE_SLICE_FACTORY(Slice)}; + test_case.model_ref.preprocess_outputs_of_main = {{set_transpose_for}, {{0}}}; + test_case.model_ref.model_template = create_model; + + return wrapper(test_case); +}; + +INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonSliceForward, TransposeSinkingTestFixture, test_forward_slice()); + +auto test_forward_reshape_squeeze = []() { + TestCase test_case; + + // Initialize common attributes + test_case.transformation = CREATE_PASS_FACTORY(TSSqueezeForward); + test_case.num_main_ops = {1}; + test_case.inputs_to_main = { + parameter(element::f32, {6, 1, 5, 1, 4}), + constant(element::i32, {3}, {4, 5, 6}), + }; + + // Test model description: + test_case.model.preprocess_inputs_to_main = {{set_transpose_for}, {{0}}}; + test_case.model.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model.model_template = create_model; + + // Reference model description: + auto new_constant = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector new_out_vec(out_vec.size()); + new_out_vec[0] = out_vec[0]; + new_out_vec[1] = + make_shared(out_vec[1].get_element_type(), out_vec[1].get_shape(), std::vector{6, 5, 4}); + return new_out_vec; + }; + test_case.model_ref.preprocess_inputs_to_main = {{new_constant}, {{1}}}; + test_case.model_ref.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model_ref.preprocess_outputs_of_main = {{set_transpose_for}, {{0}}}; + test_case.model_ref.model_template = create_model; + + return wrapper(test_case); +}; + +INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonReshapeSqueezeForward, + TransposeSinkingTestFixture, + test_forward_reshape_squeeze()); + +auto test_forward_reshape_unsqueeze = []() { + TestCase test_case; + + // Initialize common attributes + test_case.transformation = CREATE_PASS_FACTORY(TSUnsqueezeForward); + test_case.num_main_ops = {1}; + test_case.inputs_to_main = { + parameter(element::f32, {6, 5, 4}), + constant(element::i32, {5}, {4, 1, 5, 1, 6}), + }; + + // Test model description: + test_case.model.preprocess_inputs_to_main = {{set_transpose_for}, {{0}}}; + test_case.model.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model.model_template = create_model; + + // Reference model description: + auto new_transpose = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector new_out_vec(out_vec.size()); + auto order = make_shared(element::i32, Shape{5}, std::vector{4, 1, 2, 3, 0}); + new_out_vec[0] = make_shared(out_vec[0], order); + return new_out_vec; + }; + auto new_constant = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector new_out_vec(out_vec.size()); + new_out_vec[0] = out_vec[0]; + new_out_vec[1] = make_shared(out_vec[1].get_element_type(), + out_vec[1].get_shape(), + std::vector{6, 1, 5, 1, 4}); + return new_out_vec; + }; + test_case.model_ref.preprocess_inputs_to_main = {{new_constant}, {{1}}}; + test_case.model_ref.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model_ref.preprocess_outputs_of_main = {{new_transpose}, {{0}}}; + test_case.model_ref.model_template = create_model; + + return wrapper(test_case); +}; + +INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonReshapeUnsqueezeForward, + TransposeSinkingTestFixture, + test_forward_reshape_unsqueeze()); // ------------------ BACKWARD -------------------- auto test_backward_unary = []() { @@ -1003,7 +1162,7 @@ auto test_backward_squeeze = []() { TestCase test_case; // Initialize common attributes - test_case.transformation = CREATE_PASS_FACTORY(TSReductionBackward); + test_case.transformation = CREATE_PASS_FACTORY(TSSqueezeBackward); test_case.num_main_ops = {1}; test_case.inputs_to_main = { parameter(element::f32, {32, 1, 2, 1}), @@ -1036,7 +1195,7 @@ auto test_backward_unsqueeze = []() { TestCase test_case; // Initialize common attributes - test_case.transformation = CREATE_PASS_FACTORY(TSReductionBackward); + test_case.transformation = CREATE_PASS_FACTORY(TSUnsqueezeBackward); test_case.num_main_ops = {1}; test_case.inputs_to_main = { parameter(element::f32, {32, 3, 2, 1}), @@ -1066,6 +1225,126 @@ auto test_backward_unsqueeze = []() { INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonUnsqueezeBackward, TransposeSinkingTestFixture, test_backward_unsqueeze()); + +auto test_backward_slice = []() { + TestCase test_case; + + // Initialize common attributes + test_case.transformation = CREATE_PASS_FACTORY(TSSliceBackward); + test_case.num_main_ops = {1}; + test_case.inputs_to_main = { + parameter(element::f32, {6, 4, 5, 3}), + constant(element::i32, {3}, {1, 2, 3}), + constant(element::i32, {3}, {0, 4, 11}), + constant(element::i32, {3}, {1, 2, -1}), + constant(element::i32, {3}, {0, 1, 2}), + }; + + // Test model description: + test_case.model.main_op = {CREATE_SLICE_FACTORY(Slice)}; + test_case.model.preprocess_outputs_of_main = {{set_transpose_for}, {{0}}}; + test_case.model.model_template = create_model; + + // Reference model description: + auto set_specific_gather_for = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector result = out_vec; + for (const auto& idx : idxs) { + const auto& out = out_vec[idx]; + vector transpose_order(out_vec[0].get_shape().size()); + iota(transpose_order.begin(), transpose_order.end(), 0); + reverse(transpose_order.begin(), transpose_order.end()); + auto data = make_shared(element::i32, Shape{transpose_order.size()}, transpose_order); + auto axis = make_shared(element::i32, Shape{}, 0); + auto transpose = make_shared(data, out, axis); + result[idx] = transpose; + } + return result; + }; + test_case.model_ref.preprocess_inputs_to_main = {{set_transpose_for, set_specific_gather_for}, {{0}, {4}}}; + test_case.model_ref.main_op = {CREATE_SLICE_FACTORY(SliceFactory)}; + test_case.model_ref.model_template = create_model; + return wrapper(test_case); +}; + +INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonSliceBackward, TransposeSinkingTestFixture, test_backward_slice()); + +auto test_backward_reshape_squeeze = []() { + TestCase test_case; + + // Initialize common attributes + test_case.transformation = CREATE_PASS_FACTORY(TSSqueezeBackward); + test_case.num_main_ops = {1}; + test_case.inputs_to_main = { + parameter(element::f32, {4, 1, 5, 1, 6}), + constant(element::i32, {3}, {4, 5, 6}), + }; + + // Test model description: + test_case.model.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model.preprocess_outputs_of_main = {{set_transpose_for}, {{0}}}; + test_case.model.model_template = create_model; + + // Reference model description: + auto new_transpose = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector new_out_vec(out_vec.size()); + auto order = make_shared(element::i32, Shape{5}, std::vector{4, 1, 2, 3, 0}); + new_out_vec[0] = make_shared(out_vec[0], order); + new_out_vec[1] = out_vec[1]; + return new_out_vec; + }; + auto new_constant = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector new_out_vec(out_vec.size()); + new_out_vec[0] = out_vec[0]; + new_out_vec[1] = + make_shared(out_vec[1].get_element_type(), out_vec[1].get_shape(), std::vector{6, 5, 4}); + return new_out_vec; + }; + test_case.model_ref.preprocess_inputs_to_main = {{new_transpose, new_constant}, {{0}, {1}}}; + test_case.model_ref.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model_ref.model_template = create_model; + + return wrapper(test_case); +}; + +INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonReshapeSqueezeBackward, + TransposeSinkingTestFixture, + test_backward_reshape_squeeze()); + +auto test_backward_reshape_unsqueeze = []() { + TestCase test_case; + + // Initialize common attributes + test_case.transformation = CREATE_PASS_FACTORY(TSUnsqueezeBackward); + test_case.num_main_ops = {1}; + test_case.inputs_to_main = { + parameter(element::f32, {4, 5, 6}), + constant(element::i32, {5}, {4, 1, 5, 1, 6}), + }; + + // Test model description: + test_case.model.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model.preprocess_outputs_of_main = {{set_transpose_for}, {{0}}}; + test_case.model.model_template = create_model; + + // Reference model description: + auto new_constant = [](const vector& idxs, const OutputVector& out_vec) -> OutputVector { + OutputVector new_out_vec(out_vec.size()); + new_out_vec[0] = out_vec[0]; + new_out_vec[1] = make_shared(out_vec[1].get_element_type(), + out_vec[1].get_shape(), + std::vector{6, 1, 5, 1, 4}); + return new_out_vec; + }; + test_case.model_ref.preprocess_inputs_to_main = {{set_transpose_for, new_constant}, {{0}, {1}}}; + test_case.model_ref.main_op = {CREATE_RESHAPE_FACTORY(Reshape)}; + test_case.model_ref.model_template = create_model; + + return wrapper(test_case); +}; + +INSTANTIATE_TEST_SUITE_P(TransposeSinkingCommonReshapeUnsqueezeBackward, + TransposeSinkingTestFixture, + test_backward_reshape_unsqueeze()); } // namespace common } // namespace testing -} // namespace transpose_sinking \ No newline at end of file +} // namespace transpose_sinking diff --git a/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp index ee077f667340ab..358bab067e4cd8 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_general_test.cpp @@ -412,6 +412,29 @@ TEST_F(TransformationTestsF, TSGeneralTestMultipleTypes) { manager.register_pass(); } +TEST_F(TransformationTestsF, TSGeneralCheckShapeOfConstFoldingDisabled) { + using namespace transpose_sinking::testing::general; + ov::Shape input_shape = {96, 40, 55}; + ov::Shape reshape_shape = {1, 96, 40, 55}; + ov::element::Type input_type = ov::element::f32; + { + auto X = std::make_shared(input_type, input_shape); + auto Shape = std::make_shared(input_type, reshape_shape); + + auto order = std::make_shared(ov::element::u64, ov::Shape{3}, ov::Shape{0, 2, 1}); + auto transpose = std::make_shared(X, order); + + auto shape_of = std::make_shared(Shape); + auto reshape = std::make_shared(transpose, shape_of, false); + + auto ng_order1 = std::make_shared(ov::element::u64, ov::Shape{4}, ov::Shape{0, 3, 1, 2}); + auto transpose1 = std::make_shared(reshape, ng_order1); + + function = std::make_shared(transpose1, ov::ParameterVector{X, Shape}); + } + manager.register_pass(); +} + } // namespace general } // namespace testing } // namespace transpose_sinking \ No newline at end of file diff --git a/src/common/transformations/tests/utils/compare_functions_test.cpp b/src/common/transformations/tests/utils/compare_functions_test.cpp index f27b30f18b933b..c89eddb97a4611 100644 --- a/src/common/transformations/tests/utils/compare_functions_test.cpp +++ b/src/common/transformations/tests/utils/compare_functions_test.cpp @@ -201,7 +201,7 @@ TEST(TransformationTests, CompareFunctoinsTINegative) { const auto fc = FunctionsComparator::with_default().enable(FunctionsComparator::ATTRIBUTES); auto res = fc(f, f_ref); EXPECT_FALSE(res.valid); - EXPECT_THAT(res.message, HasSubstr("LSTMCell/4 != Relu/0")); + EXPECT_THAT(res.message, HasSubstr("LSTMCell/opset4 != Relu/opset1")); } TEST(TransformationTests, CompareFunctoinsTINegativeDifferentElementTypeBetweenSubGraphsInputs) { @@ -514,7 +514,7 @@ class DummyConstant : public ngraph::op::Op { DummyConstant& operator=(const DummyConstant&) = delete; const NodeTypeInfo& get_type_info() const override { - static const NodeTypeInfo type_info{typeid(this).name(), static_cast(0)}; + static const NodeTypeInfo type_info{typeid(this).name(), "0"}; return type_info; } diff --git a/src/common/util/CMakeLists.txt b/src/common/util/CMakeLists.txt index a589c283390761..4b538aab7c7461 100644 --- a/src/common/util/CMakeLists.txt +++ b/src/common/util/CMakeLists.txt @@ -24,6 +24,10 @@ endif() # Create named folders for the sources within the .vcproj # Empty name lists them directly under the .vcproj +set_source_files_properties( + "${CMAKE_CURRENT_SOURCE_DIR}/src/file_util.cpp" + PROPERTIES COMPILE_DEFINITIONS OpenVINO_VERSION="${OpenVINO_VERSION}") + source_group("src" FILES ${LIBRARY_SRC}) source_group("include" FILES ${PUBLIC_HEADERS}) diff --git a/src/common/util/include/openvino/util/file_util.hpp b/src/common/util/include/openvino/util/file_util.hpp index 00d8dbe073cd61..be1f35d779faba 100644 --- a/src/common/util/include/openvino/util/file_util.hpp +++ b/src/common/util/include/openvino/util/file_util.hpp @@ -123,6 +123,15 @@ void create_directory_recursive(const std::string& path); */ bool directory_exists(const std::string& path); +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +/** + * @brief Interface function to check if directory exists for given path + * @param path - path to directory wide-string + * @return true if directory exists, false otherwise + */ +bool directory_exists(const std::wstring& path); +#endif + /** * @brief Returns file size for file * @param[in] path The file name @@ -260,6 +269,14 @@ inline std::basic_string make_plugin_library_name(const std::basic_string& */ FilePath get_plugin_path(const std::string& plugin); +/** + * @brief Find the plugins which are located together with OV library + * @param plugin - Path (absolute or relative) or name of a plugin. Depending on platform, `plugin` is wrapped with + * shared library suffix and prefix to identify library full name + * @return absolute path or file name with extension (to be found in ENV) + */ +FilePath get_compiled_plugin_path(const std::string& plugin); + /** * @brief Format plugin path (canonicalize, complete to absolute or complete to file name) for further * dynamic loading by OS diff --git a/src/common/util/src/file_util.cpp b/src/common/util/src/file_util.cpp index bcbd3fe2f906c7..2ec2f9cfb7f575 100644 --- a/src/common/util/src/file_util.cpp +++ b/src/common/util/src/file_util.cpp @@ -27,6 +27,9 @@ # define get_absolute_path(result, path) _fullpath(result, path.c_str(), MAX_ABS_PATH) /// @brief Windows-specific 'stat' wrapper # define stat _stat +# ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +# define wstat _wstat +# endif /// @brief Windows-specific 'mkdir' wrapper # define makedir(dir) _mkdir(dir) // Copied from linux libc sys/stat.h: @@ -403,6 +406,21 @@ bool ov::util::directory_exists(const std::string& path) { return false; } +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +bool ov::util::directory_exists(const std::wstring& path) { +# ifdef _WIN32 + struct stat sb; + + if (wstat(path.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) { + return true; + } + return false; +# else + return directory_exists(wstring_to_string(path)); +# endif +} +#endif + namespace { template ::library_ext())) + lib_name = ov::util::make_plugin_library_name({}, plugin); + + // For 4th case + auto lib_path = ov::util::to_file_path(ov::util::get_absolute_file_path(lib_name)); + if (ov::util::file_exists(lib_path)) + return lib_path; + return ov::util::to_file_path(lib_name); +} + ov::util::FilePath ov::util::get_plugin_path(const std::string& plugin, const std::string& xml_path, bool as_abs_only) { // Assume `plugin` (from XML "location" record) contains only: // 1. /path/to/libexample.so absolute path diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 5469e02a9ea82f..566a085c9f9003 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -31,10 +31,6 @@ set_property(SOURCE ${MIXED_SRC} $/src $) - -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - ie_add_compiler_flags(/wd4244) -endif() # Create named folders for the sources within the .vcproj # Empty name lists them directly under the .vcproj @@ -80,7 +76,7 @@ ie_faster_build(ngraph_obj UNITY PCH PRIVATE "src/precomp.hpp") -addVersionDefines(src/version.cpp CI_BUILD_NUMBER) +ov_add_version_defines(src/version.cpp ngraph_obj) target_link_libraries(ngraph_obj PRIVATE ngraph::builder ngraph::reference openvino::util openvino::pugixml ov_shape_inference openvino::core::dev) @@ -112,7 +108,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") endif() target_link_options(ngraph_obj ${link_type} "/IGNORE:4217,4286") - ie_add_compiler_flags(/wd4267) endif() # some sources are located in ngraph, while headers are in inference_engine_transformations diff --git a/src/core/dev_api/meta_data.hpp b/src/core/dev_api/meta_data.hpp index 57b262debb4ca3..3f992ef6f1b462 100644 --- a/src/core/dev_api/meta_data.hpp +++ b/src/core/dev_api/meta_data.hpp @@ -25,6 +25,11 @@ class OPENVINO_API Meta { * @return const ov::AnyMap with meta information */ virtual operator const ov::AnyMap&() const = 0; + + /** + * @brief Destructor + */ + virtual ~Meta() = default; }; } // namespace ov diff --git a/src/core/dev_api/validation_util.hpp b/src/core/dev_api/validation_util.hpp new file mode 100644 index 00000000000000..452445a055b8a5 --- /dev/null +++ b/src/core/dev_api/validation_util.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/node.hpp" + +namespace ov { +namespace util { +/// \brief Normalize value to the max if value is negative. +/// +/// \param value Input value to normalize. +/// \param max Value used for normalization +/// +/// \return Value if positive otherwise return value + max +OPENVINO_API int64_t normalize(const int64_t& value, const int64_t& max); + +/// \brief Check if values in vector are unique. +/// +/// \param data Input data to check. +/// +/// \return True if unique otherwise false. +OPENVINO_API bool are_unique(const std::vector& data); + +/// \brief Clip value to minimum if below min, or to maximum of above max. +/// +/// \param value Value to be clipped. +/// \param min Minimum value bound. +/// \param max Maximum value boiund +/// +/// \return Value if between min, max otherwise min or max. +OPENVINO_API int64_t clip(const int64_t& value, const int64_t& min, const int64_t& max); + +/// \brief Constant folds a subgraph to a constant node +/// +/// \param subgraph sink +/// +/// \return Constant node or nullptr if unable to constantfold the subgraph +OPENVINO_API std::shared_ptr constantfold_subgraph(const Output& subgraph_sink); +} // namespace util +} // namespace ov diff --git a/src/core/include/ngraph/node.hpp b/src/core/include/ngraph/node.hpp index 776dcb6bebc653..76090d3cca32b1 100644 --- a/src/core/include/ngraph/node.hpp +++ b/src/core/include/ngraph/node.hpp @@ -150,21 +150,19 @@ using ov::check_new_args_count; } #endif -#define _NGRAPH_RTTI_DEFINITION_WITH_PARENT(CLASS, TYPE_NAME, _VERSION_INDEX, PARENT_CLASS) \ - const ::ngraph::Node::type_info_t& CLASS::get_type_info_static() { \ - static const ::ngraph::Node::type_info_t type_info_static{TYPE_NAME, \ - static_cast(_VERSION_INDEX), \ - &PARENT_CLASS::get_type_info_static()}; \ - return type_info_static; \ - } \ - _NGRAPH_RTTI_DEFINITION_COMMON(CLASS) - -#define _NGRAPH_RTTI_DEFINITION_NO_PARENT(CLASS, TYPE_NAME, _VERSION_INDEX) \ +#define _NGRAPH_RTTI_DEFINITION_WITH_PARENT(CLASS, TYPE_NAME, PARENT_CLASS) \ const ::ngraph::Node::type_info_t& CLASS::get_type_info_static() { \ - static const ::ngraph::Node::type_info_t type_info_static{TYPE_NAME, static_cast(_VERSION_INDEX)}; \ + static const ::ngraph::Node::type_info_t type_info_static{TYPE_NAME, &PARENT_CLASS::get_type_info_static()}; \ return type_info_static; \ } \ _NGRAPH_RTTI_DEFINITION_COMMON(CLASS) + +#define _NGRAPH_RTTI_DEFINITION_NO_PARENT(CLASS, TYPE_NAME) \ + const ::ngraph::Node::type_info_t& CLASS::get_type_info_static() { \ + static const ::ngraph::Node::type_info_t type_info_static{TYPE_NAME}; \ + return type_info_static; \ + } \ + _NGRAPH_RTTI_DEFINITION_COMMON(CLASS) #define NGRAPH_RTTI_DEFINITION(...) \ _OPENVINO_RTTI_EXPAND(_OPENVINO_RTTI_DEFINITION_SELECTOR(__VA_ARGS__, \ _NGRAPH_RTTI_DEFINITION_WITH_PARENT, \ diff --git a/src/core/include/ngraph/validation_util.hpp b/src/core/include/ngraph/validation_util.hpp index 739f923edd526f..4aadf9d210cd21 100644 --- a/src/core/include/ngraph/validation_util.hpp +++ b/src/core/include/ngraph/validation_util.hpp @@ -22,14 +22,17 @@ using ov::infer_convolution_forward; using ov::normalize_axes; using ov::normalize_axis; +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API Strides conv_default_strides(const Node* node, const PartialShape& data_batch_shape, const PartialShape& filters_shape); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API CoordinateDiff conv_default_padding(const Node* node, const PartialShape& data_batch_shape, const PartialShape& filters_shape); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API PartialShape infer_windowed_reduction_output_shape(const Node* node, const PartialShape& data_shape, @@ -42,6 +45,7 @@ PartialShape infer_windowed_reduction_output_shape(const Node* node, bool is_window_all_in_padding_allowed, bool ceil_mode = false); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") void validate_conv_params_spatial_dimensions(const Node* node, const size_t num_spatial_dims, const op::PadType auto_pad, @@ -62,6 +66,7 @@ void validate_conv_params_spatial_dimensions(const Node* node, /// \param pads_end Pads end. /// /// \return Partial shape of the output. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") PartialShape validate_and_infer_convolution_forward_output_shape(const Node* node, const Rank& result_ps_rank, const PartialShape& data_batch_pshape, @@ -72,6 +77,7 @@ PartialShape validate_and_infer_convolution_forward_output_shape(const Node* nod CoordinateDiff& pads_begin, CoordinateDiff& pads_end); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API PartialShape infer_batched_pooling_forward(const Node* node, const PartialShape& data_batch_shape, @@ -83,6 +89,7 @@ PartialShape infer_batched_pooling_forward(const Node* node, bool ceil_mode = false, const Strides& window_dilation = Strides{}); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API std::tuple infer_batch_norm_forward(const Node* node, element::Type input_element_type, @@ -96,6 +103,7 @@ std::tuple infer_batch_norm_forward(c const PartialShape& mean_shape, const PartialShape& variance_shape); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API std::tuple infer_batch_norm_forward(const Node* node, element::Type input_element_type, @@ -119,6 +127,7 @@ std::tuple infer_batch_norm_forward(c /// /// \return true if auto padding was applied successfully (all needed informations such as /// spatial dims are known), false otherwise. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API bool try_apply_auto_padding(const PartialShape& image_shape, const Shape& filter_shape, @@ -128,6 +137,7 @@ bool try_apply_auto_padding(const PartialShape& image_shape, CoordinateDiff& padding_above, CoordinateDiff& padding_below); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API PartialShape infer_slice_shape(const Node* node, const PartialShape& input_shape, @@ -155,22 +165,27 @@ NGRAPH_API std::pair maximum_value(const Output& value); /// \param outputs Root set of values to try to compute /// \param evaluation_context Storage of additional settings and attributes that can be used /// when evaluating the function. This additional information can be shared across nodes. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API void evaluate_nodes(std::map& value_map, std::map& output_tensor_map, const OutputVector& outputs, const EvaluationContext& evaluation_context = EvaluationContext()); /// \brief Returns a Constant storing scalar value equal to std::numeric_limits::max() +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API std::shared_ptr get_constant_max_of_type(element::Type_t t); /// \brief Returns a Constant storing scalar value equal to std::numeric_limits::min() +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API std::shared_ptr get_constant_min_of_type(element::Type_t t); /// \brief Returns a Constant storing scalar value equal to std::numeric_limits::lowest() +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API std::shared_ptr get_constant_lowest_of_type(element::Type_t t); /// \brief Checks if size of HostTensorVector is the same as passed size attribute. Then checks /// that all the HostTensorPtrs are not equal to nullptr +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API bool validate_host_tensor_vector(const HostTensorVector& v, const size_t& size); namespace opset1 { @@ -187,6 +202,7 @@ namespace opset1 { /// \param pads_begin The placeholder for paddings at the beginning of axis. /// \param pads_end The placeholder for paddings at the end of axis. /// +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") NGRAPH_API void infer_conv_backprop_auto_padding(const Shape& input_data_shape, const Shape& filters_shape, diff --git a/src/core/include/openvino/core/any.hpp b/src/core/include/openvino/core/any.hpp index 0e070f3a0e7c71..00df6f531be4d4 100644 --- a/src/core/include/openvino/core/any.hpp +++ b/src/core/include/openvino/core/any.hpp @@ -496,7 +496,7 @@ class OPENVINO_API Any { } protected: - ~Base() = default; + virtual ~Base() = default; }; template diff --git a/src/core/include/openvino/core/descriptor/tensor.hpp b/src/core/include/openvino/core/descriptor/tensor.hpp index 7aa5b225a8cda5..d264d568ab4db5 100644 --- a/src/core/include/openvino/core/descriptor/tensor.hpp +++ b/src/core/include/openvino/core/descriptor/tensor.hpp @@ -70,9 +70,6 @@ class OPENVINO_API Tensor { OPENVINO_DEPRECATED( "set_element_type() is deprecated. To change Tensor element type please change the Parameter type") void set_element_type(const element::Type& elemenet_type); - OPENVINO_DEPRECATED( - "set_partial_shape() is deprecated. To change Tensor partial shape please change the Parameter partial shape") - void set_partial_shape(const PartialShape& partial_shape); /// \brief sets lower bound value description void set_lower_value(const ov::Tensor& value); @@ -148,6 +145,7 @@ class OPENVINO_API Tensor { friend OPENVINO_API std::string get_ov_tensor_legacy_name(const Tensor& tensor); friend OPENVINO_API void set_ov_tensor_legacy_name(Tensor& tensor, const std::string& tensor_name); friend class pass::ReverseShapeAndTypeInfer; + friend class ngraph::runtime::HostTensor; }; OPENVINO_API diff --git a/src/core/include/openvino/core/model.hpp b/src/core/include/openvino/core/model.hpp index 159a2c57533a77..dae178b011a93f 100644 --- a/src/core/include/openvino/core/model.hpp +++ b/src/core/include/openvino/core/model.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "openvino/core/any.hpp" @@ -47,7 +48,7 @@ class OPENVINO_API Model : public std::enable_shared_from_this { public: _OPENVINO_HIDDEN_METHOD static const ::ov::DiscreteTypeInfo& get_type_info_static() { - static const ::ov::DiscreteTypeInfo type_info_static{"Model", static_cast(0)}; + static const ::ov::DiscreteTypeInfo type_info_static{"Model"}; return type_info_static; } const ::ov::DiscreteTypeInfo& get_type_info() const { @@ -554,6 +555,7 @@ class OPENVINO_API Model : public std::enable_shared_from_this { // of weak_ptr not to increase node ref counter to prevent the situation when // node has no consumers but still exists in a graph. mutable std::vector> m_cached_ordered_ops; + mutable std::unordered_set m_cached_ops; mutable std::unordered_map> m_cached_output_names; mutable std::unordered_map> m_cached_op_names; diff --git a/src/core/include/openvino/core/node.hpp b/src/core/include/openvino/core/node.hpp index 549fca57e2c73a..c2bb3fef23404f 100644 --- a/src/core/include/openvino/core/node.hpp +++ b/src/core/include/openvino/core/node.hpp @@ -409,14 +409,6 @@ class OPENVINO_API Node : public std::enable_shared_from_this { /// Get all the nodes that uses the current node NodeVector get_users(bool check_is_used = false) const; - /// \return Version of this node - OPENVINO_DEPRECATED("This method is deprecated and will be removed soon.") - virtual size_t get_version() const { - OPENVINO_SUPPRESS_DEPRECATED_START - return get_type_info().version; - OPENVINO_SUPPRESS_DEPRECATED_END - } - /// Use instance ids for comparison instead of memory addresses to improve determinism bool operator<(const Node& other) const { return m_instance_id < other.m_instance_id; diff --git a/src/core/include/openvino/core/op_extension.hpp b/src/core/include/openvino/core/op_extension.hpp index b81aab87305591..e9a756ea2d32c1 100644 --- a/src/core/include/openvino/core/op_extension.hpp +++ b/src/core/include/openvino/core/op_extension.hpp @@ -60,6 +60,7 @@ namespace detail { static auto collect_attached_extensions_##FRAMEWORK(ov::Any)->void {} OV_COLLECT_ATTACHED_EXTENSIONS(onnx) +OV_COLLECT_ATTACHED_EXTENSIONS(paddle) OV_COLLECT_ATTACHED_EXTENSIONS(tensorflow) } // namespace detail @@ -95,6 +96,7 @@ class OpExtension : public BaseOpExtension { std::vector get_attached_extensions() const override { std::vector res; detail::collect_attached_extensions_onnx(res); + detail::collect_attached_extensions_paddle(res); detail::collect_attached_extensions_tensorflow(res); return res; } diff --git a/src/core/include/openvino/core/rtti.hpp b/src/core/include/openvino/core/rtti.hpp index 505d6e687eb29b..06b541204d88c3 100644 --- a/src/core/include/openvino/core/rtti.hpp +++ b/src/core/include/openvino/core/rtti.hpp @@ -7,14 +7,14 @@ #include "openvino/core/type.hpp" #include "openvino/core/visibility.hpp" -#define _OPENVINO_RTTI_EXPAND(X) X -#define _OPENVINO_RTTI_DEFINITION_SELECTOR(_1, _2, _3, _4, NAME, ...) NAME +#define _OPENVINO_RTTI_EXPAND(X) X +#define _OPENVINO_RTTI_DEFINITION_SELECTOR(_1, _2, _3, NAME, ...) NAME #define _OPENVINO_RTTI_WITH_TYPE(TYPE_NAME) _OPENVINO_RTTI_WITH_TYPE_VERSION(TYPE_NAME, "util") #define _OPENVINO_RTTI_WITH_TYPE_VERSION(TYPE_NAME, VERSION_NAME) \ _OPENVINO_HIDDEN_METHOD static const ::ov::DiscreteTypeInfo& get_type_info_static() { \ - static ::ov::DiscreteTypeInfo type_info_static{TYPE_NAME, 0, VERSION_NAME}; \ + static ::ov::DiscreteTypeInfo type_info_static{TYPE_NAME, VERSION_NAME}; \ type_info_static.hash(); \ return type_info_static; \ } \ @@ -23,19 +23,18 @@ } #define _OPENVINO_RTTI_WITH_TYPE_VERSION_PARENT(TYPE_NAME, VERSION_NAME, PARENT_CLASS) \ - _OPENVINO_RTTI_WITH_TYPE_VERSIONS_PARENT(TYPE_NAME, VERSION_NAME, PARENT_CLASS, 0) + _OPENVINO_RTTI_WITH_TYPE_VERSIONS_PARENT(TYPE_NAME, VERSION_NAME, PARENT_CLASS) -#define _OPENVINO_RTTI_WITH_TYPE_VERSIONS_PARENT(TYPE_NAME, VERSION_NAME, PARENT_CLASS, OLD_VERSION) \ - _OPENVINO_HIDDEN_METHOD static const ::ov::DiscreteTypeInfo& get_type_info_static() { \ - static ::ov::DiscreteTypeInfo type_info_static{TYPE_NAME, \ - OLD_VERSION, \ - VERSION_NAME, \ - &PARENT_CLASS::get_type_info_static()}; \ - type_info_static.hash(); \ - return type_info_static; \ - } \ - const ::ov::DiscreteTypeInfo& get_type_info() const override { \ - return get_type_info_static(); \ +#define _OPENVINO_RTTI_WITH_TYPE_VERSIONS_PARENT(TYPE_NAME, VERSION_NAME, PARENT_CLASS) \ + _OPENVINO_HIDDEN_METHOD static const ::ov::DiscreteTypeInfo& get_type_info_static() { \ + static ::ov::DiscreteTypeInfo type_info_static{TYPE_NAME, \ + VERSION_NAME, \ + &PARENT_CLASS::get_type_info_static()}; \ + type_info_static.hash(); \ + return type_info_static; \ + } \ + const ::ov::DiscreteTypeInfo& get_type_info() const override { \ + return get_type_info_static(); \ } /// Helper macro that puts necessary declarations of RTTI block inside a class definition. @@ -92,11 +91,10 @@ /// OPENVINO_RTTI(name, version_id) /// OPENVINO_RTTI(name, version_id, parent) /// OPENVINO_RTTI(name, version_id, parent, old_version) -#define OPENVINO_RTTI(...) \ - _OPENVINO_RTTI_EXPAND(_OPENVINO_RTTI_DEFINITION_SELECTOR(__VA_ARGS__, \ - _OPENVINO_RTTI_WITH_TYPE_VERSIONS_PARENT, \ - _OPENVINO_RTTI_WITH_TYPE_VERSION_PARENT, \ - _OPENVINO_RTTI_WITH_TYPE_VERSION, \ +#define OPENVINO_RTTI(...) \ + _OPENVINO_RTTI_EXPAND(_OPENVINO_RTTI_DEFINITION_SELECTOR(__VA_ARGS__, \ + _OPENVINO_RTTI_WITH_TYPE_VERSION_PARENT, \ + _OPENVINO_RTTI_WITH_TYPE_VERSION, \ _OPENVINO_RTTI_WITH_TYPE)(__VA_ARGS__)) /// Note: Please don't use this macros for new operations diff --git a/src/core/include/openvino/core/runtime_attribute.hpp b/src/core/include/openvino/core/runtime_attribute.hpp index 4aca1b00330fdb..5502c3eb8ff280 100644 --- a/src/core/include/openvino/core/runtime_attribute.hpp +++ b/src/core/include/openvino/core/runtime_attribute.hpp @@ -20,7 +20,7 @@ class Any; class OPENVINO_API RuntimeAttribute { public: _OPENVINO_HIDDEN_METHOD static const DiscreteTypeInfo& get_type_info_static() { - static const ::ov::DiscreteTypeInfo type_info_static{"RuntimeAttribute", static_cast(0)}; + static const ::ov::DiscreteTypeInfo type_info_static{"RuntimeAttribute"}; return type_info_static; } virtual const DiscreteTypeInfo& get_type_info() const { diff --git a/src/core/include/openvino/core/type.hpp b/src/core/include/openvino/core/type.hpp index 6fa3e3d56f6285..6ceaa39cbe08e0 100644 --- a/src/core/include/openvino/core/type.hpp +++ b/src/core/include/openvino/core/type.hpp @@ -30,14 +30,11 @@ namespace ov { */ struct OPENVINO_API DiscreteTypeInfo { const char* name; - OPENVINO_DEPRECATED("This member was deprecated. Please use version_id instead.") - uint64_t version; const char* version_id; // A pointer to a parent type info; used for casting and inheritance traversal, not for // exact type identification const DiscreteTypeInfo* parent; - OPENVINO_SUPPRESS_DEPRECATED_START DiscreteTypeInfo() = default; DiscreteTypeInfo(const DiscreteTypeInfo&) = default; DiscreteTypeInfo(DiscreteTypeInfo&&) = default; @@ -47,29 +44,16 @@ struct OPENVINO_API DiscreteTypeInfo { const char* _version_id, const DiscreteTypeInfo* _parent = nullptr) : name(_name), - version(0), version_id(_version_id), parent(_parent), hash_value(0) {} - constexpr DiscreteTypeInfo(const char* _name, uint64_t _version, const DiscreteTypeInfo* _parent = nullptr) + constexpr DiscreteTypeInfo(const char* _name, const DiscreteTypeInfo* _parent = nullptr) : name(_name), - version(_version), version_id(nullptr), parent(_parent), hash_value(0) {} - constexpr DiscreteTypeInfo(const char* _name, - uint64_t _version, - const char* _version_id, - const DiscreteTypeInfo* _parent = nullptr) - : name(_name), - version(_version), - version_id(_version_id), - parent(_parent), - hash_value(0) {} - OPENVINO_SUPPRESS_DEPRECATED_END - bool is_castable(const DiscreteTypeInfo& target_type) const; std::string get_version() const; diff --git a/src/core/include/openvino/core/type/element_type.hpp b/src/core/include/openvino/core/type/element_type.hpp index 649589a8c5dff7..eaef94b3e402dc 100644 --- a/src/core/include/openvino/core/type/element_type.hpp +++ b/src/core/include/openvino/core/type/element_type.hpp @@ -61,6 +61,7 @@ class OPENVINO_API Type { Type(const Type&) = default; constexpr Type(const Type_t t) : m_type{t} {} Type(size_t bitwidth, bool is_real, bool is_signed, bool is_quantized, const std::string& cname); + explicit Type(const std::string& type); Type& operator=(const Type&) = default; std::string c_type_string() const; size_t size() const; diff --git a/src/core/include/openvino/core/validation_util.hpp b/src/core/include/openvino/core/validation_util.hpp index 3b88a51a2f5d7a..5039ed342dfeef 100644 --- a/src/core/include/openvino/core/validation_util.hpp +++ b/src/core/include/openvino/core/validation_util.hpp @@ -11,6 +11,7 @@ namespace ov { +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API PartialShape infer_convolution_forward(const Node* node, const PartialShape& data_batch_shape, @@ -21,6 +22,7 @@ PartialShape infer_convolution_forward(const Node* node, const Strides& filter_strides, const Strides& filter_dilation); +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API void infer_auto_padding(const Shape& image_shape, const Shape& filter_shape, @@ -30,15 +32,6 @@ void infer_auto_padding(const Shape& image_shape, CoordinateDiff& padding_above, CoordinateDiff& padding_below); -/// \brief Normalize value to the max if value is negative. -/// -/// \param value Input value to normalize. -/// \param max Value used for normalization -/// -/// \return Value if positive otherwise return value + max -OPENVINO_API -int64_t normalize(const int64_t& value, const int64_t& max); - /// \brief Handle out of range axis. /// /// \param[in] node The node with requested axis. @@ -48,6 +41,7 @@ int64_t normalize(const int64_t& value, const int64_t& max); /// \return Checking if axis is in range [-tensor_rank, tensor_rank-1], otherwise /// returns error. If negative axis, it counts from the last to the first axis, /// by adding tensor_rank to axis. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API int64_t normalize_axis(const Node* node, std::int64_t axis, const Rank& tensor_rank); @@ -60,6 +54,7 @@ int64_t normalize_axis(const Node* node, std::int64_t axis, const Rank& tensor_r /// \return If any negative axis in vector, it counts from the last to the first /// axis, by adding tensor_rank to axis. /// +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API std::vector normalize_axes(const std::string& node_description, const std::vector& axes, @@ -74,6 +69,7 @@ std::vector normalize_axes(const std::string& node_description, /// \return Checking if axis is in range [-tensor_rank, tensor_rank-1], otherwise /// returns error. If negative axis, it counts from the last to the first axis, /// by adding tensor_rank to axis. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API int64_t normalize_axis(const std::string& node_description, std::int64_t axis, const Rank& tensor_rank); @@ -88,6 +84,7 @@ int64_t normalize_axis(const std::string& node_description, std::int64_t axis, c /// \return Checking if axis is in range [axis_range_min, axis_range_max], otherwise /// returns error. If negative axis, it counts from the last to the first axis, /// by adding tensor_rank to axis. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API int64_t normalize_axis(const Node* node, std::int64_t axis, @@ -106,6 +103,7 @@ int64_t normalize_axis(const Node* node, /// \return Checking if axis is in range [axis_range_min, axis_range_max], otherwise /// returns error. If negative axis, it counts from the last to the first axis, /// by adding tensor_rank to axis. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API int64_t normalize_axis(const std::string& node_description, std::int64_t axis, @@ -121,6 +119,7 @@ int64_t normalize_axis(const std::string& node_description, /// \param[in] tensor_rank The corresponding tensor rank. /// \param[in,out] axes The requested vector of axes. /// +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API void normalize_axes(const Node* node, const int64_t& tensor_rank, std::vector& axes); @@ -129,10 +128,12 @@ void normalize_axes(const Node* node, const int64_t& tensor_rank, std::vector& output, PartialShape& pshape); /// \brief Runs an estimation of source tensor. If it succeeded to calculate both bounds and /// they are the same returns Constant operation from the resulting bound, otherwise nullptr. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API std::shared_ptr get_constant_from_source(const Output& source); /// \brief Propagates value label from 0 input to the only output through an operation. @@ -141,6 +142,7 @@ OPENVINO_API std::shared_ptr get_constant_from_source(const Ou /// \param node Operation to be performed /// \param output_labels Vector of TensorLabel objects representing resulting value labels /// \return boolean status if label evaluation was successful. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API bool default_label_evaluator(const Node* node, TensorLabelVector& output_labels); /// \brief Generates transpose default axes order at end of input vector. @@ -149,6 +151,7 @@ OPENVINO_API bool default_label_evaluator(const Node* node, TensorLabelVector& o /// /// \param axes_order Vector where default order will be generated. /// \param length Sequence length of axes order. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API void generate_transpose_default_order(std::vector& axes_order, const size_t length); /// \brief Check if vector of axes order has got valid values. @@ -159,12 +162,14 @@ OPENVINO_API void generate_transpose_default_order(std::vector& axes_or /// \param size Input for transpose rank size. /// /// \return true if axes order is valid otherwise false. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API bool is_valid_axes_order(const std::vector& axes_order, const size_t size); /// \brief Checks label tensor if there is no label /// /// \param labels Label tensor for check. /// \return True if there is no labels, otherwise false. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API bool has_no_labels(const TensorLabel& labels); /// \brief Get the node input partial shapes. @@ -172,6 +177,7 @@ OPENVINO_API bool has_no_labels(const TensorLabel& labels); /// \param node Node to extract input shapes. /// /// \return Vector of PartialShapes of each input. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API std::vector get_node_input_partial_shapes(const ov::Node& node); /// \brief Check if rank is compatible to any of rank from container. @@ -180,28 +186,7 @@ OPENVINO_API std::vector get_node_input_partial_shapes(const ov::N /// \param ranks VEctor of ranks used to check input rank compatibility. /// /// \return True if rank compatible to any from ranks, otherwise false. +OPENVINO_DEPRECATED("This function is deprecated and will be moved to dev api in 2024.0 release.") OPENVINO_API bool is_rank_compatible_any_of(const ov::Rank& rank, const std::vector& ranks); -/// \brief Check if values in vector are unique. -/// -/// \param data Input data to check. -/// -/// \return True if unique otherwise false. -OPENVINO_API bool are_unique(const std::vector& data); - -/// \brief Clip value to minimum if below min, or to maximum of above max. -/// -/// \param value Value to be clipped. -/// \param min Minimum value bound. -/// \param max Maximum value boiund -/// -/// \return Value if between min, max otherwise min or max. -OPENVINO_API int64_t clip(const int64_t& value, const int64_t& min, const int64_t& max); - -/// \brief Constant folds a subgraph to a constant node -/// -/// \param subgraph sink -/// -/// \return Constant node or nullptr if unable to constantfold the subgraph -OPENVINO_API std::shared_ptr constantfold_subgraph(const Output& subgraph_sink); } // namespace ov diff --git a/src/core/include/openvino/op/acosh.hpp b/src/core/include/openvino/op/acosh.hpp index 97224708df1f50..59a2dd45e4b06c 100644 --- a/src/core/include/openvino/op/acosh.hpp +++ b/src/core/include/openvino/op/acosh.hpp @@ -16,7 +16,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Acosh : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("Acosh", "opset4", util::UnaryElementwiseArithmetic, 3); + OPENVINO_OP("Acosh", "opset4", util::UnaryElementwiseArithmetic); /// \brief Constructs an Acosh operation. Acosh() = default; diff --git a/src/core/include/openvino/op/add.hpp b/src/core/include/openvino/op/add.hpp index 0e2a812a9dc938..054ec3302d846c 100644 --- a/src/core/include/openvino/op/add.hpp +++ b/src/core/include/openvino/op/add.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Add : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Add", "opset1", util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Add", "opset1", util::BinaryElementwiseArithmetic); /// \brief Constructs an uninitialized addition operation Add() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/asinh.hpp b/src/core/include/openvino/op/asinh.hpp index 8fa8eedd687dc4..96a7f02bfc7e12 100644 --- a/src/core/include/openvino/op/asinh.hpp +++ b/src/core/include/openvino/op/asinh.hpp @@ -15,7 +15,7 @@ namespace v3 { /// class OPENVINO_API Asinh : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("Asinh", "opset4", util::UnaryElementwiseArithmetic, 3); + OPENVINO_OP("Asinh", "opset4", util::UnaryElementwiseArithmetic); /// \brief Constructs an Asinh operation. Asinh() = default; diff --git a/src/core/include/openvino/op/assign.hpp b/src/core/include/openvino/op/assign.hpp index 7c5f2b477a573e..2ba16d46195ffc 100644 --- a/src/core/include/openvino/op/assign.hpp +++ b/src/core/include/openvino/op/assign.hpp @@ -14,7 +14,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Assign : public util::AssignBase { public: - OPENVINO_OP("Assign", "opset3", util::AssignBase, 3); + OPENVINO_OP("Assign", "opset3", util::AssignBase); Assign() = default; /// \brief Constructs an Assign operation. @@ -44,7 +44,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Assign : public util::AssignBase { public: - OPENVINO_OP("Assign", "opset6", util::AssignBase, 6); + OPENVINO_OP("Assign", "opset6", util::AssignBase); Assign() = default; /// \brief Constructs an Assign operation. diff --git a/src/core/include/openvino/op/atanh.hpp b/src/core/include/openvino/op/atanh.hpp index 5fc62facb560af..4d4220b6fdcaa5 100644 --- a/src/core/include/openvino/op/atanh.hpp +++ b/src/core/include/openvino/op/atanh.hpp @@ -16,7 +16,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Atanh : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("Atanh", "opset4", util::UnaryElementwiseArithmetic, 3); + OPENVINO_OP("Atanh", "opset4", util::UnaryElementwiseArithmetic); /// \brief Constructs an Atanh operation. Atanh() = default; diff --git a/src/core/include/openvino/op/avg_pool.hpp b/src/core/include/openvino/op/avg_pool.hpp index a69e6cc8502da0..24298c8602bbd2 100644 --- a/src/core/include/openvino/op/avg_pool.hpp +++ b/src/core/include/openvino/op/avg_pool.hpp @@ -14,7 +14,7 @@ namespace v1 { /// class OPENVINO_API AvgPool : public Op { public: - OPENVINO_OP("AvgPool", "opset1", op::Op, 1); + OPENVINO_OP("AvgPool", "opset1", op::Op); /// \brief Constructs a batched average pooling operation. AvgPool() = default; diff --git a/src/core/include/openvino/op/batch_norm.hpp b/src/core/include/openvino/op/batch_norm.hpp index 7131f0078d665d..3800090312cb7f 100644 --- a/src/core/include/openvino/op/batch_norm.hpp +++ b/src/core/include/openvino/op/batch_norm.hpp @@ -59,7 +59,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API BatchNormInference : public Op { public: - OPENVINO_OP("BatchNormInference", "opset5", op::Op, 5); + OPENVINO_OP("BatchNormInference", "opset5", op::Op); BatchNormInference() = default; /// \param input [., C, ...] /// \param gamma gamma scaling for normalized value. [C] diff --git a/src/core/include/openvino/op/batch_to_space.hpp b/src/core/include/openvino/op/batch_to_space.hpp index 4d28c16d067229..6609e539087628 100644 --- a/src/core/include/openvino/op/batch_to_space.hpp +++ b/src/core/include/openvino/op/batch_to_space.hpp @@ -23,7 +23,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API BatchToSpace : public Op { public: - OPENVINO_OP("BatchToSpace", "opset2", op::Op, 1); + OPENVINO_OP("BatchToSpace", "opset2", op::Op); BatchToSpace() = default; /// \brief Constructs a BatchToSpace operation. /// diff --git a/src/core/include/openvino/op/binary_convolution.hpp b/src/core/include/openvino/op/binary_convolution.hpp index 0dbb2d494bd6eb..c327df0181297f 100644 --- a/src/core/include/openvino/op/binary_convolution.hpp +++ b/src/core/include/openvino/op/binary_convolution.hpp @@ -7,6 +7,7 @@ #include "openvino/core/coordinate_diff.hpp" #include "openvino/op/op.hpp" #include "openvino/op/util/attr_types.hpp" +#include "openvino/op/util/convolution_base.hpp" namespace ov { namespace op { @@ -14,9 +15,9 @@ namespace v1 { /// \brief BinaryConvolution operation. /// /// \ingroup ov_ops_cpp_api -class OPENVINO_API BinaryConvolution : public Op { +class OPENVINO_API BinaryConvolution : public util::ConvolutionFwdPropBase { public: - OPENVINO_OP("BinaryConvolution", "opset1", op::Op, 1); + OPENVINO_OP("BinaryConvolution", "opset1", op::util::ConvolutionFwdPropBase); enum class BinaryConvolutionMode { // Interpret input data and kernel values: 0 as -1, 1 as 1 @@ -63,46 +64,6 @@ class OPENVINO_API BinaryConvolution : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - /// \return The strides. - const Strides& get_strides() const { - return m_strides; - } - void set_strides(const Strides& strides) { - m_strides = strides; - } - /// \return The dilations. - const Strides& get_dilations() const { - return m_dilations; - } - void set_dilations(const Strides& dilations) { - m_dilations = dilations; - } - /// \return The padding-below sizes (possibly negative). - const CoordinateDiff& get_pads_begin() const { - return m_pads_begin; - } - void set_pads_begin(const CoordinateDiff& pads_begin) { - m_pads_begin = pads_begin; - } - /// \return The padding-above sizes (possibly negative). - const CoordinateDiff& get_pads_end() const { - return m_pads_end; - } - OPENVINO_DEPRECATED("This method is deprecated and will be removed soon. Please use set_pads_end instead.") - void set_adding_above(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - - void set_pads_end(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - /// \return The pad type for convolution. - const PadType& get_auto_pad() const { - return m_auto_pad; - } - void set_auto_pad(const PadType& auto_pad) { - m_auto_pad = auto_pad; - } /// \return The mode of convolution. const BinaryConvolutionMode& get_mode() const { return m_mode; @@ -120,13 +81,9 @@ class OPENVINO_API BinaryConvolution : public Op { protected: BinaryConvolutionMode mode_from_string(const std::string& mode) const; - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; + BinaryConvolutionMode m_mode; float m_pad_value; - PadType m_auto_pad; }; } // namespace v1 } // namespace op diff --git a/src/core/include/openvino/op/broadcast.hpp b/src/core/include/openvino/op/broadcast.hpp index e41c7e6c601193..fccffc5ed1d22e 100644 --- a/src/core/include/openvino/op/broadcast.hpp +++ b/src/core/include/openvino/op/broadcast.hpp @@ -17,7 +17,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Broadcast : public util::BroadcastBase { public: - OPENVINO_OP("Broadcast", "opset3", op::util::BroadcastBase, 3); + OPENVINO_OP("Broadcast", "opset3", op::util::BroadcastBase); /// \brief Constructs a broadcast operation. Broadcast() = default; @@ -81,7 +81,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Broadcast : public util::BroadcastBase { public: - OPENVINO_OP("Broadcast", "opset1", op::util::BroadcastBase, 1); + OPENVINO_OP("Broadcast", "opset1", op::util::BroadcastBase); /// \brief Constructs a broadcast operation. Broadcast() = default; diff --git a/src/core/include/openvino/op/bucketize.hpp b/src/core/include/openvino/op/bucketize.hpp index fa8d34579e7273..33bbfbc8003862 100644 --- a/src/core/include/openvino/op/bucketize.hpp +++ b/src/core/include/openvino/op/bucketize.hpp @@ -13,7 +13,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Bucketize : public Op { public: - OPENVINO_OP("Bucketize", "opset3", op::Op, 3); + OPENVINO_OP("Bucketize", "opset3", op::Op); Bucketize() = default; /// \brief Constructs a Bucketize node diff --git a/src/core/include/openvino/op/convert_like.hpp b/src/core/include/openvino/op/convert_like.hpp index 27bd89c6e9f106..2621a1ce860131 100644 --- a/src/core/include/openvino/op/convert_like.hpp +++ b/src/core/include/openvino/op/convert_like.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ConvertLike : public Op { public: - OPENVINO_OP("ConvertLike", "opset1", op::Op, 1); + OPENVINO_OP("ConvertLike", "opset1", op::Op); /// \brief Constructs a conversion operation. ConvertLike() = default; diff --git a/src/core/include/openvino/op/convolution.hpp b/src/core/include/openvino/op/convolution.hpp index 87c17f8167c41d..de40ae07605ea9 100644 --- a/src/core/include/openvino/op/convolution.hpp +++ b/src/core/include/openvino/op/convolution.hpp @@ -7,6 +7,8 @@ #include "openvino/core/coordinate_diff.hpp" #include "openvino/op/op.hpp" #include "openvino/op/util/attr_types.hpp" +#include "openvino/op/util/convolution_backprop_base.hpp" +#include "openvino/op/util/convolution_base.hpp" namespace ov { namespace op { @@ -14,9 +16,9 @@ namespace v1 { /// \brief Batched convolution operation, with optional window dilation and stride. /// /// \ingroup ov_ops_cpp_api -class OPENVINO_API Convolution : public Op { +class OPENVINO_API Convolution : public util::ConvolutionFwdPropBase { public: - OPENVINO_OP("Convolution", "opset1", op::Op, 1); + OPENVINO_OP("Convolution", "opset1", op::util::ConvolutionFwdPropBase); /// \brief Constructs a batched convolution operation. Convolution() = default; @@ -51,85 +53,13 @@ class OPENVINO_API Convolution : public Op { bool visit_attributes(AttributeVisitor& visitor) override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - - /// \return The strides. - const Strides& get_strides() const { - return m_strides; - } - void set_strides(const Strides& strides) { - m_strides = strides; - } - /// \return The dilations. - const Strides& get_dilations() const { - return m_dilations; - } - void set_dilations(const Strides& dilations) { - m_dilations = dilations; - } - /// \return The padding-below sizes (possibly negative). - const CoordinateDiff& get_pads_begin() const { - return m_pads_begin; - } - void set_pads_begin(const CoordinateDiff& pads_begin) { - m_pads_begin = pads_begin; - } - /// \return The padding-above sizes (possibly negative). - const CoordinateDiff& get_pads_end() const { - return m_pads_end; - } - void set_pads_end(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - OPENVINO_DEPRECATED("This method is deprecated and will be removed soon. Please use set_pads_end instead.") - void set_adding_above(const CoordinateDiff& pads_end) { - set_pads_end(pads_end); - } - /// \return The pad type for convolution. - const PadType& get_auto_pad() const { - return m_auto_pad; - } - void set_auto_pad(const PadType& auto_pad) { - m_auto_pad = auto_pad; - } - -protected: - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; - PadType m_auto_pad; - int64_t m_num_spatial = -1; - -private: - template - friend int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void update_and_validate_attributes(ConvType* op, int64_t num_spatial); - template - friend bool resolve_auto_pad_for_shape(const ConvType* op, - CoordinateDiff& pads_begin, - CoordinateDiff& pads_end, - const std::vector& input_shapes, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - template - friend void shape_infer(const Convolution* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const std::vector& input_shapes, - std::vector& output_shapes); }; /// \brief Data batch backprop for batched convolution operation. /// \ingroup ov_ops_cpp_api -class OPENVINO_API ConvolutionBackpropData : public Op { +class OPENVINO_API ConvolutionBackpropData : public util::ConvolutionBackPropBase { public: - OPENVINO_OP("ConvolutionBackpropData", "opset1", op::Op, 1); + OPENVINO_OP("ConvolutionBackpropData", "opset1", op::util::ConvolutionBackPropBase); /// \brief Constructs a batched-convolution data batch-backprop operation. ConvolutionBackpropData() = default; @@ -197,48 +127,7 @@ class OPENVINO_API ConvolutionBackpropData : public Op { /// \return The output spatial dimensions shape. const PartialShape get_output_shape() const; void set_output_shape(const Shape& output_shape); - /// \return The strides from the forward prop. - const Strides& get_strides() const { - return m_strides; - } - void set_strides(const Strides& strides) { - m_strides = strides; - } - /// \return The dilations from the forward prop. - const Strides& get_dilations() const { - return m_dilations; - } - void set_dilations(const Strides& dilations) { - m_dilations = dilations; - } - /// \return The padding-below sizes (possibly negative) from the forward prop. - const CoordinateDiff& get_pads_begin() const { - return m_pads_begin; - } - void set_pads_begin(const CoordinateDiff& pads_begin) { - m_pads_begin = pads_begin; - } - /// \return The padding-above sizes (possibly negative) from the forward prop. - const CoordinateDiff& get_pads_end() const { - return m_pads_end; - } - void set_pads_end(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - /// \return The auto pad. - const PadType& get_auto_pad() const { - return m_auto_pad; - } - void set_auto_pad(const PadType& auto_pad) { - m_auto_pad = auto_pad; - } - /// \return The output padding. - const CoordinateDiff& get_output_padding() const { - return m_output_padding; - } - void set_output_padding(const CoordinateDiff& output_padding) { - m_output_padding = output_padding; - } + /// \brief Calculates output spatial features size. /// /// \param[in] input_data_shape The input data partial shape @@ -251,6 +140,7 @@ class OPENVINO_API ConvolutionBackpropData : public Op { /// \param output_spatial_shape The placeholder for computed output spatial partial /// shape. /// + OPENVINO_DEPRECATED("This member function is deprecated and will be removed soon.") void infer_conv_backprop_output_spatial_shape(const std::vector& input_data_shape, const std::vector& filters_shape, const Strides& strides, @@ -259,54 +149,6 @@ class OPENVINO_API ConvolutionBackpropData : public Op { const CoordinateDiff& pads_end, const CoordinateDiff& output_padding, std::vector& output_spatial_shape); - -protected: - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; - PadType m_auto_pad; - CoordinateDiff m_output_padding; - - int64_t m_num_spatial = -1; - -private: - template - friend int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const PartialShape& output_shapes_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void update_and_validate_attributes(ConvType* op, int64_t num_spatial); - template - friend void update_and_validate_attributes_back_prop(ConvType* op, int64_t num_spatial); - - template - friend bool resolve_auto_pad_for_shape_back_prop(const ConvType* op, - CoordinateDiff& pads_begin, - CoordinateDiff& pads_end, - const std::vector& input_shapes, - ShapeType& output_spatial_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void shape_infer(const ConvolutionBackpropData* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const T& output_shape_from_input, - const std::vector& input_shapes, - std::vector& output_shapes); }; } // namespace v1 } // namespace op diff --git a/src/core/include/openvino/op/ctc_greedy_decoder_seq_len.hpp b/src/core/include/openvino/op/ctc_greedy_decoder_seq_len.hpp index 4846f750ceaf6b..e07493a781128c 100644 --- a/src/core/include/openvino/op/ctc_greedy_decoder_seq_len.hpp +++ b/src/core/include/openvino/op/ctc_greedy_decoder_seq_len.hpp @@ -14,7 +14,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API CTCGreedyDecoderSeqLen : public Op { public: - OPENVINO_OP("CTCGreedyDecoderSeqLen", "opset6", op::Op, 6); + OPENVINO_OP("CTCGreedyDecoderSeqLen", "opset6", op::Op); CTCGreedyDecoderSeqLen() = default; /// \brief Constructs a CTCGreedyDecoderSeqLen operation /// diff --git a/src/core/include/openvino/op/ctc_loss.hpp b/src/core/include/openvino/op/ctc_loss.hpp index 21f143608fc90f..3a1077d9394e22 100644 --- a/src/core/include/openvino/op/ctc_loss.hpp +++ b/src/core/include/openvino/op/ctc_loss.hpp @@ -14,7 +14,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API CTCLoss : public Op { public: - OPENVINO_OP("CTCLoss", "opset4", op::Op, 4); + OPENVINO_OP("CTCLoss", "opset4", op::Op); CTCLoss() = default; /// \brief Constructs a CTCLoss operation diff --git a/src/core/include/openvino/op/deformable_convolution.hpp b/src/core/include/openvino/op/deformable_convolution.hpp index 3eb673cd340ad6..5d2573ec4bb988 100644 --- a/src/core/include/openvino/op/deformable_convolution.hpp +++ b/src/core/include/openvino/op/deformable_convolution.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API DeformableConvolution : public op::util::DeformableConvolutionBase { public: - OPENVINO_OP("DeformableConvolution", "opset1", op::util::DeformableConvolutionBase, 1); + OPENVINO_OP("DeformableConvolution", "opset1", op::util::DeformableConvolutionBase); /// \brief Constructs a conversion operation. DeformableConvolution() = default; @@ -53,6 +53,7 @@ class OPENVINO_API DeformableConvolution : public op::util::DeformableConvolutio const int64_t deformable_group = 1); std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + void validate_and_infer_types() override; }; } // namespace v1 diff --git a/src/core/include/openvino/op/deformable_psroi_pooling.hpp b/src/core/include/openvino/op/deformable_psroi_pooling.hpp index 60bc07f3dd2861..3e3315a95b93de 100644 --- a/src/core/include/openvino/op/deformable_psroi_pooling.hpp +++ b/src/core/include/openvino/op/deformable_psroi_pooling.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API DeformablePSROIPooling : public Op { public: - OPENVINO_OP("DeformablePSROIPooling", "opset1", op::Op, 1); + OPENVINO_OP("DeformablePSROIPooling", "opset1", op::Op); DeformablePSROIPooling() = default; /// \brief Constructs a DeformablePSROIPooling operation diff --git a/src/core/include/openvino/op/dft.hpp b/src/core/include/openvino/op/dft.hpp index 6072f711650b9c..b87262fd78b919 100644 --- a/src/core/include/openvino/op/dft.hpp +++ b/src/core/include/openvino/op/dft.hpp @@ -29,7 +29,7 @@ namespace v7 { /// \ingroup ov_ops_cpp_api class OPENVINO_API DFT : public util::FFTBase { public: - OPENVINO_OP("DFT", "opset7", util::FFTBase, 7); + OPENVINO_OP("DFT", "opset7", util::FFTBase); DFT() = default; /// \brief Constructs a DFT operation. DFT is performed for full size axes. diff --git a/src/core/include/openvino/op/divide.hpp b/src/core/include/openvino/op/divide.hpp index c4ef648683c293..4d83d0043f4a64 100644 --- a/src/core/include/openvino/op/divide.hpp +++ b/src/core/include/openvino/op/divide.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Divide : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Divide", "opset1", util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Divide", "opset1", util::BinaryElementwiseArithmetic); /// \brief Constructs a division operation. Divide() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/einsum.hpp b/src/core/include/openvino/op/einsum.hpp index e42cdc0a94f806..9d1c179c2c5503 100644 --- a/src/core/include/openvino/op/einsum.hpp +++ b/src/core/include/openvino/op/einsum.hpp @@ -13,7 +13,7 @@ namespace v7 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Einsum : public Op { public: - OPENVINO_OP("Einsum", "opset7", op::Op, 7); + OPENVINO_OP("Einsum", "opset7", op::Op); Einsum() = default; diff --git a/src/core/include/openvino/op/embedding_segments_sum.hpp b/src/core/include/openvino/op/embedding_segments_sum.hpp index 8601bd9ac4ec12..55108f6ccb9d5c 100644 --- a/src/core/include/openvino/op/embedding_segments_sum.hpp +++ b/src/core/include/openvino/op/embedding_segments_sum.hpp @@ -14,7 +14,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API EmbeddingSegmentsSum : public Op { public: - OPENVINO_OP("EmbeddingSegmentsSum", "opset3", op::Op, 3); + OPENVINO_OP("EmbeddingSegmentsSum", "opset3", op::Op); /// \brief Constructs a EmbeddingSegmentsSum operation. EmbeddingSegmentsSum() = default; /// \brief Constructs a EmbeddingSegmentsSum operation. diff --git a/src/core/include/openvino/op/embeddingbag_offsets_sum.hpp b/src/core/include/openvino/op/embeddingbag_offsets_sum.hpp index 0d88bdbd21af16..7c3ad7a7b74e7e 100644 --- a/src/core/include/openvino/op/embeddingbag_offsets_sum.hpp +++ b/src/core/include/openvino/op/embeddingbag_offsets_sum.hpp @@ -15,7 +15,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API EmbeddingBagOffsetsSum : public util::EmbeddingBagOffsetsBase { public: - OPENVINO_OP("EmbeddingBagOffsetsSum", "opset3", util::EmbeddingBagOffsetsBase, 3); + OPENVINO_OP("EmbeddingBagOffsetsSum", "opset3", util::EmbeddingBagOffsetsBase); /// \brief Constructs a EmbeddingBagOffsetsSum operation. EmbeddingBagOffsetsSum() = default; /// \brief Constructs a EmbeddingBagOffsetsSum operation. diff --git a/src/core/include/openvino/op/embeddingbag_packedsum.hpp b/src/core/include/openvino/op/embeddingbag_packedsum.hpp index 169b44484cbeab..b095b226a9e86d 100644 --- a/src/core/include/openvino/op/embeddingbag_packedsum.hpp +++ b/src/core/include/openvino/op/embeddingbag_packedsum.hpp @@ -15,7 +15,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API EmbeddingBagPackedSum : public util::EmbeddingBagPackedBase { public: - OPENVINO_OP("EmbeddingBagPackedSum", "opset3", util::EmbeddingBagPackedBase, 3); + OPENVINO_OP("EmbeddingBagPackedSum", "opset3", util::EmbeddingBagPackedBase); /// \brief Constructs a EmbeddingBagPackedSum operation. EmbeddingBagPackedSum() = default; /// \brief Constructs a EmbeddingBagPackedSum operation. diff --git a/src/core/include/openvino/op/equal.hpp b/src/core/include/openvino/op/equal.hpp index c8e2af9cd575b5..ae16f6c6d52703 100644 --- a/src/core/include/openvino/op/equal.hpp +++ b/src/core/include/openvino/op/equal.hpp @@ -29,7 +29,7 @@ namespace v1 { // clang-format on class OPENVINO_API Equal : public util::BinaryElementwiseComparison { public: - OPENVINO_OP("Equal", "opset1", op::util::BinaryElementwiseComparison, 1); + OPENVINO_OP("Equal", "opset1", op::util::BinaryElementwiseComparison); /// \brief Constructs an equal operation. Equal() : util::BinaryElementwiseComparison(AutoBroadcastType::NUMPY) {} /// \brief Constructs an equal operation. diff --git a/src/core/include/openvino/op/experimental_detectron_detection_output.hpp b/src/core/include/openvino/op/experimental_detectron_detection_output.hpp index c8e84bd6f09622..17221d907cb27e 100644 --- a/src/core/include/openvino/op/experimental_detectron_detection_output.hpp +++ b/src/core/include/openvino/op/experimental_detectron_detection_output.hpp @@ -20,7 +20,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ExperimentalDetectronDetectionOutput : public Op { public: - OPENVINO_OP("ExperimentalDetectronDetectionOutput", "opset6", op::Op, 6); + OPENVINO_OP("ExperimentalDetectronDetectionOutput", "opset6", op::Op); /// \brief Structure that specifies attributes of the operation struct Attributes { diff --git a/src/core/include/openvino/op/experimental_detectron_generate_proposals.hpp b/src/core/include/openvino/op/experimental_detectron_generate_proposals.hpp index 83bf6f769c73e1..af2bfe1511f9fe 100644 --- a/src/core/include/openvino/op/experimental_detectron_generate_proposals.hpp +++ b/src/core/include/openvino/op/experimental_detectron_generate_proposals.hpp @@ -19,7 +19,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ExperimentalDetectronGenerateProposalsSingleImage : public Op { public: - OPENVINO_OP("ExperimentalDetectronGenerateProposalsSingleImage", "opset6", op::Op, 6); + OPENVINO_OP("ExperimentalDetectronGenerateProposalsSingleImage", "opset6", op::Op); /// \brief Structure that specifies attributes of the operation struct Attributes { diff --git a/src/core/include/openvino/op/experimental_detectron_prior_grid_generator.hpp b/src/core/include/openvino/op/experimental_detectron_prior_grid_generator.hpp index 0dd697aec211a2..0865654a3dc358 100644 --- a/src/core/include/openvino/op/experimental_detectron_prior_grid_generator.hpp +++ b/src/core/include/openvino/op/experimental_detectron_prior_grid_generator.hpp @@ -19,7 +19,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ExperimentalDetectronPriorGridGenerator : public Op { public: - OPENVINO_OP("ExperimentalDetectronPriorGridGenerator", "opset6", op::Op, 6); + OPENVINO_OP("ExperimentalDetectronPriorGridGenerator", "opset6", op::Op); /// \brief Structure that specifies attributes of the operation struct Attributes { diff --git a/src/core/include/openvino/op/experimental_detectron_roi_feature.hpp b/src/core/include/openvino/op/experimental_detectron_roi_feature.hpp index 109115d08abec9..b6b3d73b47ddbe 100644 --- a/src/core/include/openvino/op/experimental_detectron_roi_feature.hpp +++ b/src/core/include/openvino/op/experimental_detectron_roi_feature.hpp @@ -20,7 +20,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ExperimentalDetectronROIFeatureExtractor : public Op { public: - OPENVINO_OP("ExperimentalDetectronROIFeatureExtractor", "opset6", op::Op, 6); + OPENVINO_OP("ExperimentalDetectronROIFeatureExtractor", "opset6", op::Op); /// \brief Structure that specifies attributes of the operation struct Attributes { diff --git a/src/core/include/openvino/op/experimental_detectron_topkrois.hpp b/src/core/include/openvino/op/experimental_detectron_topkrois.hpp index c12e1dcb374fc2..90bb99ebc5c89c 100644 --- a/src/core/include/openvino/op/experimental_detectron_topkrois.hpp +++ b/src/core/include/openvino/op/experimental_detectron_topkrois.hpp @@ -19,7 +19,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ExperimentalDetectronTopKROIs : public Op { public: - OPENVINO_OP("ExperimentalDetectronTopKROIs", "opset6", op::Op, 6); + OPENVINO_OP("ExperimentalDetectronTopKROIs", "opset6", op::Op); ExperimentalDetectronTopKROIs() = default; /// \brief Constructs a ExperimentalDetectronTopKROIs operation. diff --git a/src/core/include/openvino/op/extractimagepatches.hpp b/src/core/include/openvino/op/extractimagepatches.hpp index d9a0ca6ff6b89d..37d506fe1895b8 100644 --- a/src/core/include/openvino/op/extractimagepatches.hpp +++ b/src/core/include/openvino/op/extractimagepatches.hpp @@ -14,7 +14,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ExtractImagePatches : public Op { public: - OPENVINO_OP("ExtractImagePatches", "opset3", op::Op, 3); + OPENVINO_OP("ExtractImagePatches", "opset3", op::Op); ExtractImagePatches() = default; /// \brief Constructs a ExtractImagePatches operation diff --git a/src/core/include/openvino/op/floor_mod.hpp b/src/core/include/openvino/op/floor_mod.hpp index 2216de28b617e4..4df54f3bcd7334 100644 --- a/src/core/include/openvino/op/floor_mod.hpp +++ b/src/core/include/openvino/op/floor_mod.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API FloorMod : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("FloorMod", "opset1", op::util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("FloorMod", "opset1", op::util::BinaryElementwiseArithmetic); /// \brief Constructs an uninitialized addition operation FloorMod() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/gather.hpp b/src/core/include/openvino/op/gather.hpp index 55564080af1651..e752259271cfb4 100644 --- a/src/core/include/openvino/op/gather.hpp +++ b/src/core/include/openvino/op/gather.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Gather : public op::util::GatherBase { public: - OPENVINO_OP("Gather", "opset1", op::util::GatherBase, 1); + OPENVINO_OP("Gather", "opset1", op::util::GatherBase); static constexpr int64_t AXIS_NOT_SET_VALUE = std::numeric_limits::max(); Gather() = default; /// \param data The tensor from which slices are gathered @@ -33,7 +33,7 @@ namespace v7 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Gather : public op::util::GatherBase { public: - OPENVINO_OP("Gather", "opset7", op::util::GatherBase, 7); + OPENVINO_OP("Gather", "opset7", op::util::GatherBase); Gather() = default; /// \param data The tensor from which slices are gathered diff --git a/src/core/include/openvino/op/gather_elements.hpp b/src/core/include/openvino/op/gather_elements.hpp index c184d86e3628b4..4d8c419e616a10 100644 --- a/src/core/include/openvino/op/gather_elements.hpp +++ b/src/core/include/openvino/op/gather_elements.hpp @@ -14,7 +14,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API GatherElements : public Op { public: - OPENVINO_OP("GatherElements", "opset6", op::Op, 6); + OPENVINO_OP("GatherElements", "opset6", op::Op); GatherElements() = default; /// \brief Constructs a GatherElements operation. diff --git a/src/core/include/openvino/op/gather_nd.hpp b/src/core/include/openvino/op/gather_nd.hpp index 146a1511664968..59680a37d4e4c6 100644 --- a/src/core/include/openvino/op/gather_nd.hpp +++ b/src/core/include/openvino/op/gather_nd.hpp @@ -13,7 +13,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API GatherND : public op::util::GatherNDBase { public: - OPENVINO_OP("GatherND", "opset5", op::util::GatherNDBase, 5); + OPENVINO_OP("GatherND", "opset5", op::util::GatherNDBase); GatherND() = default; /// \brief Constructs a GatherND operation. diff --git a/src/core/include/openvino/op/gather_tree.hpp b/src/core/include/openvino/op/gather_tree.hpp index 67f455be8a2aa6..bc2169019a8cea 100644 --- a/src/core/include/openvino/op/gather_tree.hpp +++ b/src/core/include/openvino/op/gather_tree.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API GatherTree : public Op { public: - OPENVINO_OP("GatherTree", "opset1", op::Op, 1); + OPENVINO_OP("GatherTree", "opset1", op::Op); GatherTree() = default; /// \param step_ids Tensor of shape [MAX_TIME, BATCH_SIZE, BEAM_WIDTH] with diff --git a/src/core/include/openvino/op/gelu.hpp b/src/core/include/openvino/op/gelu.hpp index 021a7e0ef8bd02..ae868e3909bbfd 100644 --- a/src/core/include/openvino/op/gelu.hpp +++ b/src/core/include/openvino/op/gelu.hpp @@ -15,7 +15,7 @@ namespace v0 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Gelu : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("Gelu", "opset2", util::UnaryElementwiseArithmetic, 0); + OPENVINO_OP("Gelu", "opset2", util::UnaryElementwiseArithmetic); Gelu(); /// \brief Constructs a Gelu operation. @@ -43,7 +43,7 @@ namespace v7 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Gelu : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("Gelu", "opset7", util::UnaryElementwiseArithmetic, 7); + OPENVINO_OP("Gelu", "opset7", util::UnaryElementwiseArithmetic); Gelu() = default; /// \brief Constructs a Gelu operation. diff --git a/src/core/include/openvino/op/greater.hpp b/src/core/include/openvino/op/greater.hpp index 1693c09579b1e7..de889a0acae370 100644 --- a/src/core/include/openvino/op/greater.hpp +++ b/src/core/include/openvino/op/greater.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Greater : public util::BinaryElementwiseComparison { public: - OPENVINO_OP("Greater", "opset1", op::util::BinaryElementwiseComparison, 1); + OPENVINO_OP("Greater", "opset1", op::util::BinaryElementwiseComparison); /// \brief Constructs a greater-than operation. Greater() : util::BinaryElementwiseComparison(AutoBroadcastType::NUMPY) {} /// \brief Constructs a greater-than operation. diff --git a/src/core/include/openvino/op/greater_eq.hpp b/src/core/include/openvino/op/greater_eq.hpp index f4731a2da74a50..1f5fe1f984c95d 100644 --- a/src/core/include/openvino/op/greater_eq.hpp +++ b/src/core/include/openvino/op/greater_eq.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API GreaterEqual : public util::BinaryElementwiseComparison { public: - OPENVINO_OP("GreaterEqual", "opset1", op::util::BinaryElementwiseComparison, 1); + OPENVINO_OP("GreaterEqual", "opset1", op::util::BinaryElementwiseComparison); /// \brief Constructs a greater-than-or-equal operation. GreaterEqual() : util::BinaryElementwiseComparison(AutoBroadcastType::NUMPY) {} /// \brief Constructs a greater-than-or-equal operation. diff --git a/src/core/include/openvino/op/group_conv.hpp b/src/core/include/openvino/op/group_conv.hpp index f01c2a7f3dc891..cfb7c6e313c640 100644 --- a/src/core/include/openvino/op/group_conv.hpp +++ b/src/core/include/openvino/op/group_conv.hpp @@ -7,14 +7,15 @@ #include "openvino/op/convolution.hpp" #include "openvino/op/op.hpp" #include "openvino/op/util/attr_types.hpp" +#include "openvino/op/util/convolution_base.hpp" namespace ov { namespace op { namespace v1 { /// \brief Batched convolution operation, with optional window dilation and stride. -class OPENVINO_API GroupConvolution : public Op { +class OPENVINO_API GroupConvolution : public util::ConvolutionFwdPropBase { public: - OPENVINO_OP("GroupConvolution", "opset1", op::Op, 1); + OPENVINO_OP("GroupConvolution", "opset1", op::util::ConvolutionFwdPropBase); /// \brief Constructs a batched convolution operation. GroupConvolution() = default; @@ -49,84 +50,12 @@ class OPENVINO_API GroupConvolution : public Op { void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - /// \return The strides. - const Strides& get_strides() const { - return m_strides; - } - void set_strides(const Strides& strides) { - m_strides = strides; - } - /// \return The dilations. - const Strides& get_dilations() const { - return m_dilations; - } - void set_dilations(const Strides& dilations) { - m_dilations = dilations; - } - /// \return The padding-below sizes (possibly negative). - const CoordinateDiff& get_pads_begin() const { - return m_pads_begin; - } - void set_pads_begin(const CoordinateDiff& pads_begin) { - m_pads_begin = pads_begin; - } - /// \return The padding-above sizes (possibly negative). - const CoordinateDiff& get_pads_end() const { - return m_pads_end; - } - void set_pads_end(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - OPENVINO_DEPRECATED("This method is deprecated and will be removed soon. Please use set_pads_end instead.") - void set_adding_above(const CoordinateDiff& pads_end) { - set_pads_end(pads_end); - } - /// \return The pad type for convolution. - const PadType& get_auto_pad() const { - return m_auto_pad; - } - void set_auto_pad(const PadType& auto_pad) { - m_auto_pad = auto_pad; - } - -protected: - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; - PadType m_auto_pad; - int64_t m_num_spatial = -1; - -private: - template - friend int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void update_and_validate_attributes(ConvType* op, int64_t num_spatial); - template - friend bool resolve_auto_pad_for_shape(const ConvType* op, - CoordinateDiff& pads_begin, - CoordinateDiff& pads_end, - const std::vector& input_shapes, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void shape_infer(const GroupConvolution* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const std::vector& input_shapes, - std::vector& output_shapes); }; /// \brief Data batch backprop for batched convolution operation. -class OPENVINO_API GroupConvolutionBackpropData : public Op { +class OPENVINO_API GroupConvolutionBackpropData : public util::ConvolutionBackPropBase { public: - OPENVINO_OP("GroupConvolutionBackpropData", "opset1", op::Op, 1); + OPENVINO_OP("GroupConvolutionBackpropData", "opset1", op::util::ConvolutionBackPropBase); /// \brief Constructs a batched-convolution data batch-backprop operation. GroupConvolutionBackpropData(); @@ -243,97 +172,6 @@ class OPENVINO_API GroupConvolutionBackpropData : public Op { /// \return The spatial shape of the output. const PartialShape get_convolution_output_shape() const; void set_output_shape(const Shape& output_shape); - /// \return The strides from the forward prop. - const Strides& get_strides() const { - return m_strides; - } - void set_strides(const Strides& strides) { - m_strides = strides; - } - /// \return The dilations from the forward prop. - const Strides& get_dilations() const { - return m_dilations; - } - void set_dilations(const Strides& dilations) { - m_dilations = dilations; - } - /// \return The number of pixels to add to the beginning along each axis. - const CoordinateDiff& get_pads_begin() const { - return m_pads_begin; - } - void set_pads_begin(const CoordinateDiff& pads_begin) { - m_pads_begin = pads_begin; - } - /// \return The number of pixels to add to the ending along each axis. - const CoordinateDiff& get_pads_end() const { - return m_pads_end; - } - void set_pads_end(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - /// \return The auto pad. - const PadType& get_auto_pad() const { - return m_auto_pad; - } - void set_auto_pad(const PadType& auto_pad) { - m_auto_pad = auto_pad; - } - /// \return The output padding. - const CoordinateDiff& get_output_padding() const { - return m_output_padding; - } - void set_output_padding(const CoordinateDiff& output_padding) { - m_output_padding = output_padding; - } - -protected: - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; - PadType m_auto_pad; - CoordinateDiff m_output_padding; - - int64_t m_num_spatial = -1; - -private: - template - friend int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const PartialShape& output_shapes_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void update_and_validate_attributes(ConvType* op, int64_t num_spatial); - - template - friend void update_and_validate_attributes_back_prop(ConvType* op, int64_t num_spatial); - - template - friend bool resolve_auto_pad_for_shape_back_prop(const ConvType* op, - CoordinateDiff& pads_begin, - CoordinateDiff& pads_end, - const std::vector& input_shapes, - ShapeType& output_spatial_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims); - - template - friend void shape_infer(const GroupConvolutionBackpropData* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const T& output_shape_from_input, - const std::vector& input_shapes, - std::vector& output_shapes); }; } // namespace v1 } // namespace op diff --git a/src/core/include/openvino/op/gru_cell.hpp b/src/core/include/openvino/op/gru_cell.hpp index 2610c4731ae139..15543eec2b943d 100644 --- a/src/core/include/openvino/op/gru_cell.hpp +++ b/src/core/include/openvino/op/gru_cell.hpp @@ -24,7 +24,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API GRUCell : public util::RNNCellBase { public: - OPENVINO_OP("GRUCell", "opset3", op::util::RNNCellBase, 3); + OPENVINO_OP("GRUCell", "opset3", op::util::RNNCellBase); GRUCell(); /// /// \brief Constructs GRUCell node. diff --git a/src/core/include/openvino/op/gru_sequence.hpp b/src/core/include/openvino/op/gru_sequence.hpp index 1fc9e7c9147fcf..fae54509ad0c64 100644 --- a/src/core/include/openvino/op/gru_sequence.hpp +++ b/src/core/include/openvino/op/gru_sequence.hpp @@ -19,7 +19,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API GRUSequence : public util::RNNCellBase { public: - OPENVINO_OP("GRUSequence", "opset5", op::Op, 5); + OPENVINO_OP("GRUSequence", "opset5", op::Op); GRUSequence(); GRUSequence(const Output& X, diff --git a/src/core/include/openvino/op/hsigmoid.hpp b/src/core/include/openvino/op/hsigmoid.hpp index abf8d2e1cf87a9..71b07ada902617 100644 --- a/src/core/include/openvino/op/hsigmoid.hpp +++ b/src/core/include/openvino/op/hsigmoid.hpp @@ -17,7 +17,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API HSigmoid : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("HSigmoid", "opset5", op::util::UnaryElementwiseArithmetic, 5); + OPENVINO_OP("HSigmoid", "opset5", op::util::UnaryElementwiseArithmetic); HSigmoid() = default; /// \brief Constructs a HSigmoid operation. diff --git a/src/core/include/openvino/op/hswish.hpp b/src/core/include/openvino/op/hswish.hpp index b20e3498b90dd0..34cff2955b5ab6 100644 --- a/src/core/include/openvino/op/hswish.hpp +++ b/src/core/include/openvino/op/hswish.hpp @@ -17,7 +17,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API HSwish : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("HSwish", "opset4", op::util::UnaryElementwiseArithmetic, 4); + OPENVINO_OP("HSwish", "opset4", op::util::UnaryElementwiseArithmetic); HSwish() = default; /// \brief Constructs a HSwish (hard version of Swish) operation. diff --git a/src/core/include/openvino/op/idft.hpp b/src/core/include/openvino/op/idft.hpp index ea6b0a737a44ba..1fd0948bc9c8a2 100644 --- a/src/core/include/openvino/op/idft.hpp +++ b/src/core/include/openvino/op/idft.hpp @@ -17,7 +17,7 @@ namespace v7 { /// \ingroup ov_ops_cpp_api class OPENVINO_API IDFT : public util::FFTBase { public: - OPENVINO_OP("IDFT", "opset7", util::FFTBase, 7); + OPENVINO_OP("IDFT", "opset7", util::FFTBase); IDFT() = default; /// \brief Constructs a IDFT operation. IDFT is performed for full size axes. diff --git a/src/core/include/openvino/op/interpolate.hpp b/src/core/include/openvino/op/interpolate.hpp index cec3a88e3f5f3f..d6e32cc28cb3c2 100644 --- a/src/core/include/openvino/op/interpolate.hpp +++ b/src/core/include/openvino/op/interpolate.hpp @@ -83,7 +83,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Interpolate : public util::InterpolateBase { public: - OPENVINO_OP("Interpolate", "opset4", util::InterpolateBase, 4); + OPENVINO_OP("Interpolate", "opset4", util::InterpolateBase); Interpolate() = default; /// \brief Constructs a Interpolate operation without 'axes' input. @@ -190,7 +190,7 @@ namespace v11 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Interpolate : public util::InterpolateBase { public: - OPENVINO_OP("Interpolate", "opset11", util::InterpolateBase, 11); + OPENVINO_OP("Interpolate", "opset11", util::InterpolateBase); Interpolate() = default; /// \brief Constructs a Interpolate operation without 'axes' input. /// diff --git a/src/core/include/openvino/op/less.hpp b/src/core/include/openvino/op/less.hpp index d11285b871e392..3d87ab9b6ffbe3 100644 --- a/src/core/include/openvino/op/less.hpp +++ b/src/core/include/openvino/op/less.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Less : public util::BinaryElementwiseComparison { public: - OPENVINO_OP("Less", "opset1", op::util::BinaryElementwiseComparison, 1); + OPENVINO_OP("Less", "opset1", op::util::BinaryElementwiseComparison); /// \brief Constructs a less-than operation. Less() : util::BinaryElementwiseComparison(AutoBroadcastType::NUMPY) {} /// \brief Constructs a less-than operation. diff --git a/src/core/include/openvino/op/less_eq.hpp b/src/core/include/openvino/op/less_eq.hpp index 6b725f749c23ae..111f4c07140af5 100644 --- a/src/core/include/openvino/op/less_eq.hpp +++ b/src/core/include/openvino/op/less_eq.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LessEqual : public util::BinaryElementwiseComparison { public: - OPENVINO_OP("LessEqual", "opset1", op::util::BinaryElementwiseComparison, 1); + OPENVINO_OP("LessEqual", "opset1", op::util::BinaryElementwiseComparison); /// \brief Constructs a less-than-or-equal operation. LessEqual() : util::BinaryElementwiseComparison(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/log_softmax.hpp b/src/core/include/openvino/op/log_softmax.hpp index d441e645998aea..b4bca830a0832f 100644 --- a/src/core/include/openvino/op/log_softmax.hpp +++ b/src/core/include/openvino/op/log_softmax.hpp @@ -14,7 +14,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LogSoftmax : public Op { public: - OPENVINO_OP("LogSoftmax", "opset5", op::Op, 5); + OPENVINO_OP("LogSoftmax", "opset5", op::Op); LogSoftmax() = default; /// \brief Constructs a LogSoftmax operation. /// diff --git a/src/core/include/openvino/op/logical_and.hpp b/src/core/include/openvino/op/logical_and.hpp index 8580accfe46df7..6d55f8f3585e0f 100644 --- a/src/core/include/openvino/op/logical_and.hpp +++ b/src/core/include/openvino/op/logical_and.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LogicalAnd : public util::BinaryElementwiseLogical { public: - OPENVINO_OP("LogicalAnd", "opset1", util::BinaryElementwiseLogical, 1); + OPENVINO_OP("LogicalAnd", "opset1", util::BinaryElementwiseLogical); /// \brief Constructs a logical-and operation. LogicalAnd() = default; diff --git a/src/core/include/openvino/op/logical_not.hpp b/src/core/include/openvino/op/logical_not.hpp index 9b50a8e2ecce1e..c5421b8db14a47 100644 --- a/src/core/include/openvino/op/logical_not.hpp +++ b/src/core/include/openvino/op/logical_not.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LogicalNot : public Op { public: - OPENVINO_OP("LogicalNot", "opset1", op::Op, 1); + OPENVINO_OP("LogicalNot", "opset1", op::Op); /// \brief Constructs a logical negation operation. LogicalNot() = default; /// \brief Constructs a logical negation operation. diff --git a/src/core/include/openvino/op/logical_or.hpp b/src/core/include/openvino/op/logical_or.hpp index 379b773d37617f..15c00eea04baf3 100644 --- a/src/core/include/openvino/op/logical_or.hpp +++ b/src/core/include/openvino/op/logical_or.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LogicalOr : public util::BinaryElementwiseLogical { public: - OPENVINO_OP("LogicalOr", "opset1", util::BinaryElementwiseLogical, 1); + OPENVINO_OP("LogicalOr", "opset1", util::BinaryElementwiseLogical); LogicalOr() = default; /// \brief Constructs a logical-or operation. /// diff --git a/src/core/include/openvino/op/logical_xor.hpp b/src/core/include/openvino/op/logical_xor.hpp index 9e94a1756f98c0..41ad89abca2638 100644 --- a/src/core/include/openvino/op/logical_xor.hpp +++ b/src/core/include/openvino/op/logical_xor.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LogicalXor : public util::BinaryElementwiseLogical { public: - OPENVINO_OP("LogicalXor", "opset2", util::BinaryElementwiseLogical, 1); + OPENVINO_OP("LogicalXor", "opset2", util::BinaryElementwiseLogical); LogicalXor() = default; /// \brief Constructs a logical-xor operation. /// diff --git a/src/core/include/openvino/op/loop.hpp b/src/core/include/openvino/op/loop.hpp index 7bbc00dc75c7fb..cb174d588b7bc3 100644 --- a/src/core/include/openvino/op/loop.hpp +++ b/src/core/include/openvino/op/loop.hpp @@ -31,7 +31,7 @@ class OPENVINO_API Loop : public op::util::SubGraphOp { int64_t body_condition_output_idx = -1; }; - OPENVINO_OP("Loop", "opset5", op::util::SubGraphOp, 5); + OPENVINO_OP("Loop", "opset5", op::util::SubGraphOp); /// \brief Constructs a Loop operation. Loop() = default; diff --git a/src/core/include/openvino/op/lstm_cell.hpp b/src/core/include/openvino/op/lstm_cell.hpp index 3c9e53be10c46b..249b3dccdc2bf6 100644 --- a/src/core/include/openvino/op/lstm_cell.hpp +++ b/src/core/include/openvino/op/lstm_cell.hpp @@ -278,7 +278,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LSTMCell : public util::RNNCellBase { public: - OPENVINO_OP("LSTMCell", "opset4", op::util::RNNCellBase, 4); + OPENVINO_OP("LSTMCell", "opset4", op::util::RNNCellBase); LSTMCell(); /// diff --git a/src/core/include/openvino/op/lstm_sequence.hpp b/src/core/include/openvino/op/lstm_sequence.hpp index 3296bd54208134..1e7599a35ba982 100644 --- a/src/core/include/openvino/op/lstm_sequence.hpp +++ b/src/core/include/openvino/op/lstm_sequence.hpp @@ -127,7 +127,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API LSTMSequence : public util::RNNCellBase { public: - OPENVINO_OP("LSTMSequence", "opset5", util::RNNCellBase, 5); + OPENVINO_OP("LSTMSequence", "opset5", util::RNNCellBase); LSTMSequence() = default; using direction = RecurrentSequenceDirection; diff --git a/src/core/include/openvino/op/max_pool.hpp b/src/core/include/openvino/op/max_pool.hpp index 2acfdb9d12b6ce..c1741eef6cb717 100644 --- a/src/core/include/openvino/op/max_pool.hpp +++ b/src/core/include/openvino/op/max_pool.hpp @@ -15,7 +15,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API MaxPool : public op::util::MaxPoolBase { public: - OPENVINO_OP("MaxPool", "opset1", op::util::MaxPoolBase, 1); + OPENVINO_OP("MaxPool", "opset1", op::util::MaxPoolBase); /// \brief Constructs a batched max pooling operation. MaxPool() = default; diff --git a/src/core/include/openvino/op/maximum.hpp b/src/core/include/openvino/op/maximum.hpp index 5c21463c2ec727..742878b09c4eba 100644 --- a/src/core/include/openvino/op/maximum.hpp +++ b/src/core/include/openvino/op/maximum.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Maximum : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Maximum", "opset1", op::util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Maximum", "opset1", op::util::BinaryElementwiseArithmetic); /// \brief Constructs a maximum operation. Maximum() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/minimum.hpp b/src/core/include/openvino/op/minimum.hpp index afd75c41ea577b..c8cfc5c9d7c999 100644 --- a/src/core/include/openvino/op/minimum.hpp +++ b/src/core/include/openvino/op/minimum.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Minimum : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Minimum", "opset1", op::util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Minimum", "opset1", op::util::BinaryElementwiseArithmetic); /// \brief Constructs a minimum operation. Minimum() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/mish.hpp b/src/core/include/openvino/op/mish.hpp index 1d3a53ca7534c4..455bd713166f95 100644 --- a/src/core/include/openvino/op/mish.hpp +++ b/src/core/include/openvino/op/mish.hpp @@ -15,7 +15,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Mish : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("Mish", "opset4", util::UnaryElementwiseArithmetic, 4); + OPENVINO_OP("Mish", "opset4", util::UnaryElementwiseArithmetic); Mish() = default; /// \brief Constructs an Mish operation. diff --git a/src/core/include/openvino/op/mod.hpp b/src/core/include/openvino/op/mod.hpp index 749a7ae2b7e5d4..5e58a2ec03d733 100644 --- a/src/core/include/openvino/op/mod.hpp +++ b/src/core/include/openvino/op/mod.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Mod : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Mod", "opset1", op::util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Mod", "opset1", op::util::BinaryElementwiseArithmetic); /// \brief Constructs a Mod node. Mod() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/multiply.hpp b/src/core/include/openvino/op/multiply.hpp index 259c0b9f03a117..2e2f3bd4c73000 100644 --- a/src/core/include/openvino/op/multiply.hpp +++ b/src/core/include/openvino/op/multiply.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Multiply : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Multiply", "opset1", util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Multiply", "opset1", util::BinaryElementwiseArithmetic); /// \brief Constructs a multiplication operation. Multiply() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/mvn.hpp b/src/core/include/openvino/op/mvn.hpp index c9a3920aedbb4b..7f198ec1444047 100644 --- a/src/core/include/openvino/op/mvn.hpp +++ b/src/core/include/openvino/op/mvn.hpp @@ -99,7 +99,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API MVN : public Op { public: - OPENVINO_OP("MVN", "opset6", op::Op, 6); + OPENVINO_OP("MVN", "opset6", op::Op); MVN() = default; /// \brief Constructs an MVN operation. diff --git a/src/core/include/openvino/op/non_max_suppression.hpp b/src/core/include/openvino/op/non_max_suppression.hpp index 143df579e0ac69..5216c53700684f 100644 --- a/src/core/include/openvino/op/non_max_suppression.hpp +++ b/src/core/include/openvino/op/non_max_suppression.hpp @@ -16,7 +16,7 @@ class OPENVINO_API NonMaxSuppression : public Op { public: enum class BoxEncodingType { CORNER, CENTER }; - OPENVINO_OP("NonMaxSuppression", "opset1", op::Op, 1); + OPENVINO_OP("NonMaxSuppression", "opset1", op::Op); NonMaxSuppression() = default; @@ -86,7 +86,7 @@ class OPENVINO_API NonMaxSuppression : public Op { public: enum class BoxEncodingType { CORNER, CENTER }; - OPENVINO_OP("NonMaxSuppression", "opset3", op::Op, 3); + OPENVINO_OP("NonMaxSuppression", "opset3", op::Op); NonMaxSuppression() = default; /// \brief Constructs a NonMaxSuppression operation. @@ -166,7 +166,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API NonMaxSuppression : public op::v3::NonMaxSuppression { public: - OPENVINO_OP("NonMaxSuppression", "opset4", op::v3::NonMaxSuppression, 4); + OPENVINO_OP("NonMaxSuppression", "opset4", op::v3::NonMaxSuppression); NonMaxSuppression() = default; /// \brief Constructs a NonMaxSuppression operation. @@ -217,7 +217,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API NonMaxSuppression : public Op { public: - OPENVINO_OP("NonMaxSuppression", "opset5", op::Op, 5); + OPENVINO_OP("NonMaxSuppression", "opset5", op::Op); enum class BoxEncodingType { CORNER, CENTER }; NonMaxSuppression() = default; @@ -365,7 +365,7 @@ namespace v9 { /// class OPENVINO_API NonMaxSuppression : public Op { public: - OPENVINO_OP("NonMaxSuppression", "opset9", op::Op, 9); + OPENVINO_OP("NonMaxSuppression", "opset9", op::Op); enum class BoxEncodingType { CORNER, CENTER }; NonMaxSuppression() = default; diff --git a/src/core/include/openvino/op/non_zero.hpp b/src/core/include/openvino/op/non_zero.hpp index f45ae824a47cd5..e14d757e5ff341 100644 --- a/src/core/include/openvino/op/non_zero.hpp +++ b/src/core/include/openvino/op/non_zero.hpp @@ -20,7 +20,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API NonZero : public Op { public: - OPENVINO_OP("NonZero", "opset3", op::Op, 3); + OPENVINO_OP("NonZero", "opset3", op::Op); /// \brief Constructs a NonZero operation. NonZero() = default; /// \brief Constructs a NonZero operation. diff --git a/src/core/include/openvino/op/not_equal.hpp b/src/core/include/openvino/op/not_equal.hpp index 930244094d37a6..dfae8b59a8fd0e 100644 --- a/src/core/include/openvino/op/not_equal.hpp +++ b/src/core/include/openvino/op/not_equal.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API NotEqual : public util::BinaryElementwiseComparison { public: - OPENVINO_OP("NotEqual", "opset1", op::util::BinaryElementwiseComparison, 1); + OPENVINO_OP("NotEqual", "opset1", op::util::BinaryElementwiseComparison); /// \brief Constructs a not-equal operation. NotEqual() : util::BinaryElementwiseComparison(AutoBroadcastType::NUMPY) {} /// \brief Constructs a not-equal operation. diff --git a/src/core/include/openvino/op/one_hot.hpp b/src/core/include/openvino/op/one_hot.hpp index e911d838a52baa..621fd8483c0649 100644 --- a/src/core/include/openvino/op/one_hot.hpp +++ b/src/core/include/openvino/op/one_hot.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API OneHot : public Op { public: - OPENVINO_OP("OneHot", "opset1", op::Op, 1); + OPENVINO_OP("OneHot", "opset1", op::Op); /// \brief Constructs a one-hot operation. OneHot() = default; diff --git a/src/core/include/openvino/op/op.hpp b/src/core/include/openvino/op/op.hpp index c0a57a90323660..7964007180775f 100644 --- a/src/core/include/openvino/op/op.hpp +++ b/src/core/include/openvino/op/op.hpp @@ -16,7 +16,6 @@ #define OPENVINO_OP(...) \ _OPENVINO_RTTI_EXPAND(_OPENVINO_RTTI_DEFINITION_SELECTOR(__VA_ARGS__, \ - _OPENVINO_RTTI_WITH_TYPE_VERSIONS_PARENT, \ _OPENVINO_RTTI_WITH_TYPE_VERSION_PARENT, \ _OPENVINO_RTTI_OP_WITH_TYPE_VERSION, \ _OPENVINO_RTTI_OP_WITH_TYPE)(__VA_ARGS__)) \ @@ -40,7 +39,7 @@ class OPENVINO_API Op : public Node { public: _OPENVINO_HIDDEN_METHOD static const ::ov::Node::type_info_t& get_type_info_static() { - static ::ov::Node::type_info_t info{"Op", 0, "util"}; + static ::ov::Node::type_info_t info{"Op", "util"}; info.hash(); return info; } diff --git a/src/core/include/openvino/op/pad.hpp b/src/core/include/openvino/op/pad.hpp index 4f8779f6d16b4b..a45c1f33dd52ef 100644 --- a/src/core/include/openvino/op/pad.hpp +++ b/src/core/include/openvino/op/pad.hpp @@ -15,7 +15,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Pad : public Op { public: - OPENVINO_OP("Pad", "opset1", op::Op, 1); + OPENVINO_OP("Pad", "opset1", op::Op); /// \brief Constructs a generic padding operation. /// diff --git a/src/core/include/openvino/op/power.hpp b/src/core/include/openvino/op/power.hpp index c2bb08f4154bcf..c89a98c61a203b 100644 --- a/src/core/include/openvino/op/power.hpp +++ b/src/core/include/openvino/op/power.hpp @@ -28,7 +28,7 @@ namespace v1 { // clang-format on class OPENVINO_API Power : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Power", "opset1", op::util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Power", "opset1", op::util::BinaryElementwiseArithmetic); Power() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/proposal.hpp b/src/core/include/openvino/op/proposal.hpp index 6f7960133d3171..c09282594d3bcb 100644 --- a/src/core/include/openvino/op/proposal.hpp +++ b/src/core/include/openvino/op/proposal.hpp @@ -78,7 +78,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Proposal : public op::v0::Proposal { public: - OPENVINO_OP("Proposal", "opset4", op::Op, 4); + OPENVINO_OP("Proposal", "opset4", op::Op); Proposal() = default; /// \brief Constructs a Proposal operation /// diff --git a/src/core/include/openvino/op/psroi_pooling.hpp b/src/core/include/openvino/op/psroi_pooling.hpp index 39610f8bd06c05..9f35139ac03e5c 100644 --- a/src/core/include/openvino/op/psroi_pooling.hpp +++ b/src/core/include/openvino/op/psroi_pooling.hpp @@ -43,21 +43,56 @@ class OPENVINO_API PSROIPooling : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + /** + * @brief Set the output channel dimension size. + * @param output_dim Channel dimension size. + */ + void set_output_dim(size_t output_dim); size_t get_output_dim() const { return m_output_dim; } + + /** + * @brief Set the output groups number. + * @param group_size Number of groups. + */ + void set_group_size(size_t group_size); size_t get_group_size() const { return m_group_size; } + + /** + * @brief Set the spatial scale. + * @param scale Spatial scale value. + */ + void set_spatial_scale(float scale); float get_spatial_scale() const { return m_spatial_scale; } + + /** + * @brief Set the number of bins over image width. + * @param x Number of bins over width (x) axis. + */ + void set_spatial_bins_x(int x); int get_spatial_bins_x() const { return m_spatial_bins_x; } + + /** + * @brief Set the number of bins over image height. + * @param y Number of bins over height (y) axis. + */ + void set_spatial_bins_y(int y); int get_spatial_bins_y() const { return m_spatial_bins_y; } + + /** + * @brief Set the pooling mode. + * @param mode Pooling mode name. + */ + void set_mode(std::string mode); const std::string& get_mode() const { return m_mode; } diff --git a/src/core/include/openvino/op/range.hpp b/src/core/include/openvino/op/range.hpp index 1cd44aed49f2de..1222d14874dba0 100644 --- a/src/core/include/openvino/op/range.hpp +++ b/src/core/include/openvino/op/range.hpp @@ -13,7 +13,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Range : public Op { public: - OPENVINO_OP("Range", "opset4", op::Op, 4); + OPENVINO_OP("Range", "opset4", op::Op); /// \brief Constructs an unitialized range operation. Range() = default; diff --git a/src/core/include/openvino/op/read_value.hpp b/src/core/include/openvino/op/read_value.hpp index 87a861b4a1e3bf..38c539427b0c4c 100644 --- a/src/core/include/openvino/op/read_value.hpp +++ b/src/core/include/openvino/op/read_value.hpp @@ -15,7 +15,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReadValue : public util::ReadValueBase { public: - OPENVINO_OP("ReadValue", "opset3", util::ReadValueBase, 3); + OPENVINO_OP("ReadValue", "opset3", util::ReadValueBase); ReadValue() = default; /// \brief Constructs a ReadValue operation. @@ -45,7 +45,7 @@ namespace v6 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReadValue : public util::ReadValueBase { public: - OPENVINO_OP("ReadValue", "opset6", util::ReadValueBase, 6); + OPENVINO_OP("ReadValue", "opset6", util::ReadValueBase); ReadValue() = default; /// \brief Constructs a ReadValue operation. diff --git a/src/core/include/openvino/op/reduce_l1.hpp b/src/core/include/openvino/op/reduce_l1.hpp index a9f5024c6aaf06..4889e5c52a6aba 100644 --- a/src/core/include/openvino/op/reduce_l1.hpp +++ b/src/core/include/openvino/op/reduce_l1.hpp @@ -16,7 +16,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceL1 : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceL1", "opset4", util::ArithmeticReductionKeepDims, 4); + OPENVINO_OP("ReduceL1", "opset4", util::ArithmeticReductionKeepDims); /// \brief Constructs a reducet L1-norm operation. ReduceL1() = default; /// \brief Constructs a reduce L1-norm operation. diff --git a/src/core/include/openvino/op/reduce_l2.hpp b/src/core/include/openvino/op/reduce_l2.hpp index c2a18ac3668483..9f9b38b7dc5747 100644 --- a/src/core/include/openvino/op/reduce_l2.hpp +++ b/src/core/include/openvino/op/reduce_l2.hpp @@ -15,7 +15,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceL2 : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceL2", "opset4", util::ArithmeticReductionKeepDims, 4); + OPENVINO_OP("ReduceL2", "opset4", util::ArithmeticReductionKeepDims); /// \brief Constructs a reducet L2-norm operation. ReduceL2() = default; /// \brief Constructs a reduce L2-norm operation. diff --git a/src/core/include/openvino/op/reduce_logical_and.hpp b/src/core/include/openvino/op/reduce_logical_and.hpp index b7e839ab1069da..1358702a1fd39a 100644 --- a/src/core/include/openvino/op/reduce_logical_and.hpp +++ b/src/core/include/openvino/op/reduce_logical_and.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceLogicalAnd : public util::LogicalReductionKeepDims { public: - OPENVINO_OP("ReduceLogicalAnd", "opset1", util::LogicalReductionKeepDims, 1); + OPENVINO_OP("ReduceLogicalAnd", "opset1", util::LogicalReductionKeepDims); ReduceLogicalAnd() = default; /// \brief Constructs a ReduceLogicalAnd node. /// diff --git a/src/core/include/openvino/op/reduce_logical_or.hpp b/src/core/include/openvino/op/reduce_logical_or.hpp index 67fe065db4585b..36a3fd34759b24 100644 --- a/src/core/include/openvino/op/reduce_logical_or.hpp +++ b/src/core/include/openvino/op/reduce_logical_or.hpp @@ -16,7 +16,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceLogicalOr : public util::LogicalReductionKeepDims { public: - OPENVINO_OP("ReduceLogicalOr", "opset1", util::LogicalReductionKeepDims, 1); + OPENVINO_OP("ReduceLogicalOr", "opset1", util::LogicalReductionKeepDims); ReduceLogicalOr() = default; /// \brief Constructs a ReduceLogicalOr node. /// diff --git a/src/core/include/openvino/op/reduce_max.hpp b/src/core/include/openvino/op/reduce_max.hpp index b1579ad5ccbf03..499dec82bb9f77 100644 --- a/src/core/include/openvino/op/reduce_max.hpp +++ b/src/core/include/openvino/op/reduce_max.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceMax : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceMax", "opset1", util::ArithmeticReductionKeepDims, 1); + OPENVINO_OP("ReduceMax", "opset1", util::ArithmeticReductionKeepDims); /// \brief Constructs a summation operation. ReduceMax() = default; /// \brief Constructs a summation operation. diff --git a/src/core/include/openvino/op/reduce_mean.hpp b/src/core/include/openvino/op/reduce_mean.hpp index 41459857e081da..7b50dd57b7dafc 100644 --- a/src/core/include/openvino/op/reduce_mean.hpp +++ b/src/core/include/openvino/op/reduce_mean.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceMean : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceMean", "opset1", util::ArithmeticReductionKeepDims, 1); + OPENVINO_OP("ReduceMean", "opset1", util::ArithmeticReductionKeepDims); ReduceMean() = default; /// \param arg The tensor to be summed. diff --git a/src/core/include/openvino/op/reduce_min.hpp b/src/core/include/openvino/op/reduce_min.hpp index 464b232ed5fbf9..830021a0bb2ae0 100644 --- a/src/core/include/openvino/op/reduce_min.hpp +++ b/src/core/include/openvino/op/reduce_min.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceMin : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceMin", "opset1", util::ArithmeticReductionKeepDims, 1); + OPENVINO_OP("ReduceMin", "opset1", util::ArithmeticReductionKeepDims); /// \brief Constructs a summation operation. ReduceMin() = default; /// \brief Constructs a summation operation. diff --git a/src/core/include/openvino/op/reduce_prod.hpp b/src/core/include/openvino/op/reduce_prod.hpp index c8697c81bcdc2a..4a9af6339b6797 100644 --- a/src/core/include/openvino/op/reduce_prod.hpp +++ b/src/core/include/openvino/op/reduce_prod.hpp @@ -15,7 +15,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ReduceProd : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceProd", "opset1", util::ArithmeticReductionKeepDims, 1); + OPENVINO_OP("ReduceProd", "opset1", util::ArithmeticReductionKeepDims); /// \brief Constructs a product reduction operation. ReduceProd() = default; /// \brief Constructs a product reduction operation. diff --git a/src/core/include/openvino/op/reduce_sum.hpp b/src/core/include/openvino/op/reduce_sum.hpp index 60622fd2b7e3ea..7a3221c68e52ef 100644 --- a/src/core/include/openvino/op/reduce_sum.hpp +++ b/src/core/include/openvino/op/reduce_sum.hpp @@ -61,7 +61,7 @@ namespace v1 { // clang-format on class OPENVINO_API ReduceSum : public util::ArithmeticReductionKeepDims { public: - OPENVINO_OP("ReduceSum", "opset1", util::ArithmeticReductionKeepDims, 1); + OPENVINO_OP("ReduceSum", "opset1", util::ArithmeticReductionKeepDims); /// \brief Constructs a summation operation. ReduceSum() = default; /// \brief Constructs a summation operation. diff --git a/src/core/include/openvino/op/reshape.hpp b/src/core/include/openvino/op/reshape.hpp index 2905e4ad5ec182..9d4ecc18da1cc0 100644 --- a/src/core/include/openvino/op/reshape.hpp +++ b/src/core/include/openvino/op/reshape.hpp @@ -17,7 +17,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Reshape : public Op { public: - OPENVINO_OP("Reshape", "opset1", op::Op, 1); + OPENVINO_OP("Reshape", "opset1", op::Op); Reshape() = default; /// \brief Constructs a dynamic reshape operation. This operation does not perform /// transpose. diff --git a/src/core/include/openvino/op/reverse.hpp b/src/core/include/openvino/op/reverse.hpp index 37266573f7232d..7b1a904aacf201 100644 --- a/src/core/include/openvino/op/reverse.hpp +++ b/src/core/include/openvino/op/reverse.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Reverse : public Op { public: - OPENVINO_OP("Reverse", "opset1", op::Op, 1); + OPENVINO_OP("Reverse", "opset1", op::Op); enum class Mode { INDEX, MASK }; diff --git a/src/core/include/openvino/op/rnn_sequence.hpp b/src/core/include/openvino/op/rnn_sequence.hpp index 44eaf247281a6a..0ee5543687422f 100644 --- a/src/core/include/openvino/op/rnn_sequence.hpp +++ b/src/core/include/openvino/op/rnn_sequence.hpp @@ -18,7 +18,7 @@ namespace v5 { /// \ingroup ov_ops_cpp_api class OPENVINO_API RNNSequence : public util::RNNCellBase { public: - OPENVINO_OP("RNNSequence", "opset5", util::RNNCellBase, 4); + OPENVINO_OP("RNNSequence", "opset5", util::RNNCellBase); RNNSequence(); diff --git a/src/core/include/openvino/op/roi_align.hpp b/src/core/include/openvino/op/roi_align.hpp index 2922a7bc001df3..ba1ceefe09d3ed 100644 --- a/src/core/include/openvino/op/roi_align.hpp +++ b/src/core/include/openvino/op/roi_align.hpp @@ -14,7 +14,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ROIAlign : public Op { public: - OPENVINO_OP("ROIAlign", "opset3", op::Op, 3); + OPENVINO_OP("ROIAlign", "opset3", op::Op); enum class PoolingMode { AVG, MAX }; ROIAlign() = default; diff --git a/src/core/include/openvino/op/roi_pooling.hpp b/src/core/include/openvino/op/roi_pooling.hpp index b0b04648d7b3ea..57799954a7641f 100644 --- a/src/core/include/openvino/op/roi_pooling.hpp +++ b/src/core/include/openvino/op/roi_pooling.hpp @@ -34,12 +34,30 @@ class OPENVINO_API ROIPooling : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + /// \brief Set the output ROI feature map (pooled_h, pooled_w). + /// \param output_size Shape with pooling attributes pooled_h and pooled_w sizes. + void set_output_roi(Shape output_size); + + /// \brief Get the output ROI feature map shape (H x W) + /// \return Shape with pooled_h and pooled_w attributes. + const Shape& get_output_roi() const; + + OPENVINO_DEPRECATED("Use 'get_output_roi' instead. Use of this member can be ambiguous with Node base " + "'get_output_size' which return number of outputs.") const Shape& get_output_size() const { return m_output_size; } + + /// \brief Set the spatial scale value. + /// \param scale Scale value to set. + void set_spatial_scale(float scale); float get_spatial_scale() const { return m_spatial_scale; } + + /// \brief Set the method of pooling + /// \param method_name Pooling method name. + void set_method(std::string method_name); const std::string& get_method() const { return m_method; } @@ -47,7 +65,7 @@ class OPENVINO_API ROIPooling : public Op { private: Shape m_output_size{0, 0}; - float m_spatial_scale{0}; + float m_spatial_scale{0.0f}; std::string m_method = "max"; }; } // namespace v0 diff --git a/src/core/include/openvino/op/roll.hpp b/src/core/include/openvino/op/roll.hpp index 35344dffbad668..844a39b19cfd68 100644 --- a/src/core/include/openvino/op/roll.hpp +++ b/src/core/include/openvino/op/roll.hpp @@ -13,7 +13,7 @@ namespace v7 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Roll : public Op { public: - OPENVINO_OP("Roll", "opset7", op::Op, 7); + OPENVINO_OP("Roll", "opset7", op::Op); Roll() = default; diff --git a/src/core/include/openvino/op/round.hpp b/src/core/include/openvino/op/round.hpp index 994b5507ca944c..c63cee3738464a 100644 --- a/src/core/include/openvino/op/round.hpp +++ b/src/core/include/openvino/op/round.hpp @@ -18,7 +18,7 @@ namespace v5 { class OPENVINO_API Round : public util::UnaryElementwiseArithmetic { public: enum class RoundMode { HALF_TO_EVEN, HALF_AWAY_FROM_ZERO }; - OPENVINO_OP("Round", "opset5", util::UnaryElementwiseArithmetic, 5); + OPENVINO_OP("Round", "opset5", util::UnaryElementwiseArithmetic); /// \brief Constructs a round operation. Round() = default; diff --git a/src/core/include/openvino/op/scatter_elements_update.hpp b/src/core/include/openvino/op/scatter_elements_update.hpp index 903b1fb9bab0cc..4172e99afc50df 100644 --- a/src/core/include/openvino/op/scatter_elements_update.hpp +++ b/src/core/include/openvino/op/scatter_elements_update.hpp @@ -14,7 +14,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ScatterElementsUpdate : public Op { public: - OPENVINO_OP("ScatterElementsUpdate", "opset3", op::Op, 3); + OPENVINO_OP("ScatterElementsUpdate", "opset3", op::Op); ScatterElementsUpdate() = default; /// \brief Constructs a ScatterElementsUpdate node diff --git a/src/core/include/openvino/op/scatter_nd_update.hpp b/src/core/include/openvino/op/scatter_nd_update.hpp index 1c7ac4355e1009..ab28cd374dfc6b 100644 --- a/src/core/include/openvino/op/scatter_nd_update.hpp +++ b/src/core/include/openvino/op/scatter_nd_update.hpp @@ -13,7 +13,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ScatterNDUpdate : public util::ScatterNDBase { public: - OPENVINO_OP("ScatterNDUpdate", "opset4", util::ScatterNDBase, 3); + OPENVINO_OP("ScatterNDUpdate", "opset4", util::ScatterNDBase); ScatterNDUpdate() = default; /// \param inputs Tensor /// \param indices Index tensor: Data type must be `element::i32` or `element::i64` diff --git a/src/core/include/openvino/op/scatter_update.hpp b/src/core/include/openvino/op/scatter_update.hpp index 66936cf2f73654..4e29bf9ab2ac2a 100644 --- a/src/core/include/openvino/op/scatter_update.hpp +++ b/src/core/include/openvino/op/scatter_update.hpp @@ -15,7 +15,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ScatterUpdate : public util::ScatterBase { public: - OPENVINO_OP("ScatterUpdate", "opset3", util::ScatterBase, 3); + OPENVINO_OP("ScatterUpdate", "opset3", util::ScatterBase); ScatterUpdate() = default; /// /// \brief Constructs ScatterUpdate operator object. diff --git a/src/core/include/openvino/op/select.hpp b/src/core/include/openvino/op/select.hpp index 5d2bec70d6779f..78e8e802ab94d3 100644 --- a/src/core/include/openvino/op/select.hpp +++ b/src/core/include/openvino/op/select.hpp @@ -30,7 +30,7 @@ namespace v1 { // clang-format on class OPENVINO_API Select : public Op { public: - OPENVINO_OP("Select", "opset1", op::Op, 1); + OPENVINO_OP("Select", "opset1", op::Op); /// \brief Constructs a selection operation. Select() : m_auto_broadcast(AutoBroadcastSpec(AutoBroadcastType::NUMPY)) {} diff --git a/src/core/include/openvino/op/shape_of.hpp b/src/core/include/openvino/op/shape_of.hpp index 5e2cace77f9d51..e7ec34c1c87d86 100644 --- a/src/core/include/openvino/op/shape_of.hpp +++ b/src/core/include/openvino/op/shape_of.hpp @@ -13,7 +13,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API ShapeOf : public util::ShapeOfBase { public: - OPENVINO_OP("ShapeOf", "opset3", util::ShapeOfBase, 3); + OPENVINO_OP("ShapeOf", "opset3", util::ShapeOfBase); ShapeOf() = default; /// \brief Constructs a shape-of operation. ShapeOf(const Output& arg, const element::Type output_type = element::i64); diff --git a/src/core/include/openvino/op/softmax.hpp b/src/core/include/openvino/op/softmax.hpp index fc9c414df5af02..8a43c6dae7bdef 100644 --- a/src/core/include/openvino/op/softmax.hpp +++ b/src/core/include/openvino/op/softmax.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Softmax : public Op { public: - OPENVINO_OP("Softmax", "opset1", op::Op, 1); + OPENVINO_OP("Softmax", "opset1", op::Op); Softmax() = default; /// \brief Constructs a softmax operation. diff --git a/src/core/include/openvino/op/softplus.hpp b/src/core/include/openvino/op/softplus.hpp index d3358268ac326c..aaff04caa53471 100644 --- a/src/core/include/openvino/op/softplus.hpp +++ b/src/core/include/openvino/op/softplus.hpp @@ -15,7 +15,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API SoftPlus : public util::UnaryElementwiseArithmetic { public: - OPENVINO_OP("SoftPlus", "opset4", util::UnaryElementwiseArithmetic, 4); + OPENVINO_OP("SoftPlus", "opset4", util::UnaryElementwiseArithmetic); SoftPlus() = default; /// \brief Constructs an SoftPlus operation. diff --git a/src/core/include/openvino/op/space_to_batch.hpp b/src/core/include/openvino/op/space_to_batch.hpp index ceaac33345b2c3..83d47b96ba4c45 100644 --- a/src/core/include/openvino/op/space_to_batch.hpp +++ b/src/core/include/openvino/op/space_to_batch.hpp @@ -23,7 +23,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API SpaceToBatch : public Op { public: - OPENVINO_OP("SpaceToBatch", "opset2", op::Op, 1); + OPENVINO_OP("SpaceToBatch", "opset2", op::Op); SpaceToBatch() = default; diff --git a/src/core/include/openvino/op/split.hpp b/src/core/include/openvino/op/split.hpp index 6a2b70434af549..918457c0d84a05 100644 --- a/src/core/include/openvino/op/split.hpp +++ b/src/core/include/openvino/op/split.hpp @@ -17,7 +17,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Split : public Op { public: - OPENVINO_OP("Split", "opset1", op::Op, 1); + OPENVINO_OP("Split", "opset1", op::Op); /// \brief Constructs a split operation. Split() = default; diff --git a/src/core/include/openvino/op/strided_slice.hpp b/src/core/include/openvino/op/strided_slice.hpp index f52b095424c196..5535e2925800c9 100644 --- a/src/core/include/openvino/op/strided_slice.hpp +++ b/src/core/include/openvino/op/strided_slice.hpp @@ -18,7 +18,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API StridedSlice : public Op { public: - OPENVINO_OP("StridedSlice", "opset1", op::Op, 1); + OPENVINO_OP("StridedSlice", "opset1", op::Op); StridedSlice() = default; diff --git a/src/core/include/openvino/op/subtract.hpp b/src/core/include/openvino/op/subtract.hpp index 3c129cb1f9b233..5fd58da3bd6ff5 100644 --- a/src/core/include/openvino/op/subtract.hpp +++ b/src/core/include/openvino/op/subtract.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Subtract : public util::BinaryElementwiseArithmetic { public: - OPENVINO_OP("Subtract", "opset1", util::BinaryElementwiseArithmetic, 1); + OPENVINO_OP("Subtract", "opset1", util::BinaryElementwiseArithmetic); Subtract() : util::BinaryElementwiseArithmetic(AutoBroadcastType::NUMPY) {} diff --git a/src/core/include/openvino/op/swish.hpp b/src/core/include/openvino/op/swish.hpp index 1395e62e409522..bc9935d7f3e88f 100644 --- a/src/core/include/openvino/op/swish.hpp +++ b/src/core/include/openvino/op/swish.hpp @@ -16,7 +16,7 @@ namespace v4 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Swish : public Op { public: - OPENVINO_OP("Swish", "opset4", op::Op, 4); + OPENVINO_OP("Swish", "opset4", op::Op); Swish() = default; /// \brief Constructs an Swish operation. diff --git a/src/core/include/openvino/op/topk.hpp b/src/core/include/openvino/op/topk.hpp index 2af15c7baefb92..9c2ec7a9ce1492 100644 --- a/src/core/include/openvino/op/topk.hpp +++ b/src/core/include/openvino/op/topk.hpp @@ -18,7 +18,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API TopK : public util::TopKBase { public: - OPENVINO_OP("TopK", "opset1", op::util::TopKBase, 1); + OPENVINO_OP("TopK", "opset1", op::util::TopKBase); using SortType = TopKSortType; using Mode = TopKMode; @@ -69,7 +69,7 @@ namespace v3 { /// \ingroup ov_ops_cpp_api class OPENVINO_API TopK : public util::TopKBase { public: - OPENVINO_OP("TopK", "opset3", op::util::TopKBase, 3); + OPENVINO_OP("TopK", "opset3", op::util::TopKBase); /// \brief Constructs a TopK operation TopK() = default; /// \brief Constructs a TopK operation with two outputs: values and indices. @@ -111,7 +111,7 @@ namespace v11 { /// \ingroup ov_ops_cpp_api class OPENVINO_API TopK : public util::TopKBase { public: - OPENVINO_OP("TopK", "opset11", op::util::TopKBase, 11); + OPENVINO_OP("TopK", "opset11", op::util::TopKBase); /// \brief Constructs a TopK operation TopK() = default; /// \brief Constructs a TopK operation with two outputs: values and indices. diff --git a/src/core/include/openvino/op/transpose.hpp b/src/core/include/openvino/op/transpose.hpp index 341906128d3273..2b4af853893270 100644 --- a/src/core/include/openvino/op/transpose.hpp +++ b/src/core/include/openvino/op/transpose.hpp @@ -13,7 +13,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API Transpose : public Op { public: - OPENVINO_OP("Transpose", "opset1", op::Op, 1); + OPENVINO_OP("Transpose", "opset1", op::Op); Transpose() = default; /// diff --git a/src/core/include/openvino/op/util/convolution_backprop_base.hpp b/src/core/include/openvino/op/util/convolution_backprop_base.hpp new file mode 100644 index 00000000000000..2adeec2e887bf1 --- /dev/null +++ b/src/core/include/openvino/op/util/convolution_backprop_base.hpp @@ -0,0 +1,64 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "convolution_base.hpp" + +namespace ov { +namespace op { +namespace util { +/// \brief Base class for operations like back propagation convolution +class OPENVINO_API ConvolutionBackPropBase : public ConvolutionBase { +public: + OPENVINO_OP("ConvolutionBackPropBase", "util"); + + /// \brief Constructs a conversion operation. + ConvolutionBackPropBase() = default; + + /// \brief Constructs a conversion operation. + /// \param strides Convolution strides. + /// \param pads_begin Amount of padding to be added to the beginning along + /// each axis. For example in case of a 2D input the value + /// of (1, 2) means that 1 element will be added to the + /// top and 2 elements to the left. + /// \param pads_end Amount of padding to be added to the end along each + /// axis. + /// \param dilations The distance in width and height between the weights + /// in the filters tensor. + /// \param auto_pad Specifies how the automatic calculation of padding + /// should be done. + /// \param output_padding The output padding adds additional amount of paddings per + /// each spatial axis in the output tensor. clang-format on + ConvolutionBackPropBase(const OutputVector& arguments, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad = PadType::EXPLICIT, + const CoordinateDiff& output_padding = {}) + : ConvolutionBase(arguments, strides, pads_begin, pads_end, dilations, auto_pad), + m_output_padding{output_padding} {} + + const CoordinateDiff& get_output_padding() const { + return m_output_padding; + } + void set_output_padding(const CoordinateDiff& output_padding) { + m_output_padding = output_padding; + } + +protected: + CoordinateDiff m_output_padding; + + void resize_attributes(size_t num_spatial) { + ConvolutionBase::resize_attributes(num_spatial); + + if (m_output_padding.empty()) { + m_output_padding.resize(num_spatial, 0); + } + } +}; +} // namespace util +} // namespace op +} // namespace ov diff --git a/src/core/include/openvino/op/util/convolution_base.hpp b/src/core/include/openvino/op/util/convolution_base.hpp new file mode 100644 index 00000000000000..0681d1a2e8e2b0 --- /dev/null +++ b/src/core/include/openvino/op/util/convolution_base.hpp @@ -0,0 +1,144 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/coordinate_diff.hpp" +#include "openvino/op/op.hpp" +#include "openvino/op/util/attr_types.hpp" + +namespace ov { +namespace op { +namespace util { +/// \brief Base class for operations like convolutions +class OPENVINO_API ConvolutionBase : public Op { +public: + OPENVINO_OP("ConvolutionBase", "util"); + + /// \brief Constructs a conversion operation. + ConvolutionBase() = default; + + /// \brief Constructs a conversion operation. + /// \param strides Convolution strides. + /// \param pads_begin Amount of padding to be added to the beginning along + /// each axis. For example in case of a 2D input the value + /// of (1, 2) means that 1 element will be added to the + /// top and 2 elements to the left. + /// \param pads_end Amount of padding to be added to the end along each + /// axis. + /// \param dilations The distance in width and height between the weights + /// in the filters tensor. + /// \param auto_pad Specifies how the automatic calculation of padding + /// should be done. + ConvolutionBase(const OutputVector& arguments, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad = PadType::EXPLICIT) + : Op(arguments), + m_strides(strides), + m_dilations(dilations), + m_pads_begin(pads_begin), + m_pads_end(pads_end), + m_auto_pad(auto_pad) {} + + const Strides& get_strides() const { + return m_strides; + } + void set_strides(const Strides& strides) { + m_strides = strides; + } + const Strides& get_dilations() const { + return m_dilations; + } + void set_dilations(const Strides& dilations) { + m_dilations = dilations; + } + const CoordinateDiff& get_pads_begin() const { + return m_pads_begin; + } + void set_pads_begin(const CoordinateDiff& pads_begin) { + m_pads_begin = pads_begin; + } + const CoordinateDiff& get_pads_end() const { + return m_pads_end; + } + OPENVINO_DEPRECATED("This method is deprecated and will be removed soon. Please use set_pads_end instead.") + void set_adding_above(const CoordinateDiff& pads_end) { + set_pads_end(pads_end); + } + void set_pads_end(const CoordinateDiff& pads_end) { + m_pads_end = pads_end; + } + const PadType& get_auto_pad() const { + return m_auto_pad; + } + void set_auto_pad(const PadType& auto_pad) { + m_auto_pad = auto_pad; + } + +protected: + Strides m_strides; + Strides m_dilations; + CoordinateDiff m_pads_begin; + CoordinateDiff m_pads_end; + PadType m_auto_pad; + size_t m_num_spatial = std::numeric_limits::max(); + + void resize_attributes(size_t num_spatial) { + if (m_strides.empty()) { + m_strides.resize(num_spatial, 1); + } + if (m_dilations.empty()) { + m_dilations.resize(num_spatial, 1); + } + } + + void set_num_spatial(size_t num_spatial, const std::vector& input_shapes) { + if (input_shapes[0].rank().is_static() && input_shapes[1].rank().is_static()) { + m_num_spatial = num_spatial; + } + } + +private: + friend bool is_attr_validation_required(const ConvolutionBase* op); + friend size_t get_num_spatial(const ConvolutionBase* op); +}; + +/// \brief Base class for operations like back propagation convolution +class OPENVINO_API ConvolutionFwdPropBase : public ConvolutionBase { +public: + OPENVINO_OP("ConvolutionFwdPropBase", "util"); + + /// \brief Constructs a conversion operation. + ConvolutionFwdPropBase() = default; + + /// \brief Constructs a conversion operation. + /// \param strides Convolution strides. + /// \param pads_begin Amount of padding to be added to the beginning along + /// each axis. For example in case of a 2D input the value + /// of (1, 2) means that 1 element will be added to the + /// top and 2 elements to the left. + /// \param pads_end Amount of padding to be added to the end along each + /// axis. + /// \param dilations The distance in width and height between the weights + /// in the filters tensor. + /// \param auto_pad Specifies how the automatic calculation of padding + /// should be done. + ConvolutionFwdPropBase(const OutputVector& arguments, + const Strides& strides, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end, + const Strides& dilations, + const PadType& auto_pad = PadType::EXPLICIT) + : ConvolutionBase(arguments, strides, pads_begin, pads_end, dilations, auto_pad) {} + +private: + friend bool is_attr_validation_required(const ConvolutionBase* op); +}; + +} // namespace util +} // namespace op +} // namespace ov diff --git a/src/core/include/openvino/op/util/deformable_convolution_base.hpp b/src/core/include/openvino/op/util/deformable_convolution_base.hpp index 5bcca3091e282e..e2b1c530909159 100644 --- a/src/core/include/openvino/op/util/deformable_convolution_base.hpp +++ b/src/core/include/openvino/op/util/deformable_convolution_base.hpp @@ -7,13 +7,14 @@ #include "openvino/core/coordinate_diff.hpp" #include "openvino/op/op.hpp" #include "openvino/op/util/attr_types.hpp" +#include "openvino/op/util/convolution_base.hpp" namespace ov { namespace op { namespace util { /// \brief Base class for operations DeformableConvolution v1 and DeformableConvolution /// v8. -class OPENVINO_API DeformableConvolutionBase : public Op { +class OPENVINO_API DeformableConvolutionBase : public util::ConvolutionBase { public: OPENVINO_OP("DeformableConvolutionBase", "util"); @@ -46,38 +47,7 @@ class OPENVINO_API DeformableConvolutionBase : public Op { int64_t deformable_group = 1); bool visit_attributes(AttributeVisitor& visitor) override; - void validate_and_infer_types() override; - const Strides& get_strides() const { - return m_strides; - } - void set_strides(const Strides& strides) { - m_strides = strides; - } - const Strides& get_dilations() const { - return m_dilations; - } - void set_dilations(const Strides& dilations) { - m_dilations = dilations; - } - const CoordinateDiff& get_pads_begin() const { - return m_pads_begin; - } - void set_pads_begin(const CoordinateDiff& pads_begin) { - m_pads_begin = pads_begin; - } - const CoordinateDiff& get_pads_end() const { - return m_pads_end; - } - void set_pads_end(const CoordinateDiff& pads_end) { - m_pads_end = pads_end; - } - const PadType& get_auto_pad() const { - return m_auto_pad; - } - void set_auto_pad(const PadType& auto_pad) { - m_auto_pad = auto_pad; - } int64_t get_group() const { return m_group; } @@ -92,11 +62,6 @@ class OPENVINO_API DeformableConvolutionBase : public Op { } protected: - Strides m_strides; - Strides m_dilations; - CoordinateDiff m_pads_begin; - CoordinateDiff m_pads_end; - PadType m_auto_pad; int64_t m_group; int64_t m_deformable_group; }; diff --git a/src/core/include/openvino/op/variadic_split.hpp b/src/core/include/openvino/op/variadic_split.hpp index 8c5034cf031cc9..2d6f751d48d3ba 100644 --- a/src/core/include/openvino/op/variadic_split.hpp +++ b/src/core/include/openvino/op/variadic_split.hpp @@ -14,7 +14,7 @@ namespace v1 { /// \ingroup ov_ops_cpp_api class OPENVINO_API VariadicSplit : public Op { public: - OPENVINO_OP("VariadicSplit", "opset1", op::Op, 1); + OPENVINO_OP("VariadicSplit", "opset1", op::Op); /// \brief Constructs a variadic split operation. VariadicSplit() = default; diff --git a/src/core/include/openvino/pass/pattern/op/pattern.hpp b/src/core/include/openvino/pass/pattern/op/pattern.hpp index 694d05473e4f3e..d5004c0ba70f2b 100644 --- a/src/core/include/openvino/pass/pattern/op/pattern.hpp +++ b/src/core/include/openvino/pass/pattern/op/pattern.hpp @@ -39,6 +39,9 @@ std::function)> has_class() { OPENVINO_API std::function)> consumers_count(size_t n); +OPENVINO_API +std::function)> consumers_more_than(size_t n); + OPENVINO_API std::function)> has_static_dim(size_t pos); diff --git a/src/core/include/openvino/runtime/allocator.hpp b/src/core/include/openvino/runtime/allocator.hpp index fdf409eb8a33c4..650fc8f77c36b9 100644 --- a/src/core/include/openvino/runtime/allocator.hpp +++ b/src/core/include/openvino/runtime/allocator.hpp @@ -58,7 +58,7 @@ struct OPENVINO_DEPRECATED("Do not inherit from AllocatorImpl. This class will b virtual bool is_equal(const AllocatorImpl& other) const = 0; protected: - ~AllocatorImpl() = default; + virtual ~AllocatorImpl() = default; }; class Tensor; @@ -92,7 +92,7 @@ class OPENVINO_API Allocator { virtual bool is_equal(const Base& other) const = 0; protected: - ~Base() = default; + virtual ~Base() = default; }; template diff --git a/src/core/reference/CMakeLists.txt b/src/core/reference/CMakeLists.txt index 6f2f0c8b834ee1..60ee5956ac9d94 100644 --- a/src/core/reference/CMakeLists.txt +++ b/src/core/reference/CMakeLists.txt @@ -4,10 +4,6 @@ set(TARGET_NAME "ngraph_reference") -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - ie_add_compiler_flags(/wd4267) -endif() - file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) file(GLOB_RECURSE PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp) diff --git a/src/core/reference/include/ngraph/runtime/reference/convert.hpp b/src/core/reference/include/ngraph/runtime/reference/convert.hpp index 75a0bb847b1b40..0bab3471d8ae78 100644 --- a/src/core/reference/include/ngraph/runtime/reference/convert.hpp +++ b/src/core/reference/include/ngraph/runtime/reference/convert.hpp @@ -107,6 +107,8 @@ void convert(const uint8_t* arg, float16* out, size_t count); template <> void convert(const float16* arg, float* out, size_t count); template <> +void convert(const float* arg, float16* out, size_t count); +template <> void convert(const float* arg, int8_t* out, size_t count); template <> void convert(const float16* arg, int8_t* out, size_t count); diff --git a/src/core/reference/include/ngraph/runtime/reference/gather.hpp b/src/core/reference/include/ngraph/runtime/reference/gather.hpp index 95cdc73c83e791..82a7fb50188af7 100644 --- a/src/core/reference/include/ngraph/runtime/reference/gather.hpp +++ b/src/core/reference/include/ngraph/runtime/reference/gather.hpp @@ -34,7 +34,7 @@ void gather(const T* const data, int64_t axis_size = data_shape[axis]; int64_t data_offset, out_offset, idx; // for out of bound indices is filled with zeros - std::fill(out, out + shape_size(out_shape), 0); + std::fill(out, out + shape_size(out_shape), T{0}); for (int64_t batch = 0; batch < batch_size; batch++) for (int64_t outer_idx = 0; outer_idx < outer_size; outer_idx++) { diff --git a/src/core/reference/include/ngraph/runtime/reference/grid_sample.hpp b/src/core/reference/include/ngraph/runtime/reference/grid_sample.hpp index f2662d56173525..ed3afda4601daa 100644 --- a/src/core/reference/include/ngraph/runtime/reference/grid_sample.hpp +++ b/src/core/reference/include/ngraph/runtime/reference/grid_sample.hpp @@ -133,10 +133,11 @@ DATA_ET bilinear(const DATA_ET* data, const auto x_topleft = std::floor(x_d); const auto dy = y_d - y_topleft; const auto dx = x_d - x_topleft; - const auto v00 = get_padded(data, data_shape, n, c, y_topleft, x_topleft); - const auto v01 = get_padded(data, data_shape, n, c, y_topleft, x_topleft + 1); - const auto v10 = get_padded(data, data_shape, n, c, y_topleft + 1, x_topleft); - const auto v11 = get_padded(data, data_shape, n, c, y_topleft + 1, x_topleft + 1); + const auto v00 = get_padded(data, data_shape, n, c, static_cast(y_topleft), static_cast(x_topleft)); + const auto v01 = get_padded(data, data_shape, n, c, static_cast(y_topleft), static_cast(x_topleft + 1)); + const auto v10 = get_padded(data, data_shape, n, c, static_cast(y_topleft + 1), static_cast(x_topleft)); + const auto v11 = + get_padded(data, data_shape, n, c, static_cast(y_topleft + 1), static_cast(x_topleft + 1)); const auto q0 = (1 - dx) * v00 + dx * v01; const auto q1 = (1 - dx) * v10 + dx * v11; @@ -204,7 +205,13 @@ DATA_ET bicubic(const DATA_ET* data, const auto x_topleft = std::floor(x_d); const auto dy = y_d - y_topleft; const auto dx = x_d - x_topleft; - const auto s = gather_4x4(data, data_shape, n, c, y_topleft - 1, x_topleft - 1, get_padded); + const auto s = gather_4x4(data, + data_shape, + n, + c, + static_cast(y_topleft - 1), + static_cast(x_topleft - 1), + get_padded); const auto cy = cubic_coeffs(dy); const auto cx = cubic_coeffs(dx); diff --git a/src/core/reference/include/ngraph/runtime/reference/interpolate.hpp b/src/core/reference/include/ngraph/runtime/reference/interpolate.hpp index 909338ca897484..7db65c5d7a3f0a 100644 --- a/src/core/reference/include/ngraph/runtime/reference/interpolate.hpp +++ b/src/core/reference/include/ngraph/runtime/reference/interpolate.hpp @@ -12,9 +12,11 @@ #include #include +#include "interpolate_pil.hpp" #include "ngraph/coordinate_transform.hpp" #include "ngraph/op/interpolate.hpp" #include "ngraph/shape_util.hpp" +#include "transpose.hpp" namespace ngraph { namespace runtime { @@ -302,6 +304,12 @@ class InterpolateEval final { case InterpolateMode::CUBIC: cubic_func(input_data, out); break; + case InterpolateMode::BILINEAR_PILLOW: + bilinear_pil_func(input_data, out); + break; + case InterpolateMode::BICUBIC_PILLOW: + bicubic_pil_func(input_data, out); + break; default: OPENVINO_THROW("Unsupported interpolation mode"); break; @@ -345,6 +353,10 @@ class InterpolateEval final { /// \param input_data pointer to input data /// \param out pointer to memory block for output data void nearest_func(const T* input_data, T* out); + + void bilinear_pil_func(const T* input_data, T* out); + void bicubic_pil_func(const T* input_data, T* out); + void multidim_pil_func(const T* input_data, T* out, const interpolate_pil::filter& filterp); }; template @@ -564,6 +576,106 @@ void InterpolateEval::cubic_func(const T* input_data, T* out) { NGRAPH_SUPPRESS_DEPRECATED_END } +template +void InterpolateEval::bilinear_pil_func(const T* input_data, T* out) { + struct interpolate_pil::filter bilinear = {interpolate_pil::bilinear_filter, 1.0, m_cube_coeff}; + multidim_pil_func(input_data, out, bilinear); +} + +template +void InterpolateEval::bicubic_pil_func(const T* input_data, T* out) { + struct interpolate_pil::filter bicubic = {interpolate_pil::bicubic_filter, 2.0, m_cube_coeff}; + multidim_pil_func(input_data, out, bicubic); +} + +template +void InterpolateEval::multidim_pil_func(const T* input_data, T* out, const interpolate_pil::filter& filterp) { + OPENVINO_ASSERT(m_axes.size() == 2, "For Pillow based modes exactly two (HW) axes need to be provided."); + + auto h_dim_idx = m_axes[0]; + auto w_dim_idx = m_axes[1]; + auto h_dim_in = m_input_data_shape[h_dim_idx]; + auto w_dim_in = m_input_data_shape[w_dim_idx]; + auto h_dim_out = m_out_shape[h_dim_idx]; + auto w_dim_out = m_out_shape[w_dim_idx]; + auto in_matrix_elem_size = h_dim_in * w_dim_in; + auto out_matrix_elem_size = h_dim_out * w_dim_out; + + auto box = std::vector{0.f, 0.f, static_cast(w_dim_in), static_cast(h_dim_in)}; + + if (shape_size(m_input_data_shape) == in_matrix_elem_size) { + // Input data is 2D or ND with other dimensions equal 1 + interpolate_pil::imaging_resample_inner(input_data, + w_dim_in, + h_dim_in, + w_dim_out, + h_dim_out, + filterp, + box.data(), + out); + } else { + // Flatten other dimensions and interpolate over 2D matrices + std::vector in_transp_axes_order; + for (size_t i = 0; i < m_input_data_shape.size(); ++i) { + if (std::find(m_axes.begin(), m_axes.end(), i) == m_axes.end()) { + in_transp_axes_order.push_back(i); + } + } + in_transp_axes_order.insert(in_transp_axes_order.end(), m_axes.begin(), m_axes.end()); + + Shape transp_input_shape; + Shape transp_output_shape; + for (auto&& axis : in_transp_axes_order) { + transp_input_shape.push_back(m_input_data_shape[axis]); + transp_output_shape.push_back(m_out_shape[axis]); + } + size_t flat_batch_size = + transp_input_shape.size() > 2 + ? shape_size(transp_input_shape.begin(), transp_input_shape.begin() + transp_input_shape.size() - 2) + : 1; + + // Transpose HW dimensions to the end of the tensor shape + std::vector transposed_in(input_data, input_data + shape_size(m_input_data_shape)); + transpose(reinterpret_cast(input_data), + reinterpret_cast(transposed_in.data()), + m_input_data_shape, + sizeof(T), + in_transp_axes_order.data(), + transp_input_shape); + + std::vector transposed_out(shape_size(m_out_shape)); + T* in_matrix_ptr = transposed_in.data(); + T* out_matrix_ptr = transposed_out.data(); + + // Resample each 2D matrix + for (size_t i = 0; i < flat_batch_size; ++i) { + interpolate_pil::imaging_resample_inner(in_matrix_ptr, + w_dim_in, + h_dim_in, + w_dim_out, + h_dim_out, + filterp, + box.data(), + out_matrix_ptr); + in_matrix_ptr += in_matrix_elem_size; + out_matrix_ptr += out_matrix_elem_size; + } + + std::vector out_transp_axes_order(m_out_shape.size() - 2); + std::iota(out_transp_axes_order.begin(), out_transp_axes_order.end(), 0); + out_transp_axes_order.insert(out_transp_axes_order.begin() + h_dim_idx, transp_input_shape.size() - 2); + out_transp_axes_order.insert(out_transp_axes_order.begin() + w_dim_idx, transp_input_shape.size() - 1); + + // Transpose back to the original data dimensions order + transpose(reinterpret_cast(transposed_out.data()), + reinterpret_cast(out), + transp_output_shape, + sizeof(T), + out_transp_axes_order.data(), + m_out_shape); + } +} + template void InterpolateEval::nearest_func(const T* input_data, T* out) { NGRAPH_SUPPRESS_DEPRECATED_START diff --git a/src/core/reference/include/ngraph/runtime/reference/interpolate_pil.hpp b/src/core/reference/include/ngraph/runtime/reference/interpolate_pil.hpp new file mode 100644 index 00000000000000..93a2b9776f5bbc --- /dev/null +++ b/src/core/reference/include/ngraph/runtime/reference/interpolate_pil.hpp @@ -0,0 +1,320 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// The implementation for BILINEAR_PILLOW and BICUBIC_PILLOW is based on the +// Pillow library code from: +// https://github.com/python-pillow/Pillow/blob/9.4.0/src/libImaging/Resample.c + +// The Python Imaging Library (PIL) is + +// Copyright © 1997-2011 by Secret Labs AB +// Copyright © 1995-2011 by Fredrik Lundh + +// Pillow is the friendly PIL fork. It is + +// Copyright © 2010-2023 by Jeffrey A. Clark (Alex) and contributors. + +// Like PIL, Pillow is licensed under the open source HPND License: + +// By obtaining, using, and/or copying this software and/or its associated +// documentation, you agree that you have read, understood, and will comply +// with the following terms and conditions: + +// Permission to use, copy, modify and distribute this software and its +// documentation for any purpose and without fee is hereby granted, +// provided that the above copyright notice appears in all copies, and that +// both that copyright notice and this permission notice appear in supporting +// documentation, and that the name of Secret Labs AB or the author not be +// used in advertising or publicity pertaining to distribution of the software +// without specific, written prior permission. + +// SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +// SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +// IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, +// INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#pragma once + +#include +#include + +#include "ngraph/op/interpolate.hpp" +#include "ngraph/shape_util.hpp" + +namespace ngraph { +namespace runtime { +namespace reference { +namespace interpolate_pil { + +struct filter { + double (*filter)(double x, double coeff_a); + double support; + double coeff_a; +}; + +template +T_out round_up(T_in x) { + return (T_out)(x >= 0.0 ? x + 0.5F : x - 0.5F); +} + +template +T_out clip(const T_in& x, + const T_out& min = std::numeric_limits::min(), + const T_out& max = std::numeric_limits::max()) { + return T_out(std::max(T_in(min), std::min(x, T_in(max)))); +} + +static inline double bilinear_filter(double x, double) { + if (x < 0.0) { + x = -x; + } + if (x < 1.0) { + return 1.0 - x; + } + return 0.0; +} + +static inline double bicubic_filter(double x, double a) { + if (x < 0.0) { + x = -x; + } + if (x < 1.0) { + return ((a + 2.0) * x - (a + 3.0)) * x * x + 1; + } + if (x < 2.0) { + return (((x - 5) * x + 8) * x - 4) * a; + } + return 0.0; +} + +static inline int precompute_coeffs(int in_size, + float in0, + float in1, + int out_size, + const filter& filterp, + std::vector& bounds, + std::vector& kk) { + double support, scale, filterscale; + double center, ww, ss; + int xx, x, ksize, xmin, xmax; + /* prepare for horizontal stretch */ + filterscale = scale = (double)(in1 - in0) / out_size; + if (filterscale < 1.0) { + filterscale = 1.0; + } + + /* determine support size (length of resampling filter) */ + support = filterp.support * filterscale; + + /* maximum number of coeffs */ + ksize = (int)ceil(support) * 2 + 1; + + /* coefficient buffer */ + kk.resize(out_size * ksize); + bounds.resize(out_size * 2); + + for (xx = 0; xx < out_size; xx++) { + center = in0 + (xx + 0.5) * scale; + ww = 0.0; + ss = 1.0 / filterscale; + // Round the value + xmin = (int)(center - support + 0.5); + if (xmin < 0) { + xmin = 0; + } + // Round the value + xmax = (int)(center + support + 0.5); + if (xmax > in_size) { + xmax = in_size; + } + xmax -= xmin; + double* k = &kk[xx * ksize]; + for (x = 0; x < xmax; x++) { + double w = filterp.filter((x + xmin - center + 0.5) * ss, filterp.coeff_a); + k[x] = w; + ww += w; + } + for (x = 0; x < xmax; x++) { + if (ww != 0.0) { + k[x] /= ww; + } + } + // Remaining values should stay empty if they are used despite of xmax. + for (; x < ksize; x++) { + k[x] = 0; + } + bounds[xx * 2 + 0] = xmin; + bounds[xx * 2 + 1] = xmax; + } + return ksize; +} + +template +void imaging_resample_horizontal(T* im_out, + Shape im_out_shape, + const T* im_in, + Shape im_in_shape, + int offset, + int ksize, + std::vector& bounds, + std::vector& kk) { + double ss; + int x, xmin, xmax; + double* k; + + for (size_t yy = 0; yy < im_out_shape[0]; yy++) { + for (size_t xx = 0; xx < im_out_shape[1]; xx++) { + xmin = bounds[xx * 2 + 0]; + xmax = bounds[xx * 2 + 1]; + k = &kk[xx * ksize]; + ss = 0.0; + for (x = 0; x < xmax; x++) { + size_t in_idx = (yy + offset) * im_in_shape[1] + (x + xmin); + ss += im_in[in_idx] * k[x]; + } + size_t out_idx = yy * im_out_shape[1] + xx; + if (std::is_integral()) { + im_out[out_idx] = T(clip(round_up(ss))); + } else { + im_out[out_idx] = T(ss); + } + } + } +} + +template +void imaging_resample_vertical(T* im_out, + Shape im_out_shape, + const T* im_in, + Shape im_in_shape, + int offset, + int ksize, + std::vector& bounds, + std::vector& kk) { + double ss; + int y, ymin, ymax; + double* k; + + for (size_t yy = 0; yy < im_out_shape[0]; yy++) { + ymin = bounds[yy * 2 + 0]; + ymax = bounds[yy * 2 + 1]; + k = &kk[yy * ksize]; + for (size_t xx = 0; xx < im_out_shape[1]; xx++) { + ss = 0.0; + for (y = 0; y < ymax; y++) { + size_t in_idx = (y + ymin) * im_in_shape[1] + xx; + ss += im_in[in_idx] * k[y]; + } + size_t out_idx = yy * im_out_shape[1] + xx; + if (std::is_integral()) { + im_out[out_idx] = T(clip(round_up(ss))); + } else { + im_out[out_idx] = T(ss); + } + } + } +} + +template +void imaging_resample_inner(const T* im_in, + size_t im_in_xsize, + size_t im_in_ysize, + size_t xsize, + size_t ysize, + const filter& filterp, + float* box, + T* im_out) { + int ybox_first, ybox_last; + int ksize_horiz, ksize_vert; + + std::vector bounds_horiz; + std::vector bounds_vert; + std::vector kk_horiz; + std::vector kk_vert; + + auto need_horizontal = xsize != im_in_xsize || bool(box[0]) || box[2] != xsize; + auto need_vertical = ysize != im_in_ysize || bool(box[1]) || box[3] != ysize; + + ksize_horiz = precompute_coeffs(static_cast(im_in_xsize), + box[0], + box[2], + static_cast(xsize), + filterp, + bounds_horiz, + kk_horiz); + ksize_vert = precompute_coeffs(static_cast(im_in_ysize), + box[1], + box[3], + static_cast(ysize), + filterp, + bounds_vert, + kk_vert); + + // First used row in the source image + ybox_first = bounds_vert[0]; + // Last used row in the source image + ybox_last = bounds_vert[ysize * 2 - 2] + bounds_vert[ysize * 2 - 1]; + + size_t im_temp_ysize = (ybox_last - ybox_first); + auto im_temp_elem_count = im_temp_ysize * xsize; + auto im_temp = std::vector(im_temp_elem_count, 0); + + /* two-pass resize, horizontal pass */ + if (need_horizontal) { + // Shift bounds for vertical pass + for (size_t i = 0; i < ysize; i++) { + bounds_vert[i * 2] -= ybox_first; + } + + if (im_temp.size() > 0) { + imaging_resample_horizontal(im_temp.data(), + Shape{im_temp_ysize, xsize}, + im_in, + Shape{im_in_ysize, im_in_xsize}, + ybox_first, + ksize_horiz, + bounds_horiz, + kk_horiz); + } + } + + /* vertical pass */ + if (need_vertical) { + /* im_in can be the original image or horizontally resampled one */ + if (need_horizontal) { + imaging_resample_vertical(im_out, + Shape{ysize, xsize}, + im_temp.data(), + Shape{im_temp_ysize, xsize}, + 0, + ksize_vert, + bounds_vert, + kk_vert); + } else { + imaging_resample_vertical(im_out, + Shape{ysize, xsize}, + im_in, + Shape{im_in_ysize, im_in_xsize}, + 0, + ksize_vert, + bounds_vert, + kk_vert); + } + } + /* none of the previous steps are performed, copying */ + if (!need_horizontal && !need_vertical) { + std::copy(im_in, im_in + (im_in_xsize * im_in_ysize), im_out); + } else if (need_horizontal && !need_vertical) { + std::copy(im_temp.begin(), im_temp.end(), im_out); + } + + return; +} + +} // namespace interpolate_pil +} // namespace reference +} // namespace runtime +} // namespace ngraph diff --git a/src/core/reference/src/runtime/reference/convert.cpp b/src/core/reference/src/runtime/reference/convert.cpp index 4c64ce82f43880..3e43e779a3637f 100644 --- a/src/core/reference/src/runtime/reference/convert.cpp +++ b/src/core/reference/src/runtime/reference/convert.cpp @@ -43,6 +43,16 @@ void jit_convert_vec(jit::Generator& gen, const Xbyak::RegExp& s gen.vmovups(gen.yword[dst], f32vec); } +template <> +void jit_convert_vec(jit::Generator& gen, const Xbyak::RegExp& src, const Xbyak::RegExp& dst) { + auto f16vec = gen.xmm3; + auto f32vec = gen.ymm4; + + gen.vmovups(f32vec, gen.yword[src]); + gen.vcvtps2ph(f16vec, f32vec, 0); + gen.vmovdqu(gen.xword[dst], f16vec); +} + template <> void jit_convert_vec_prepare(jit::Generator& gen) { auto order = gen.ymm1; @@ -206,6 +216,11 @@ void convert(const float16* arg, float* out, size_t count) { convert_impl(arg, out, count); } +template <> +void convert(const float* arg, float16* out, size_t count) { + convert_impl(arg, out, count); +} + template <> void convert(const float* arg, int8_t* out, size_t count) { convert_impl(arg, out, count); diff --git a/src/core/reference/src/runtime/reference/einsum.cpp b/src/core/reference/src/runtime/reference/einsum.cpp index e6b19719708e1a..8d168c91a83177 100644 --- a/src/core/reference/src/runtime/reference/einsum.cpp +++ b/src/core/reference/src/runtime/reference/einsum.cpp @@ -124,16 +124,16 @@ std::unordered_map compute_label_dim_map(const Ran if (label == ellipsis) { ov::TensorLabel label_dims; for (size_t ind = 0; ind < num_broadcasted_dims; ++ind) { - label_dims.push_back(current_dim + ind); + label_dims.push_back(static_cast(current_dim + ind)); } resulted_map[label] = label_dims; current_dim += num_broadcasted_dims; } else if (resulted_map.find(label) != resulted_map.end()) { - resulted_map[label].push_back(current_dim); + resulted_map[label].push_back(static_cast(current_dim)); ++current_dim; } else { ov::TensorLabel label_dims; - label_dims.push_back(current_dim); + label_dims.push_back(static_cast(current_dim)); resulted_map[label] = label_dims; ++current_dim; } diff --git a/src/core/reference/src/runtime/reference/generate_proposal.cpp b/src/core/reference/src/runtime/reference/generate_proposal.cpp index 3c6eb774520779..3aa82a11eba380 100644 --- a/src/core/reference/src/runtime/reference/generate_proposal.cpp +++ b/src/core/reference/src/runtime/reference/generate_proposal.cpp @@ -325,11 +325,11 @@ void generate_proposals(const std::vector& im_info, std::vector& output_scores, std::vector& num_rois) { const auto im_info_size = - std::accumulate(im_info_shape.begin() + 1, im_info_shape.end(), 1, std::multiplies()); + std::accumulate(im_info_shape.begin() + 1, im_info_shape.end(), size_t(1), std::multiplies()); const auto deltas_size = - std::accumulate(deltas_shape.begin() + 1, deltas_shape.end(), 1, std::multiplies()); + std::accumulate(deltas_shape.begin() + 1, deltas_shape.end(), size_t(1), std::multiplies()); const auto scores_size = - std::accumulate(scores_shape.begin() + 1, scores_shape.end(), 1, std::multiplies()); + std::accumulate(scores_shape.begin() + 1, scores_shape.end(), size_t(1), std::multiplies()); for (size_t i = 0; i < im_info_shape[0]; i++) { std::vector cur_im_info(im_info.begin() + i * im_info_size, im_info.begin() + i * im_info_size + im_info_size); diff --git a/src/core/shape_inference/CMakeLists.txt b/src/core/shape_inference/CMakeLists.txt index c3bd1a0f8ae53c..958833ccaa902d 100644 --- a/src/core/shape_inference/CMakeLists.txt +++ b/src/core/shape_inference/CMakeLists.txt @@ -4,10 +4,6 @@ set(TARGET_NAME "ov_shape_inference") -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - ie_add_compiler_flags(/wd4018) -endif() - file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) file(GLOB_RECURSE PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp) diff --git a/src/core/shape_inference/include/avg_pool_shape_inference.hpp b/src/core/shape_inference/include/avg_pool_shape_inference.hpp index 577d014a6c9fda..fe6df898366db4 100644 --- a/src/core/shape_inference/include/avg_pool_shape_inference.hpp +++ b/src/core/shape_inference/include/avg_pool_shape_inference.hpp @@ -36,23 +36,23 @@ inline void valid_dilated_kernel_with_padding(const v1::AvgPool* op, } // namespace pooling namespace v1 { - -template -std::vector shape_infer(const AvgPool* op, const std::vector& input_shapes) { +template +std::vector shape_infer(const AvgPool* op, + const std::vector& input_shapes, + TContainer& pads_begin, + TContainer& pads_end, + const std::map& constant_data = {}) { NODE_VALIDATION_CHECK(op, input_shapes.size() == 1); const auto& data_shape = input_shapes[0]; - const auto dilations = Strides(op->get_kernel().size(), 1); - pooling::update_and_validate_attributes(const_cast(op), data_shape, dilations); - - auto output_shape = pooling::out_shape_infer(op, data_shape, dilations); - return {output_shape}; -} + auto num_spatial = dilations.size(); + pooling::resize_empty_padding(num_spatial, pads_begin, pads_end); + pooling::validate::padding(op, pads_begin, pads_end); + pooling::validate::attributes(op, data_shape, dilations); + pooling::apply_padding(op, data_shape, dilations, pads_begin, pads_end); -template -void shape_infer(const AvgPool* op, const std::vector& input_shapes, std::vector& output_shapes) { - output_shapes = shape_infer(op, input_shapes); + return {pooling::out_shape_infer(op, data_shape, pads_begin, pads_end, dilations)}; } } // namespace v1 } // namespace op diff --git a/src/core/shape_inference/include/binary_convolution_shape_inference.hpp b/src/core/shape_inference/include/binary_convolution_shape_inference.hpp new file mode 100644 index 00000000000000..a5d8bc0ac273f8 --- /dev/null +++ b/src/core/shape_inference/include/binary_convolution_shape_inference.hpp @@ -0,0 +1,25 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "openvino/op/binary_convolution.hpp" +#include "utils.hpp" + +namespace ov { +namespace op { +namespace convolution { +namespace validate { +/** + * @brief Specific check of data shape for binary convolution data shape must be rank 4. + * + * The shape_infer is same as for Convolution operator except this check. @see convolution_shape_inference.hpp + */ +template +void data_shape(const v1::BinaryConvolution* op, const TShape& data_shape) { + NODE_VALIDATION_CHECK(op, data_shape.rank().compatible(4), "Expected 4D for the input. Got: ", data_shape); +} +} // namespace validate +} // namespace convolution +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/convolution_backprop_shape_inference.hpp b/src/core/shape_inference/include/convolution_backprop_shape_inference.hpp new file mode 100644 index 00000000000000..b4585434dc2e2c --- /dev/null +++ b/src/core/shape_inference/include/convolution_backprop_shape_inference.hpp @@ -0,0 +1,80 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "convolution_backprop_shape_inference_util.hpp" +#include "convolution_shape_inference_util.hpp" +#include "openvino/op/convolution.hpp" +#include "utils.hpp" + +namespace ov { +namespace op { +namespace v1 { +template +std::vector shape_infer(const ConvolutionBackpropData* op, + const std::vector& input_shapes, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + const std::map& constant_data = {}) { + const auto inputs_count = input_shapes.size(); + const auto has_spatial_shape = inputs_count >= 3; + NODE_VALIDATION_CHECK(op, inputs_count >= 2); + using namespace ov::util; + + TShape out_spatial_shape; + if (has_spatial_shape) { + const auto& spatial_shape = input_shapes[2]; + NODE_VALIDATION_CHECK(op, + spatial_shape.rank().compatible(1), + "Input delivering output shape must have rank 1."); + + if (!get_data_as_shape(2, op, out_spatial_shape, constant_data)) { + if (spatial_shape.is_static()) { + out_spatial_shape.resize(spatial_shape[0].get_length()); + } else { + out_spatial_shape = PartialShape::dynamic(); + } + } + } + + const auto num_spatial = convolution::calculate_num_spatial(op, input_shapes, out_spatial_shape); + + TShape output_shape; + if (num_spatial != convolution::num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + + NODE_VALIDATION_CHECK( + op, + !has_spatial_shape || out_spatial_shape.rank().is_dynamic() || out_spatial_shape.size() == num_spatial, + "Output shape should be defined for all and only spatial dimensions."); + + convolution::resize_empty_padding(num_spatial, pads_begin, pads_end); + convolution::validate::filter_shape(op, filters_shape, data_shape); + if (is_attr_validation_required(op)) { + convolution::validate::data_shape(op, data_shape); + convolution::validate::common_attributes(op, num_spatial, pads_begin, pads_end); + } + convolution::apply_padding(op, input_shapes, out_spatial_shape, pads_begin, pads_end); + + output_shape.reserve(util::spatial_dim_offset + num_spatial); + output_shape.emplace_back(data_shape.rank().is_static() ? data_shape[0] : dim::inf_bound); + output_shape.emplace_back(filters_shape.rank().is_static() ? filters_shape[1] : dim::inf_bound); + + if (has_spatial_shape) { + output_shape.insert(output_shape.end(), + std::make_move_iterator(out_spatial_shape.begin()), + std::make_move_iterator(out_spatial_shape.end())); + } else { + convolution::append_spatial_shape(op, data_shape, filters_shape, pads_begin, pads_end, output_shape); + } + } else { + output_shape = PartialShape::dynamic(); + } + + return {output_shape}; +} +} // namespace v1 +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/convolution_backprop_shape_inference_util.hpp b/src/core/shape_inference/include/convolution_backprop_shape_inference_util.hpp new file mode 100644 index 00000000000000..eaa8b86748a7a6 --- /dev/null +++ b/src/core/shape_inference/include/convolution_backprop_shape_inference_util.hpp @@ -0,0 +1,191 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "convolution_shape_inference_util.hpp" +#include "openvino/op/util/convolution_backprop_base.hpp" + +namespace ov { +namespace op { +namespace convolution { +namespace validate { +template +void filter_shape(const ov::op::util::ConvolutionBackPropBase* op, + const TShape& filters_shape, + const TShape& data_shape) { + const auto& data_rank = data_shape.rank(); + const auto& filters_rank = filters_shape.rank(); + + NODE_VALIDATION_CHECK(op, + data_rank.compatible(filters_rank), + "Data batch and filters rank do not match (data batch shape: ", + data_shape, + ", filters shape: ", + filters_shape, + ")."); + + NODE_VALIDATION_CHECK( + op, + data_rank.is_dynamic() || filters_rank.is_dynamic() || data_shape[1].compatible(filters_shape[0]), + "Data batch channel count (", + data_shape[1], + ") does not match filter input channel count (", + filters_shape[0], + ")."); +} +} // namespace validate + +template ::value>::type* = nullptr> +size_t calculate_num_spatial(const TOp* op, const std::vector& input_shapes, const TShape& out_spatial_shape) { + NODE_VALIDATION_CHECK(op, input_shapes.size() > 1); + + auto num_spatial = util::get_num_spatial(op); + if (num_spatial == num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + num_spatial = util::num_spatial_from_shapes(data_shape, filters_shape, filter_non_spatial_dims_count()); + } + + if (num_spatial == num_spatial_undefined && out_spatial_shape.rank().is_static() && out_spatial_shape.size() > 0) { + num_spatial = out_spatial_shape.size(); + } + + if (num_spatial == num_spatial_undefined) { + num_spatial = num_spatial_from_attr(op); + } + return num_spatial; +} + +/** + * @brief Apply auto padding for backward convolution. + * + * The auto padding can be applied only if inputs and attributes of operator are validated. + * The input shapes must have got static ranks. + * + * @param op Pointer to convolution operator. + * @param data_shape Input data shape (must be static rank). + * @param filters_shape Input filter shape (must be static rank). + * @param out_spatial_shape Reference to input with out spatial shape. + * @param pads_begin Iterator to begin of pads begin. + * @param pads_end Iterator to begin of pads end. + */ +template +void apply_auto_pad(const TOp* op, + const TShape& data_shape, + const TShape& filters_shape, + const TShape& out_spatial_shape, + TIter pads_begin, + TIter pads_end) { + const auto& strides = op->get_strides(); + const auto& dilations = op->get_dilations(); + const auto& out_padding = op->get_output_padding(); + + const auto num_spatial = strides.size(); + auto data_dim = data_shape.cend() - num_spatial; + auto filter_dim = filters_shape.cend() - num_spatial; + + const auto padding_swap = op->get_auto_pad() == PadType::SAME_UPPER; + auto& pad_b = padding_swap ? pads_end : pads_begin; + auto& pad_e = padding_swap ? pads_begin : pads_end; + + for (size_t i = 0; i < num_spatial; ++i, ++pad_b, ++pad_e, ++data_dim, ++filter_dim) { + using namespace ov::util; + if (data_dim->is_static() && filter_dim->is_static() && out_spatial_shape[i].is_static()) { + const auto dilated_filter = dim::dilated(*filter_dim, dilations[i]); + const auto dim_len = static_cast(data_dim->get_length() - 1); + const auto padding = std::max( + dim_len * strides[i] + dilated_filter.get_length() - out_spatial_shape[i].get_length() + out_padding[i], + 0); + + *pad_b = padding / 2; + *pad_e = padding - *pad_b; + } else { + *pad_b = 0; + *pad_e = 0; + } + } +} + +/** + * @brief Apply auto padding for back propagation convolutions. + * + * @tparam TShape Shape type. + * @param op Pointer to back propagation convolution operator. + * @param data_shape Input data shape. + * @param filters_shape Input filter shape. + * @param out_spatial_shape Input output spatial shape. + */ +template +void apply_padding(const util::ConvolutionBackPropBase* op, + const std::vector& input_shapes, + const TShape& out_spatial_shape, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + + // apply padding if required + if (input_shapes.size() >= 3 && convolution::is_auto_pad(op) && data_shape.rank().is_static() && + filters_shape.rank().is_static()) { + convolution::apply_auto_pad(op, + data_shape, + filters_shape, + out_spatial_shape, + pads_begin.begin(), + pads_end.begin()); + } else if (convolution::is_auto_pad(op) || op->get_auto_pad() == op::PadType::VALID) { + std::fill(pads_begin.begin(), pads_begin.end(), 0); + std::fill(pads_end.begin(), pads_end.end(), 0); + } else if (op->get_auto_pad() == op::PadType::EXPLICIT) { + std::copy(op->get_pads_begin().begin(), op->get_pads_begin().end(), pads_begin.begin()); + std::copy(op->get_pads_end().begin(), op->get_pads_end().end(), pads_end.begin()); + } +} + +/** + * @brief Append spatial dimension at end of output shape of back propagation convolution. + * + * @tparam TOp Back propagation convolution operator type. + * @tparam TShape Type of shape. + * @param op Pointer to operator. + * @param data_shape Input data shape. + * @param filters_shape Input filter shape. + * @param out_shape Output shape to append spatial dimensions. + */ +template ::value>::type* = nullptr> +void append_spatial_shape(const TOp* op, + const TShape& data_shape, + const TShape& filters_shape, + const TContainer& pads_begin, + const TContainer& pads_end, + TShape& out_shape) { + using namespace ov::util; + + const auto& strides = op->get_strides(); + const auto& dilations = op->get_dilations(); + const auto& output_padding = op->get_output_padding(); + + const auto spatial_num = strides.size(); + + const auto& d_shape = data_shape.rank().is_static() ? data_shape : PartialShape::dynamic(spatial_num); + auto data_dim = d_shape.cend() - spatial_num; + + const auto& f_shape = filters_shape.rank().is_static() ? filters_shape : PartialShape::dynamic(spatial_num); + auto filters_dim = f_shape.cend() - spatial_num; + + for (size_t i = 0; i < spatial_num; ++i, ++data_dim, ++filters_dim) { + auto dim = (*data_dim - 1) * strides[i]; + dim += dim::dilated(*filters_dim, dilations[i]); + out_shape.push_back(dim::padded(dim, output_padding[i] - pads_begin[i] - pads_end[i])); + } +} +} // namespace convolution +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/convolution_shape_inference.hpp b/src/core/shape_inference/include/convolution_shape_inference.hpp index 9717fafbd4f2f0..b1bdeb4317e93b 100644 --- a/src/core/shape_inference/include/convolution_shape_inference.hpp +++ b/src/core/shape_inference/include/convolution_shape_inference.hpp @@ -3,736 +3,49 @@ // #pragma once -#include -#include - +#include "convolution_shape_inference_util.hpp" +#include "openvino/op/convolution.hpp" #include "utils.hpp" namespace ov { namespace op { namespace v1 { - -template -int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - int64_t num_spatial = op->m_num_spatial; - if (num_spatial == -1) { - const auto& input_rank = input_shape.rank(); - const auto& filters_rank = filters_shape.rank(); - - if (input_rank.is_static()) - num_spatial = input_rank.get_length() - num_non_spatial_data_dims; - if (filters_rank.is_static()) - num_spatial = filters_rank.get_length() - num_non_spatial_filter_dims; - - if (const auto& size = op->m_dilations.size()) { - NODE_VALIDATION_CHECK(op, - num_spatial == -1 || num_spatial == static_cast(size), - "Dilations should be defined for all and only spatial dimensions."); - num_spatial = static_cast(size); - } - if (const auto& size = op->m_strides.size()) { - NODE_VALIDATION_CHECK(op, - num_spatial == -1 || num_spatial == static_cast(size), - "Strides should be defined for all and only spatial dimensions."); - num_spatial = static_cast(size); - } - if (const auto& size = op->m_pads_begin.size()) { - NODE_VALIDATION_CHECK(op, - num_spatial == -1 || num_spatial == static_cast(size), - "Pads begin should be defined for all and only spatial dimensions."); - num_spatial = static_cast(size); - } - if (const auto& size = op->m_pads_end.size()) { - NODE_VALIDATION_CHECK(op, - num_spatial == -1 || num_spatial == static_cast(size), - "Pads end should be defined for all and only spatial dimensions."); - num_spatial = static_cast(size); - } - } - return num_spatial; -} - -template -int64_t calculate_num_spatial(const ConvType* op, - const ShapeType& input_shape, - const ShapeType& filters_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - return calculate_num_spatial(op, - input_shape.to_partial_shape(), - filters_shape.to_partial_shape(), - num_non_spatial_data_dims, - num_non_spatial_filter_dims); -} - -template -void update_and_validate_attributes(ConvType* op, int64_t num_spatial) { - if (num_spatial != -1) { - auto& strides = op->m_strides; - auto& dilations = op->m_dilations; - auto& pad_begin = op->m_pads_begin; - auto& pad_end = op->m_pads_end; - auto& auto_pad = op->m_auto_pad; - - if (strides.empty()) - strides = Strides(num_spatial, 1); - if (dilations.empty()) - dilations = Strides(num_spatial, 1); - if (pad_begin.empty() || auto_pad == op::PadType::VALID) - pad_begin = CoordinateDiff(num_spatial, 0); - if (pad_end.empty() || auto_pad == op::PadType::VALID) - pad_end = CoordinateDiff(num_spatial, 0); - - NODE_VALIDATION_CHECK(op, - static_cast(strides.size()) == num_spatial, - "Strides should be defined for all and only spatial dimensions.."); - NODE_VALIDATION_CHECK(op, - static_cast(dilations.size()) == num_spatial, - "Dilations should be defined for all and only spatial dimensions.."); - NODE_VALIDATION_CHECK(op, - static_cast(pad_begin.size()) == num_spatial && - static_cast(pad_end.size()) == num_spatial, - "Pads should be defined for all and only spatial dimensions.."); - NODE_VALIDATION_CHECK(op, - std::all_of(dilations.begin(), - dilations.end(), - [](const size_t& i) { - return i > 0; - }), - "Filter dilation (", - dilations, - ") has zero dimension."); - NODE_VALIDATION_CHECK(op, - std::all_of(strides.begin(), - strides.end(), - [](const size_t& i) { - return i > 0; - }), - "Filter strides (", - strides, - ") has zero dimension."); - } else if (op->m_num_spatial != -1) { - update_and_validate_attributes(op, op->m_num_spatial); - } -} - -template -inline bool dynamic_check(const int64_t& num_spatial) { - OPENVINO_ASSERT(num_spatial != -1, - "Convolution shape inference doesn't have enough information for static shape calculation"); - return true; -} - -template <> -inline bool dynamic_check(const int64_t& num_spatial) { - return num_spatial != -1; -} - -template -bool resolve_auto_pad_for_shape(const ConvType* op, - CoordinateDiff& pads_begin, - CoordinateDiff& pads_end, - const std::vector& input_shapes, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - const auto& auto_pad = op->m_auto_pad; - if (auto_pad != op::PadType::SAME_UPPER && auto_pad != op::PadType::SAME_LOWER) { - pads_begin = op->m_pads_begin; - pads_end = op->m_pads_end; - return true; - } - - auto input_shape = input_shapes[0]; - auto filters_shape = input_shapes[1]; - - const auto num_spatial = op->m_num_spatial != -1 ? op->m_num_spatial - : calculate_num_spatial(op, - input_shape, - filters_shape, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); - if (!dynamic_check(num_spatial)) - return false; - - if (input_shape.rank().is_dynamic()) - input_shape.resize(num_spatial + num_non_spatial_data_dims); - if (filters_shape.rank().is_dynamic()) - filters_shape.resize(num_spatial + num_non_spatial_filter_dims); - - const auto& strides = op->m_strides; - const auto& dilations = op->m_dilations; - pads_begin.resize(num_spatial); - pads_end.resize(num_spatial); - - bool status = true; - for (int64_t i = 0; i < num_spatial; ++i) { - const auto& input_dim = input_shape[i + num_non_spatial_data_dims]; - const auto& filters_dim = filters_shape[i + num_non_spatial_filter_dims]; - if (input_dim.is_static() && filters_dim.is_static()) { - const int64_t& window_dilated_dim = (filters_dim.get_length() - 1) * dilations[i] + 1; - NODE_VALIDATION_CHECK(op, - window_dilated_dim > 0, - "Window after dilation has dimension less than 1 (dim: ", - window_dilated_dim, - ") at axis ", - i, - "."); - - const int64_t& image_size = input_dim.get_length(); - const int64_t& filter_stride = strides[i]; - const int64_t& output_size = (image_size + filter_stride - 1) / filter_stride; - - const int64_t& tmp = (output_size - 1) * filter_stride + window_dilated_dim; - const int64_t& padding_needed = tmp > image_size ? tmp - image_size : 0; - - const size_t& padding_lhs = static_cast(padding_needed / 2); - const size_t& padding_rhs = static_cast(padding_needed - padding_lhs); - - pads_begin[i] = auto_pad == op::PadType::SAME_UPPER ? padding_lhs : padding_rhs; - pads_end[i] = auto_pad == op::PadType::SAME_UPPER ? padding_rhs : padding_lhs; - } else { - status = false; - } - } - return status; -} - -template -void divide_ceil(const DimType& dividend, const typename DimType::value_type& divisor, DimType& quotient) { - OPENVINO_ASSERT(divisor >= 0, "divisor must be greater than 0"); - if (dividend.get_max_length() == -1) { - quotient = -1; - } else { - auto lb = static_cast(ceil(1. * dividend.get_min_length() / divisor)); - auto ub = static_cast(ceil(1. * dividend.get_max_length() / divisor)); - quotient = DimType(lb, ub); - } -} - -template -void divide_floor(const DimType& dividend, const typename DimType::value_type& divisor, DimType& quotient) { - OPENVINO_ASSERT(divisor >= 0, "divisor must be greater than 0"); - if (dividend.get_max_length() == -1) { - quotient = -1; - } else { - auto lb = static_cast(floor(1. * dividend.get_min_length() / divisor)); - auto ub = static_cast(floor(1. * dividend.get_max_length() / divisor)); - quotient = DimType(lb, ub); - } -} - -template -void calculate_output_spatial_dims_for_convolution(const ConvType* op, - const ShapeType& input_shape, - const ShapeType& filters_shape, - ShapeType& output_shape, - const int64_t& num_spatial, - const Strides& strides, - const Strides& dilations, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - bool auto_pad = op->get_auto_pad() == op::PadType::SAME_UPPER || op->get_auto_pad() == op::PadType::SAME_LOWER; - for (int64_t i = 0; i < num_spatial; ++i) { - auto input_dim = input_shape[i + num_non_spatial_data_dims]; - if (auto_pad) { - divide_ceil(input_dim, strides[i], output_shape[i + num_non_spatial_data_dims]); - continue; - } - const auto& filters_dim = filters_shape[i + num_non_spatial_filter_dims]; - const auto& window_dilated_dim = (filters_dim - 1) * dilations[i] + 1; - const auto& data_padded_dilated_dim = input_dim + pads_begin[i] + pads_end[i]; - if (input_dim.is_static() && filters_dim.is_static()) { - NODE_VALIDATION_CHECK(op, - window_dilated_dim.get_length() > 0, - "Window after dilation has dimension less than 1 (dim: ", - window_dilated_dim, - ") at axis ", - i, - "."); - NODE_VALIDATION_CHECK(op, - window_dilated_dim.get_length() <= data_padded_dilated_dim.get_length(), - "Window after dilation has dimension (dim: ", - window_dilated_dim, - ") larger than the data shape after padding (dim: ", - data_padded_dilated_dim, - ") at axis ", - i, - "."); - } - divide_floor(data_padded_dilated_dim - window_dilated_dim, - strides[i], - output_shape[i + num_non_spatial_data_dims]); - output_shape[i + num_non_spatial_data_dims] += 1; - } -} - -template -void shape_infer(const Convolution* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const std::vector& input_shapes, - std::vector& output_shapes) { - NODE_VALIDATION_CHECK(op, input_shapes.size() == 2 && output_shapes.size() == 1); - constexpr size_t num_non_spatial_data_dims = 2, num_non_spatial_filter_dims = 2; - auto input_shape = input_shapes[0], filters_shape = input_shapes[1]; - - const auto num_spatial = op->m_num_spatial != -1 ? op->m_num_spatial - : calculate_num_spatial(op, - input_shape, - filters_shape, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); - NODE_VALIDATION_CHECK(op, - num_spatial != -1, - "Convolution shape_infer should be provided with correct num_spatial attribute"); - - if (input_shape.rank().is_dynamic()) - input_shape.resize(num_spatial + 2); - if (filters_shape.rank().is_dynamic()) - filters_shape.resize(num_spatial + 2); - - NODE_VALIDATION_CHECK( - op, - (static_cast(input_shape.size()) == static_cast(num_spatial + num_non_spatial_data_dims)) && - (static_cast(filters_shape.size()) == - static_cast(num_spatial + num_non_spatial_filter_dims)), - "Data batch and filters rank do not match (data batch shape: ", - input_shape, - ", filters shape: ", - filters_shape, - ")."); - - // ranks are originally static or aligned with num_spatial, attributes assumed to be valid - auto& output_shape = output_shapes[0]; - output_shape.resize(num_spatial + num_non_spatial_data_dims); - output_shape[0] = input_shape[0]; - output_shape[1] = filters_shape[0]; - - NODE_VALIDATION_CHECK(op, - input_shape[1].compatible(filters_shape[1]), - "Data batch channel count (", - input_shape[1], - ") does not match filter input ", - "channel count (", - filters_shape[1], - ")."); - - calculate_output_spatial_dims_for_convolution(op, - input_shape, - filters_shape, - output_shape, - num_spatial, - op->m_strides, - op->m_dilations, - pads_begin, - pads_end, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); -} - -template -void shape_infer(const GroupConvolution* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const std::vector& input_shapes, - std::vector& output_shapes) { - NODE_VALIDATION_CHECK(op, input_shapes.size() == 2 && output_shapes.size() == 1); - auto input_shape = input_shapes[0], filters_shape = input_shapes[1]; - constexpr size_t num_non_spatial_data_dims = 2, num_non_spatial_filter_dims = 3; - - const auto num_spatial = op->m_num_spatial != -1 ? op->m_num_spatial - : calculate_num_spatial(op, - input_shape, - filters_shape, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); - NODE_VALIDATION_CHECK(op, - num_spatial != -1, - "GroupConvolution shape_infer should be provided with correct num_spatial attribute"); - - if (input_shape.rank().is_dynamic()) - input_shape.resize(num_spatial + num_non_spatial_data_dims); - if (filters_shape.rank().is_dynamic()) - filters_shape.resize(num_spatial + num_non_spatial_filter_dims); - - NODE_VALIDATION_CHECK( - op, - (static_cast(input_shape.size()) == static_cast((num_spatial + num_non_spatial_data_dims))) && - (static_cast(filters_shape.size()) == - static_cast((num_spatial + num_non_spatial_filter_dims))), - "Data batch and filters rank do not match (data batch shape: ", - input_shape, - ", filters shape: ", - filters_shape, - ")."); - - // ranks are originally static or aligned with num_spatial, attributes assumed to be valid - auto& output_shape = output_shapes[0]; - output_shape.resize(num_spatial + num_non_spatial_data_dims); - output_shape[0] = input_shape[0]; - - auto groups = filters_shape[0]; - if (groups.is_dynamic()) { - // [N, GROUPS * C_IN, ...] x [GROUPS, C_OUT, C_IN, ...] = [N, GROUPS * C_OUT, ...] - if (input_shape[1].is_static() && filters_shape[2].is_static()) { - using DimensionType = typename std::iterator_traits::value_type; - auto n_data_channels = input_shape[1].get_length(); - auto input_channels = filters_shape[2].get_length(); - NODE_VALIDATION_CHECK(op, (n_data_channels % input_channels) == 0); - groups = DimensionType(n_data_channels / input_channels); - } - } - if (input_shape[1].is_static()) { - // GROUPS and C_IN consistency checks - if (groups.is_static() && filters_shape[2].is_static()) { - NODE_VALIDATION_CHECK(op, - input_shape[1].get_length() / groups.get_length() == filters_shape[2].get_length(), - "Input channels dimension of data batch has incompatible value " - "with filter shape."); - } else if (groups.is_static()) { - NODE_VALIDATION_CHECK(op, - input_shape[1].get_length() % groups.get_length() == 0, - "Input channels dimension of data batch not a multiple of group size."); - } - } - - output_shape[1] = groups * filters_shape[1]; - - calculate_output_spatial_dims_for_convolution(op, - input_shape, - filters_shape, - output_shape, - num_spatial, - op->m_strides, - op->m_dilations, - pads_begin, - pads_end, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); -} - -template -int64_t calculate_num_spatial(const ConvType* op, - const PartialShape& input_shape, - const PartialShape& filters_shape, - const PartialShape& output_shapes_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - auto num_spatial = op->m_num_spatial; - if (num_spatial == -1) { - num_spatial = calculate_num_spatial(op, - input_shape, - filters_shape, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); - if (const auto& size = op->m_output_padding.size()) { - NODE_VALIDATION_CHECK(op, - num_spatial == -1 || num_spatial == static_cast(size), - "Output padding should be defined for all and only spatial dimensions."); - num_spatial = static_cast(size); - } - if (output_shapes_shape.is_static()) { - NODE_VALIDATION_CHECK(op, - output_shapes_shape.size() == 1, - "Input delivering output shape must have rank 1"); - NODE_VALIDATION_CHECK(op, - num_spatial == -1 || num_spatial == output_shapes_shape[0].get_length(), - "Output shape should be specified only and for all spatial dimensions."); - num_spatial = static_cast(output_shapes_shape[0].get_length()); - } - } - return num_spatial; -} - -template -int64_t calculate_num_spatial(const ConvType* op, - const ShapeType& input_shape, - const ShapeType& filters_shape, - const ShapeType& output_shapes_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - return calculate_num_spatial(op, - input_shape.to_partial_shape(), - filters_shape.to_partial_shape(), - output_shapes_shape.to_partial_shape(), - num_non_spatial_data_dims, - num_non_spatial_filter_dims); -} - -template -void update_and_validate_attributes_back_prop(ConvType* op, int64_t num_spatial) { - if (num_spatial != -1) { - update_and_validate_attributes(op, num_spatial); - auto& output_padding = op->m_output_padding; - if (output_padding.empty()) - output_padding = CoordinateDiff(num_spatial, 0); - NODE_VALIDATION_CHECK(op, - static_cast(output_padding.size()) == num_spatial, - "Output padding should be defined for all and only " - "spatial dimensions.."); - } else if (op->m_num_spatial != -1) { - update_and_validate_attributes_back_prop(op, op->m_num_spatial); - } -} - -template -bool resolve_auto_pad_for_shape_back_prop(const ConvType* op, - CoordinateDiff& pads_begin, - CoordinateDiff& pads_end, - const std::vector& input_shapes, - ShapeType& output_spatial_shape, - const int64_t& num_non_spatial_data_dims, - const int64_t& num_non_spatial_filter_dims) { - const auto& auto_pad = op->m_auto_pad; - if (auto_pad != PadType::SAME_UPPER && auto_pad != PadType::SAME_LOWER) { - pads_begin = op->m_pads_begin; - pads_end = op->m_pads_end; - return true; - } - - const auto& num_spatial = op->m_num_spatial; - if (!dynamic_check(num_spatial)) - return false; - - if (input_shapes.size() != 3) { - pads_begin = CoordinateDiff(num_spatial, 0); - pads_end = CoordinateDiff(num_spatial, 0); - return true; - } - OPENVINO_ASSERT(input_shapes.size() == 3 && (auto_pad == PadType::SAME_UPPER || auto_pad == PadType::SAME_LOWER)); - - pads_begin = CoordinateDiff(num_spatial, 0); - pads_end = CoordinateDiff(num_spatial, 0); - if (output_spatial_shape.rank().is_dynamic()) - output_spatial_shape.resize(num_spatial); - - auto input_shape = input_shapes[0]; - auto filters_shape = input_shapes[1]; - - if (input_shape.rank().is_dynamic()) - input_shape.resize(num_spatial + num_non_spatial_data_dims); - if (filters_shape.rank().is_dynamic()) - filters_shape.resize(num_spatial + num_non_spatial_filter_dims); - - bool status = true; - for (auto i = 0; i < num_spatial; ++i) { - const auto& data_dim = input_shape[i + num_non_spatial_data_dims]; - const auto& filter_dim = filters_shape[i + num_non_spatial_filter_dims]; - const auto& output_dim = output_spatial_shape[i]; - const auto& output_padding = op->m_output_padding[i]; - - if (data_dim.is_static() && filter_dim.is_static() && output_dim.is_static()) { - const auto& strides = op->m_strides[i]; - const auto& dilations = op->m_dilations[i]; - int total_padding = std::max( - static_cast(strides * (data_dim.get_length() - 1) + dilations * (filter_dim.get_length() - 1) + 1 - - output_dim.get_length() + output_padding), - 0); - if (auto_pad != op::PadType::SAME_UPPER) { - pads_begin[i] = total_padding / 2; - pads_end[i] = total_padding - pads_begin[i]; - } else { - pads_end[i] = total_padding / 2; - pads_begin[i] = total_padding - pads_end[i]; - } - } else { - status = false; - } - } - return status; -} - -template -void shape_infer(const ConvolutionBackpropData* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const T& output_shape_from_input, - const std::vector& input_shapes, - std::vector& output_shapes) { - constexpr size_t num_non_spatial_data_dims = 2, num_non_spatial_filter_dims = 2; - size_t input_size = input_shapes.size(); - NODE_VALIDATION_CHECK(op, (input_size == 2 || input_size == 3) && output_shapes.size() == 1); - auto input_shape = input_shapes[0], filters_shape = input_shapes[1]; - - const auto num_spatial = op->m_num_spatial != -1 - ? op->m_num_spatial - : input_size == 3 ? calculate_num_spatial(op, - input_shape, - filters_shape, - input_shapes[2], - num_non_spatial_data_dims, - num_non_spatial_filter_dims) - : calculate_num_spatial(op, - input_shape, - filters_shape, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); - - NODE_VALIDATION_CHECK(op, - num_spatial != -1, - "ConvolutionBackpropData shape_infer should be provided with correct num_spatial attribute"); - - NODE_VALIDATION_CHECK(op, - num_spatial == 1 || num_spatial == 2 || num_spatial == 3, - "Data and filters inputs must have rank 3, 4 or 5"); - - if (input_shape.rank().is_dynamic()) - input_shape.resize(num_spatial + num_non_spatial_data_dims); - if (filters_shape.rank().is_dynamic()) - filters_shape.resize(num_spatial + num_non_spatial_filter_dims); - - NODE_VALIDATION_CHECK( - op, - (static_cast(input_shape.size()) == static_cast(num_spatial + num_non_spatial_data_dims)) && - (static_cast(filters_shape.size()) == - static_cast(num_spatial + num_non_spatial_filter_dims)), - "Data and filters rank do not match (data batch shape: ", - input_shape, - ", filters shape: ", - filters_shape, - ")."); - - // ranks are originally static or aligned with num_spatial, attributes assumed to be valid - auto& output_shape = output_shapes[0]; - output_shape.resize(num_spatial + num_non_spatial_data_dims); - output_shape[0] = input_shape[0]; - output_shape[1] = filters_shape[1]; - - NODE_VALIDATION_CHECK(op, - input_shape[1].compatible(filters_shape[0]), - "Input channels dimension of data and filters inputs must be equal"); - - if (input_size == 3) { - if (output_shape_from_input.rank().is_static()) { - NODE_VALIDATION_CHECK(op, - static_cast(output_shape_from_input.size()) == num_spatial, - "Output shape should be specified only and for all spatial dimensions."); - for (int64_t i = 0; i < num_spatial; ++i) - output_shape[i + num_non_spatial_data_dims] = output_shape_from_input[i]; - } +template +std::vector shape_infer(const TFrowardConv* op, + const std::vector& input_shapes, + TContainer& pads_begin, + TContainer& pads_end, + const std::map& constant_data = {}) { + NODE_VALIDATION_CHECK(op, input_shapes.size() >= 2); + using namespace ov::util; + + const auto num_spatial = convolution::calculate_num_spatial(op, input_shapes); + + TShape output_shape; + if (num_spatial != util::num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + const auto data_rank = data_shape.rank(); + const auto filters_rank = filters_shape.rank(); + + convolution::resize_empty_padding(num_spatial, pads_begin, pads_end); + convolution::validate::filter_shape(op, filters_shape, data_shape); + if (is_attr_validation_required(op)) { + convolution::validate::data_shape(op, data_shape); + convolution::validate::common_attributes(op, num_spatial, pads_begin, pads_end); + } + convolution::apply_padding(op, data_shape, filters_shape, pads_begin, pads_end); + + output_shape.reserve(util::spatial_dim_offset + num_spatial); + output_shape.emplace_back(data_rank.is_static() ? data_shape[0] : dim::inf_bound); + output_shape.emplace_back(filters_rank.is_static() ? filters_shape[0] : dim::inf_bound); + convolution::append_spatial_shape(op, data_shape, filters_shape, pads_begin, pads_end, output_shape); } else { - const auto& strides = op->m_strides; - const auto& dilations = op->m_dilations; - const auto& output_padding = op->m_output_padding; - for (int64_t i = 0; i < num_spatial; ++i) { - const auto &data_idx = i + num_non_spatial_data_dims, filter_idx = i + num_non_spatial_filter_dims; - output_shape[data_idx] = (input_shape[data_idx] - 1) * strides[i] + - (filters_shape[filter_idx] - 1) * dilations[i] + 1 - pads_begin[i] - pads_end[i] + - output_padding[i]; - } - } -} - -template -void shape_infer(const GroupConvolutionBackpropData* op, - const CoordinateDiff& pads_begin, - const CoordinateDiff& pads_end, - const T& output_shape_from_input, - const std::vector& input_shapes, - std::vector& output_shapes) { - constexpr size_t num_non_spatial_data_dims = 2, num_non_spatial_filter_dims = 3; - size_t input_size = input_shapes.size(); - NODE_VALIDATION_CHECK(op, (input_size == 2 || input_size == 3) && output_shapes.size() == 1); - auto input_shape = input_shapes[0], filters_shape = input_shapes[1]; - - const auto num_spatial = op->m_num_spatial != -1 - ? op->m_num_spatial - : input_size == 3 ? calculate_num_spatial(op, - input_shape, - filters_shape, - input_shapes[2], - num_non_spatial_data_dims, - num_non_spatial_filter_dims) - : calculate_num_spatial(op, - input_shape, - filters_shape, - num_non_spatial_data_dims, - num_non_spatial_filter_dims); - - NODE_VALIDATION_CHECK( - op, - num_spatial != -1, - "GroupConvolutionBackpropData shape_infer should be provided with correct num_spatial attribute"); - - NODE_VALIDATION_CHECK(op, - num_spatial == 1 || num_spatial == 2 || num_spatial == 3, - "Data and filters inputs must have rank 3, 4 or 5"); - - if (input_shape.rank().is_dynamic()) - input_shape.resize(num_spatial + num_non_spatial_data_dims); - if (filters_shape.rank().is_dynamic()) - filters_shape.resize(num_spatial + num_non_spatial_filter_dims); - - NODE_VALIDATION_CHECK( - op, - (static_cast(input_shape.size()) == static_cast(num_spatial + num_non_spatial_data_dims)) && - (static_cast(filters_shape.size()) == - static_cast(num_spatial + num_non_spatial_filter_dims)), - "Data and filters rank do not match (data batch shape: ", - input_shape, - ", filters shape: ", - filters_shape, - ")."); - - // ranks are originally static or aligned with num_spatial, attributes assumed to be valid - auto& output_shape = output_shapes[0]; - output_shape.resize(num_spatial + num_non_spatial_data_dims); - output_shape[0] = input_shape[0]; - - auto groups = filters_shape[0]; - if (groups.is_dynamic()) { - // [N, GROUPS * C_IN, ...] x [GROUPS, C_IN, C_OUT, ...] = [N, GROUPS * C_OUT, ...] - if (input_shape[1].is_static() && filters_shape[1].is_static()) { - using DimensionType = typename std::iterator_traits::value_type; - auto n_data_channels = input_shape[1].get_length(); - auto input_channels = filters_shape[1].get_length(); - NODE_VALIDATION_CHECK(op, (n_data_channels % input_channels) == 0); - groups = DimensionType(n_data_channels / input_channels); - } + output_shape = PartialShape::dynamic(); } - if (input_shape[1].is_static()) { - // GROUPS and C_IN consistency checks - if (groups.is_static() && filters_shape[1].is_static()) { - NODE_VALIDATION_CHECK(op, - input_shape[1].get_length() / groups.get_length() == filters_shape[1].get_length(), - "Input channels dimension of data batch has incompatible value " - "with filter shape."); - } else if (groups.is_static()) { - NODE_VALIDATION_CHECK(op, - input_shape[1].get_length() % groups.get_length() == 0, - "Input channels dimension of data batch not a multiple of group size."); - } - } - - output_shape[1] = filters_shape[2] * groups; - if (input_size == 3) { - if (output_shape_from_input.rank().is_static()) { - NODE_VALIDATION_CHECK(op, - static_cast(output_shape_from_input.size()) == num_spatial, - "Output shape should be specified only and for all spatial dimensions."); - for (int64_t i = 0; i < num_spatial; ++i) - output_shape[i + num_non_spatial_data_dims] = output_shape_from_input[i]; - } - } else { - const auto& strides = op->m_strides; - const auto& dilations = op->m_dilations; - const auto& output_padding = op->m_output_padding; - for (int64_t i = 0; i < num_spatial; ++i) { - const auto &data_idx = i + num_non_spatial_data_dims, filter_idx = i + num_non_spatial_filter_dims; - output_shape[data_idx] = (input_shape[data_idx] - 1) * strides[i] + - (filters_shape[filter_idx] - 1) * dilations[i] + 1 - pads_begin[i] - pads_end[i] + - output_padding[i]; - } - } + return {output_shape}; } - } // namespace v1 } // namespace op } // namespace ov diff --git a/src/core/shape_inference/include/convolution_shape_inference_util.hpp b/src/core/shape_inference/include/convolution_shape_inference_util.hpp new file mode 100644 index 00000000000000..c1a26671e3e627 --- /dev/null +++ b/src/core/shape_inference/include/convolution_shape_inference_util.hpp @@ -0,0 +1,359 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "dimension_util.hpp" +#include "openvino/op/util/convolution_backprop_base.hpp" +#include "openvino/op/util/convolution_base.hpp" +#include "pooling_shape_inference_util.hpp" +#include "utils.hpp" + +namespace ov { +namespace op { + +namespace util { +constexpr size_t num_spatial_undefined = std::numeric_limits::max(); +constexpr size_t spatial_dim_offset = 2; + +/** + * @brief Get num of spatial form convolution operator. + * + * Tries get value from operator member if is not deduced (has -1 value) then tries evaluate it from input shapes. + * + * @tparam TConv Convolution type (this function must be a friend of TConv to access private member). + * @tparam TShape Shape type. + * @param op Pointer to convolution operator. + * @param data_shape Input data shape. + * @param flter_shape Input filter shape. + * @return Value of spatial dimension number or infinite bound (-1) if cannot evaluate. + */ +template +size_t num_spatial_from_shapes(const TShape& data_shape, + const TShape& filter_shape, + const size_t filter_non_spatial_dims_count) { + const auto& data_rank = data_shape.rank(); + const auto& filters_rank = filter_shape.rank(); + + size_t num_spatial; + + if (data_rank.is_static()) { + num_spatial = data_rank.get_length() - spatial_dim_offset; + } else if (filters_rank.is_static()) { + num_spatial = filters_rank.get_length() - filter_non_spatial_dims_count; + } else { + num_spatial = num_spatial_undefined; + } + + return num_spatial; +} + +/** + * @brief Checks if validation attributes is required. + * + * @param op Pointer to convolution base operator. + * @return True if internal number of spatial dimension not defined otherwise false. + */ +inline bool is_attr_validation_required(const ConvolutionBase* op) { + return num_spatial_undefined == op->m_num_spatial; +} + +/** + * @brief Get the num spatil object + * + * @param op + * @return size_t + */ +inline size_t get_num_spatial(const ConvolutionBase* op) { + return op->m_num_spatial; +} +} // namespace util + +namespace convolution { + +constexpr auto num_spatial_undefined = util::num_spatial_undefined; +constexpr size_t spatial_dim_offset = 2; + +/** + * @brief Provides convolution filter non spatial dimension count. + * + * @note If specific convolution operator requires different value provide specialization for this operator. + * @tparam TConv Type of convolution operator. + * @return Default value for convolution operators (2). + */ +template +constexpr size_t filter_non_spatial_dims_count() { + return 2; +} + +/** + * @brief Checks if Op property auto_pad is set to same lower or upper. + * + * @tparam TOp Type of operator (must have get_auto_pad member function). + * @param op Pointer to operator. + * @return True if auto pad enabled. + */ +template +bool is_auto_pad(const TOp* op) { + return (op->get_auto_pad() == PadType::SAME_LOWER) || (op->get_auto_pad() == PadType::SAME_UPPER); +} + +/** + * @brief Resize paddings if empty to number of spatial dimensions. + * + * @param num_spatial Number of spatial dimensions. + * @param pads_begin Begin padding to resize. + * @param pads_end End padding to resize. + */ +inline void resize_empty_padding(const size_t num_spatial, CoordinateDiff& pads_begin, CoordinateDiff& pads_end) { + if (pads_begin.empty()) { + pads_begin.resize(num_spatial); + } + + if (pads_end.empty()) { + pads_end.resize(num_spatial); + } +} + +inline size_t num_spatial_from_attr(const util::ConvolutionBase* op) { + size_t num_spatial; + + if (!op->get_strides().empty()) { + num_spatial = op->get_strides().size(); + } else if (!op->get_dilations().empty()) { + num_spatial = op->get_dilations().size(); + } else if (!op->get_pads_begin().empty()) { + num_spatial = op->get_pads_begin().size(); + } else if (!op->get_pads_end().empty()) { + num_spatial = op->get_pads_end().size(); + } else { + num_spatial = num_spatial_undefined; + } + + return num_spatial; +} + +template ::value>::type* = nullptr> +size_t calculate_num_spatial(const TOp* op, const std::vector& input_shapes) { + NODE_VALIDATION_CHECK(op, input_shapes.size() > 1); + auto num_spatial = get_num_spatial(op); + + if (num_spatial == num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + num_spatial = util::num_spatial_from_shapes(data_shape, filters_shape, filter_non_spatial_dims_count()); + } + + if (num_spatial == num_spatial_undefined) { + num_spatial = num_spatial_from_attr(op); + } + + return num_spatial; +} + +/** + * @brief Apply auto padding for forward convolution. + * + * The auto padding can be applied only if inputs and attributes of operator are validated. + * The input shapes must have got static ranks. + * + * @param op Pointer to convolution operator. + * @param data_shape Input data shape (must be static rank). + * @param filters_shape Input filter shape (must be static rank). + * @param pads_begin Iterator to begin of pads begin. + * @param pads_end Iterator to begin of pads end. + */ +template ::value || + std::is_base_of::value>::type* = nullptr> +void apply_auto_pad(const TOp* op, + const TShape& data_shape, + const TShape& filters_shape, + TIter pads_begin, + TIter pads_end) { + const auto& dilations = op->get_dilations(); + const auto& strides = op->get_strides(); + + const auto num_spatial = strides.size(); + auto data_dim = data_shape.cend() - num_spatial; + auto kernel_dim = filters_shape.cend() - num_spatial; + + const auto padding_swap = op->get_auto_pad() == PadType::SAME_UPPER; + auto& pad_b = padding_swap ? pads_begin : pads_end; + auto& pad_e = padding_swap ? pads_end : pads_begin; + + for (size_t i = 0; i < num_spatial; ++i, ++pad_b, ++pad_e, ++data_dim, ++kernel_dim) { + using namespace ov::util; + if (kernel_dim->is_static()) { + std::tie(*pad_b, *pad_e) = dim::padding(*data_dim, kernel_dim->get_length(), dilations[i], strides[i]); + } else { + *pad_b = 0; + *pad_e = 0; + } + } +} + +/** + * @brief Apply padding to forward propagation convolution besed on padding. + * + * @tparam TShape + * + * @param op Pointer to coevolution operator. + * @param data_shape Input data shapes for shape inference. + * @param filters_shape Input filters shape for shape inference. + * @param pads_begin Begin padding to updated. + * @param pads_end End padding to update. + */ +template ::value || + std::is_base_of::value>::type* = nullptr> +void apply_padding(const TOp* op, + const TShape& data_shape, + const TShape& filters_shape, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end) { + if (convolution::is_auto_pad(op) && data_shape.rank().is_static() && filters_shape.rank().is_static()) { + convolution::apply_auto_pad(op, data_shape, filters_shape, pads_begin.begin(), pads_end.begin()); + } else if (op->get_auto_pad() == op::PadType::VALID) { + std::fill(pads_begin.begin(), pads_begin.end(), 0); + std::fill(pads_end.begin(), pads_end.end(), 0); + } else if (op->get_auto_pad() == op::PadType::EXPLICIT) { + std::copy(op->get_pads_begin().begin(), op->get_pads_begin().end(), pads_begin.begin()); + std::copy(op->get_pads_end().begin(), op->get_pads_end().end(), pads_end.begin()); + } +} + +/** + * @brief Append spatial dimension at end of output shape of forward propagation convolution. + * + * @tparam TOp Forward propagation convolution operator type. + * @tparam TShape Type of shape. + * @param op Pointer to operator. + * @param data_shape Input data shape. + * @param filters_shape Input filter shape. + * @param out_shape Output shape to append spatial dimensions. + */ +template ::value || + std::is_base_of::value>::type* = nullptr> +void append_spatial_shape(const TOp* op, + const TShape& data_shape, + const TShape& filters_shape, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + TShape& out_shape) { + using namespace ov::util; + using TDim = typename TShape::value_type; + + const auto& strides = op->get_strides(); + const auto spatial_num = strides.size(); + + const auto& d_shape = data_shape.rank().is_static() ? data_shape : PartialShape::dynamic(spatial_num); + auto data_dim = d_shape.cend() - spatial_num; + + if (is_auto_pad(op)) { + std::transform(data_dim, d_shape.cend(), strides.cbegin(), std::back_inserter(out_shape), &dim::ceil_div); + } else { + const auto& f_shape = filters_shape.rank().is_static() ? filters_shape : PartialShape::dynamic(spatial_num); + auto filters_dim = f_shape.cend() - spatial_num; + const auto& dilations = op->get_dilations(); + + for (size_t i = 0; i < spatial_num; ++i, ++data_dim, ++filters_dim) { + auto dim = *data_dim + (pads_begin[i] + pads_end[i]); + const auto filter_dilated = dim::dilated(*filters_dim, dilations[i]); + + if (dim.is_static() && filter_dilated.is_static()) { + // Use check from pooling op as it is same. + pooling::valid_dilated_kernel_with_dim(op, filter_dilated.get_length(), dim, i); + } + + dim = dim::floor_div(dim - filter_dilated, strides[i]); + dim += 1; + out_shape.push_back(std::move(dim)); + } + } +} + +namespace validate { +template +void data_shape(const ov::op::util::ConvolutionBase* op, const TShape& data_shape) { + OPENVINO_SUPPRESS_DEPRECATED_START + NODE_VALIDATION_CHECK(op, + is_rank_compatible_any_of(data_shape.rank(), {3, 4, 5}), + "Expected a 3D, 4D or 5D tensor for the input. Got: ", + data_shape); + OPENVINO_SUPPRESS_DEPRECATED_END +} + +template +void filter_shape(const ov::op::util::ConvolutionBase* op, const TShape& filters_shape, const TShape& data_shape) { + const auto& data_rank = data_shape.rank(); + const auto& filters_rank = filters_shape.rank(); + + NODE_VALIDATION_CHECK(op, + data_rank.compatible(filters_rank), + "Data batch and filters rank do not match (data batch shape: ", + data_shape, + ", filters shape: ", + filters_shape, + ")."); + + NODE_VALIDATION_CHECK( + op, + data_rank.is_dynamic() || filters_rank.is_dynamic() || data_shape[1].compatible(filters_shape[1]), + "Data batch channel count (", + data_shape[1], + ") does not match filter input channel count (", + filters_shape[1], + ")."); +} + +inline void common_attributes(const util::ConvolutionBase* op, + const size_t num_spatial, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end) { + auto& strides = op->get_strides(); + auto& dilations = op->get_dilations(); + + NODE_VALIDATION_CHECK(op, + strides.size() == num_spatial, + "Strides should be defined for all and only spatial dimensions."); + NODE_VALIDATION_CHECK(op, + dilations.size() == num_spatial, + "Dilations should be defined for all and only spatial dimensions."); + NODE_VALIDATION_CHECK(op, + pads_begin.size() == num_spatial && pads_end.size() == pads_begin.size(), + "Pads begin and end should be defined for all and only spatial dimensions."); + + constexpr auto is_zero = cmp::Equal(0); + NODE_VALIDATION_CHECK(op, + std::none_of(strides.cbegin(), strides.cend(), is_zero), + "Strides has zero dimension(s). ", + strides); + NODE_VALIDATION_CHECK(op, + std::none_of(dilations.cbegin(), dilations.cend(), is_zero), + "Filter dilations has zero dimension(s). ", + dilations); +} + +inline void common_attributes(const util::ConvolutionBackPropBase* op, + const size_t num_spatial, + const CoordinateDiff& pads_begin, + const CoordinateDiff& pads_end) { + common_attributes(static_cast(op), num_spatial, pads_begin, pads_end); + NODE_VALIDATION_CHECK(op, + op->get_output_padding().size() == num_spatial, + "Output padding should be defined for all and only spatial dimensions."); +} +} // namespace validate +} // namespace convolution + +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/deformable_convolution_shape_inference.hpp b/src/core/shape_inference/include/deformable_convolution_shape_inference.hpp new file mode 100644 index 00000000000000..d78924ca615ab8 --- /dev/null +++ b/src/core/shape_inference/include/deformable_convolution_shape_inference.hpp @@ -0,0 +1,246 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "convolution_shape_inference_util.hpp" +#include "openvino/op/util/deformable_convolution_base.hpp" + +namespace ov { +namespace op { +namespace deformable_conv { +template +size_t calculate_num_spatial(const util::DeformableConvolutionBase* op, const std::vector& input_shapes) { + constexpr auto non_spatial_count = convolution::filter_non_spatial_dims_count(); + + auto num_spatial = util::num_spatial_from_shapes(input_shapes[0], input_shapes[2], non_spatial_count); + + if (num_spatial == convolution::num_spatial_undefined && input_shapes[1].rank().is_static()) { + num_spatial = input_shapes[1].size() - non_spatial_count; + } + + return num_spatial; +} + +namespace validate { +template +void input_shape(const TDeformableConv* op, const TShape& shape, const std::string& name) { + const auto& shape_rank = shape.rank(); + NODE_VALIDATION_CHECK(op, shape_rank.compatible(4), name, " must be of rank 4. Got: ", shape_rank); +} + +template +void group_attribute(const TDeformableConv* op, int64_t group, const std::string& name) { + NODE_VALIDATION_CHECK(op, group > 0, "Attribute '", name, "' must be any value starting from 1. Got: ", group); +} + +template +void group_divisible_dimension(const TDeformableConv* op, const TDim& dim, const std::string name) { + const auto group = op->get_group(); + NODE_VALIDATION_CHECK(op, + ov::util::dim::is_divisible(dim, group), + name, + " channels dimension (", + dim, + ") must be evenly divisible by the 'group': ", + group); +} + +template +void deformable_group_divisible_dimension(const TDeformableConv* op, const TDim& dim, const std::string name) { + const auto group = op->get_deformable_group(); + NODE_VALIDATION_CHECK(op, + ov::util::dim::is_divisible(dim, group), + name, + " channels dimension (", + dim, + ") must be evenly divisible by the 'deformable group': ", + group); +} +} // namespace validate +} // namespace deformable_conv + +namespace util { +template +std::vector shape_infer(const DeformableConvolutionBase* op, + const std::vector& input_shapes, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + const std::map& constant_data = {}) { + static constexpr std::array names{"Input", "Offsets", "Filters", "Mask"}; + using namespace ov::util; + using TDim = typename TShape::value_type; + + const auto num_spatial = deformable_conv::calculate_num_spatial(op, input_shapes); + + TShape output_shape; + if (num_spatial != convolution::num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& offsets_shape = input_shapes[1]; + const auto& filters_shape = input_shapes[2]; + + const auto data_rank = data_shape.rank(); + const auto filters_rank = filters_shape.rank(); + const auto offsets_rank = offsets_shape.rank(); + + output_shape.reserve(num_spatial + util::spatial_dim_offset); + + convolution::resize_empty_padding(num_spatial, pads_begin, pads_end); + for (size_t i = 0; i < input_shapes.size(); ++i) { + deformable_conv::validate::input_shape(op, input_shapes[i], names[i]); + } + deformable_conv::validate::group_attribute(op, op->get_group(), "group"); + deformable_conv::validate::group_attribute(op, op->get_deformable_group(), "deformable group"); + convolution::validate::common_attributes(op, num_spatial, pads_begin, pads_end); + convolution::apply_padding(op, data_shape, filters_shape, pads_begin, pads_end); + + // add to output shape number of batches + if (data_rank.is_static()) { + deformable_conv::validate::group_divisible_dimension(op, data_shape[1], names[0]); + + output_shape.push_back(data_shape[0]); + } else { + output_shape.emplace_back(dim::inf_bound); + } + if (offsets_rank.is_static()) { + if (filters_rank.is_static()) { + auto offsets_channels = filters_shape[2] * filters_shape[3] * 2 * op->get_deformable_group(); + + NODE_VALIDATION_CHECK(op, + offsets_shape[1].compatible(offsets_channels), + "The channels dimension of offsets input is not compatible with filters and " + "'deformable group' attribute. Offsets input shape: ", + offsets_shape, + ", deformable 'group' attribute value: ", + op->get_deformable_group(), + ", filters shape: ", + filters_shape); + } + deformable_conv::validate::deformable_group_divisible_dimension(op, offsets_shape[1], names[1]); + + NODE_VALIDATION_CHECK(op, + TDim::merge(output_shape[0], offsets_shape[0], output_shape[0]), + "Data batch and offsets batch dimension must be same value. Got: ", + output_shape[0], + " and ", + data_shape[0]); + } + + // add to output shape number output channels + if (filters_rank.is_static()) { + deformable_conv::validate::group_divisible_dimension(op, filters_shape[0], names[2]); + + NODE_VALIDATION_CHECK( + op, + data_rank.is_dynamic() || data_shape[1].compatible(filters_shape[1] * op->get_group()), + "Data batch channel count (", + data_shape[1], + ") does not match filter input channel count (", + filters_shape[1] * op->get_group(), + ")"); + + output_shape.push_back(filters_shape[0]); + } else { + output_shape.emplace_back(dim::inf_bound); + } + convolution::append_spatial_shape(op, data_shape, filters_shape, pads_begin, pads_end, output_shape); + + // post infer check. + if (offsets_rank.is_static()) { + auto offset_dim = offsets_shape.begin() + util::spatial_dim_offset; + NODE_VALIDATION_CHECK(op, + std::all_of(output_shape.begin() + util::spatial_dim_offset, + output_shape.end(), + [&offset_dim](const TDim& d) { + return d.compatible(*offset_dim++); + }), + "Spatial dimensions of offsets and output must be compatible.", + output_shape); + } + } else { + output_shape = PartialShape::dynamic(); + } + + return {output_shape}; +} +} // namespace util + +namespace v1 { +template +std::vector shape_infer(const DeformableConvolution* op, + const std::vector& input_shapes, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + const std::map& constant_data = {}) { + NODE_VALIDATION_CHECK(op, input_shapes.size() == 3); + return util::shape_infer(op, input_shapes, pads_begin, pads_end, constant_data); +} +} // namespace v1 + +namespace v8 { +template +std::vector shape_infer(const DeformableConvolution* op, + const std::vector& input_shapes, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + const std::map& constant_data = {}) { + const auto has_mask_shape = input_shapes.size() == 4; + NODE_VALIDATION_CHECK(op, input_shapes.size() == 3 || has_mask_shape); + using TDim = typename TShape::value_type; + + const auto& data_shape = input_shapes[0]; + const auto& offsets_shape = input_shapes[1]; + const auto& filters_shape = input_shapes[2]; + + const auto data_rank = data_shape.rank(); + const auto filters_rank = filters_shape.rank(); + const auto offsets_rank = offsets_shape.rank(); + + if (has_mask_shape) { + const auto mask_shape = input_shapes[3]; + if (mask_shape.rank().is_static()) { + if (filters_rank.is_static()) { + auto offsets_channels = filters_shape[2] * filters_shape[3] * op->get_deformable_group(); + + NODE_VALIDATION_CHECK(op, + mask_shape[1].compatible(offsets_channels), + "The channels dimension of mask input is not " + "compatible with filters and 'deformable group' attribute. " + "Mask input shape: ", + mask_shape, + ", deformable 'group' attribute value: ", + op->get_deformable_group(), + ", filters shape: ", + filters_shape); + } + + deformable_conv::validate::deformable_group_divisible_dimension(op, mask_shape[1], "Mask"); + + NODE_VALIDATION_CHECK(op, + data_rank.is_dynamic() || mask_shape[0].compatible(data_shape[0]), + "Data batch and mask batch dimension must be same value. Got: ", + mask_shape[0], + " and ", + data_shape[0]); + } + } + + auto output_shapes = util::shape_infer(op, input_shapes, pads_begin, pads_end, constant_data); + // post infer checks + if (has_mask_shape && input_shapes[3].rank().is_static() && output_shapes[0].rank().is_static()) { + auto mask_dim = input_shapes[3].begin() + util::spatial_dim_offset; + NODE_VALIDATION_CHECK(op, + std::all_of(output_shapes[0].begin() + util::spatial_dim_offset, + output_shapes[0].end(), + [&mask_dim](const TDim& d) { + return d.compatible(*mask_dim++); + }), + "Spatial dimensions of mask and output must be compatible."); + } + return output_shapes; +} +} // namespace v8 +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/dimension_util.hpp b/src/core/shape_inference/include/dimension_util.hpp index b3ffb07b7ec8c0..516a0aa13b190b 100644 --- a/src/core/shape_inference/include/dimension_util.hpp +++ b/src/core/shape_inference/include/dimension_util.hpp @@ -5,6 +5,7 @@ #include +#include "openvino/core/dimension.hpp" #include "openvino/util/common_util.hpp" namespace ov { @@ -25,17 +26,82 @@ constexpr auto dilated(const T dim, const T dilation) -> T { return (dim < 1) ? inf_bound : dilation * (dim - 1) + 1; } +/** + * @brief Calculate dilated dimension. + * + * @tparam TDim Dimension type. + * @param dim Dimension. + * @param dilation Dilation value. + * @return Return dimension after dilation. + */ +template +constexpr auto dilated(const TDim& dim, const typename TDim::value_type dilation) -> TDim { + return (dim - 1) * dilation + 1; +} + /** * @brief Calculate padded dimension size as dim size + padding size * + * @tparam TDim Dimension type as dimension class value type or any arithmetic value. * @param dim Dimension size value. - * @param pad_num Number of padded dimension. + * @param pad_num Number of padding to add. * @return Padded dimension value or infinite bound. */ -constexpr auto padded(const int64_t dim, const int64_t pad_num) -> int64_t { +template +constexpr typename std::enable_if::value, TDim>::type padded(const TDim dim, + const int64_t pad_num) { return ((dim == inf_bound) || (dim + pad_num < 0)) ? inf_bound : dim + pad_num; } +/** + * @brief Calculate padded dimension size as dim + padding size + * + * @note the Dimension + operator cannot be used if padding is '-1' which result add dynamic dimension. + * + * @tparam TDim Dimension type as dimension class. + * @param dim Dimension. + * @param pad_num Number padding to add. + * @return Padded dimension. + */ +template +typename std::enable_if::value, TDim>::type padded(const TDim& dim, const int64_t pad_num) { + auto ub = padded(dim.get_max_length(), pad_num); + if (dim.is_static()) { + return {ub}; + } else { + return {padded(dim.get_min_length(), pad_num), ub}; + } +} + +/** + * @brief Calculate dimension padding required by filter/kernel properties. + * + * Provides pair of padding values as left padding is total value of required padding divided by 2 and right as + * total required padding minus left padding. + * + * @param dim input dimension to calculate its padding. + * @param filter_size Kernel size for input dimension. + * @param dilation Kernel dilation. + * @param stride Kernel stride. + * @return Pair of left, right padding values for input dimension. + */ +template +inline std::pair padding(const TDim& dim, const int64_t kernel_size, const int64_t dilation, int64_t stride) { + if (dim.is_static()) { + const auto dim_size = static_cast(dim.get_length()); + const auto dilated_kernel = dilated(kernel_size, dilation); + const int64_t tmp = (dim_size + stride - 1) / stride; + + const auto padding = std::max(0, (tmp - 1) * stride + dilated_kernel - dim_size); + const auto left_padding = padding / 2; + return {left_padding, padding - left_padding}; + } else { + // If input dimension is infinite or interval the padding will be set to 0 + // as operator cannot store paddings for both bounds. + return {0, 0}; + } +} + /** * @brief Divide dimension using ceil rounding. * @@ -46,8 +112,9 @@ constexpr auto padded(const int64_t dim, const int64_t pad_num) -> int64_t { * @param divisor Dimension division. * @return Divided dimension with bounds round up. */ -template -auto ceil_div(const TDim& dim, const T divisor) -> TDim { +template +auto ceil_div(const TDim& dim, const typename TDim::value_type divisor) -> TDim { + using T = decltype(divisor); if (dim.is_static()) { return {util::ceil_div(dim.get_length(), divisor)}; } else if (dim.get_max_length() == static_cast(dim::inf_bound)) { @@ -67,8 +134,9 @@ auto ceil_div(const TDim& dim, const T divisor) -> TDim { * @param divisor Dimension division. * @return Divided dimension with bound round down. */ -template -auto floor_div(const TDim& dim, const T divisor) -> TDim { +template +auto floor_div(const TDim& dim, const typename TDim::value_type divisor) -> TDim { + using T = decltype(divisor); if (dim.is_static()) { return {dim.get_length() / divisor}; } else if (dim.get_max_length() == static_cast(dim::inf_bound)) { @@ -78,6 +146,24 @@ auto floor_div(const TDim& dim, const T divisor) -> TDim { } } +/** + * @brief Check if dimension is evenly divisible. + * + * @tparam TDim Dimension type. + * @param quotient Dimension to check. + * @param dividend Dividend to check. + * @return true if dimension is divisible other wise false. + */ +template +bool is_divisible(const TDim& quotient, const typename TDim::value_type dividend) { + return quotient / dividend != TDim{}; +} + +template <> +inline bool is_divisible(const Dimension& quotient, const typename Dimension::value_type dividend) { + return !(quotient / dividend).get_interval().empty(); +} + } // namespace dim } // namespace util } // namespace ov diff --git a/src/core/shape_inference/include/gather_elements_shape_inference.hpp b/src/core/shape_inference/include/gather_elements_shape_inference.hpp index 8bc763371c3537..8d3fedd0f31c7d 100644 --- a/src/core/shape_inference/include/gather_elements_shape_inference.hpp +++ b/src/core/shape_inference/include/gather_elements_shape_inference.hpp @@ -21,8 +21,11 @@ void shape_infer(const GatherElements* op, const std::vector& input_shapes, s auto& output_shape = output_shapes[0]; int64_t axis = op->get_axis(); - if (data_rank.is_static()) + if (data_rank.is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ov::normalize_axis(op, axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END + } NODE_VALIDATION_CHECK(op, data_rank.is_dynamic() || data_rank.get_length() >= 1, "data rank must be >= 1."); NODE_VALIDATION_CHECK(op, diff --git a/src/core/shape_inference/include/gather_shape_inference.hpp b/src/core/shape_inference/include/gather_shape_inference.hpp index 035d50f2d89871..b229ef5b882559 100644 --- a/src/core/shape_inference/include/gather_shape_inference.hpp +++ b/src/core/shape_inference/include/gather_shape_inference.hpp @@ -45,7 +45,9 @@ void shape_infer(const GatherBase* op, axis = axes_val[0]; if (data_rank.is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ov::normalize_axis(op, axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } // batch_dims, axis both can be positive by default or after normalization if data_rank & // indices_rank are static. diff --git a/src/core/shape_inference/include/group_convolution_backprop_shape_inference.hpp b/src/core/shape_inference/include/group_convolution_backprop_shape_inference.hpp new file mode 100644 index 00000000000000..d0c4844c0cf7af --- /dev/null +++ b/src/core/shape_inference/include/group_convolution_backprop_shape_inference.hpp @@ -0,0 +1,118 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "convolution_backprop_shape_inference.hpp" +#include "openvino/op/group_conv.hpp" +#include "utils.hpp" + +namespace ov { +namespace op { +namespace convolution { + +/** + * @brief Defines non-spatial dimension for filters for group convolution back propagation operator. + * @return Value of non-spatial filter dimensions (3). + */ +template <> +constexpr size_t filter_non_spatial_dims_count() { + return 3; +} +} // namespace convolution + +namespace v1 { +template +std::vector shape_infer(const GroupConvolutionBackpropData* op, + const std::vector& input_shapes, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + const std::map& constant_data = {}) { + const auto inputs_count = input_shapes.size(); + const auto has_spatial_shape = inputs_count >= 3; + NODE_VALIDATION_CHECK(op, inputs_count >= 2); + using namespace ov::util; + + TShape out_spatial_shape; + if (has_spatial_shape) { + const auto& spatial_shape = input_shapes[2]; + NODE_VALIDATION_CHECK(op, + spatial_shape.rank().compatible(1), + "Input delivering output shape must have rank 1."); + + if (!get_data_as_shape(2, op, out_spatial_shape, constant_data)) { + if (spatial_shape.is_static()) { + out_spatial_shape.resize(spatial_shape[0].get_length()); + } else { + out_spatial_shape = PartialShape::dynamic(); + } + } + } + + const auto num_spatial = convolution::calculate_num_spatial(op, input_shapes, out_spatial_shape); + + TShape output_shape; + if (num_spatial != util::num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + const auto data_rank = data_shape.rank(); + const auto filters_rank = filters_shape.rank(); + + NODE_VALIDATION_CHECK( + op, + !has_spatial_shape || out_spatial_shape.rank().is_dynamic() || out_spatial_shape.size() == num_spatial, + "Output shape should be defined for all and only spatial dimensions."); + + convolution::resize_empty_padding(num_spatial, pads_begin, pads_end); + if (is_attr_validation_required(op)) { + convolution::validate::data_shape(op, data_shape); + + NODE_VALIDATION_CHECK(op, + data_rank.compatible(filters_rank - 1), + "Data and filters rank do not match (data batch shape: ", + data_shape, + ", filters shape: ", + filters_shape, + ")."); + + convolution::validate::common_attributes(op, num_spatial, pads_begin, pads_end); + } + convolution::apply_padding(op, input_shapes, out_spatial_shape, pads_begin, pads_end); + + output_shape.reserve(util::spatial_dim_offset + num_spatial); + output_shape.emplace_back(data_rank.is_static() ? data_shape[0] : dim::inf_bound); + + // add groups dimension + if (filters_rank.is_static()) { + auto groups = filters_shape[0]; + + if (data_rank.is_static() && filters_shape[1].is_static()) { + NODE_VALIDATION_CHECK( + op, + groups.merge(groups, groups, (data_shape[1] / filters_shape[1].get_length())), + "Input channels dimension of data batch is incompatible with filter groups or input channels."); + } + + groups *= filters_shape[2]; + output_shape.push_back(std::move(groups)); + } else { + output_shape.emplace_back(dim::inf_bound); + } + + // add spatial dimensions + if (has_spatial_shape) { + output_shape.insert(output_shape.end(), + std::make_move_iterator(out_spatial_shape.begin()), + std::make_move_iterator(out_spatial_shape.end())); + } else { + convolution::append_spatial_shape(op, data_shape, filters_shape, pads_begin, pads_end, output_shape); + } + } else { + output_shape = PartialShape::dynamic(); + } + + return {output_shape}; +} +} // namespace v1 +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/group_convolution_shape_inference.hpp b/src/core/shape_inference/include/group_convolution_shape_inference.hpp new file mode 100644 index 00000000000000..f8f5bfe690d913 --- /dev/null +++ b/src/core/shape_inference/include/group_convolution_shape_inference.hpp @@ -0,0 +1,87 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "convolution_shape_inference_util.hpp" +#include "openvino/op/group_conv.hpp" +#include "utils.hpp" + +namespace ov { +namespace op { +namespace convolution { + +/** + * @brief Defines non-spatial dimension for filters for group convolution operator. + * @return Value of non-spatial filter dimensions (3). + */ +template <> +constexpr size_t filter_non_spatial_dims_count() { + return 3; +} +} // namespace convolution + +namespace v1 { +template +std::vector shape_infer(const GroupConvolution* op, + const std::vector& input_shapes, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end, + const std::map& constant_data = {}) { + NODE_VALIDATION_CHECK(op, input_shapes.size() >= 2); + using namespace ov::util; + + const auto num_spatial = convolution::calculate_num_spatial(op, input_shapes); + + TShape output_shape; + if (num_spatial != convolution::num_spatial_undefined) { + const auto& data_shape = input_shapes[0]; + const auto& filters_shape = input_shapes[1]; + const auto data_rank = data_shape.rank(); + const auto filters_rank = filters_shape.rank(); + + convolution::resize_empty_padding(num_spatial, pads_begin, pads_end); + if (is_attr_validation_required(op)) { + convolution::validate::data_shape(op, data_shape); + + NODE_VALIDATION_CHECK(op, + data_rank.compatible(filters_rank - 1), + "Data batch and filters rank do not match (data batch shape: ", + data_shape, + ", filters shape: ", + filters_shape, + ")."); + + convolution::validate::common_attributes(op, num_spatial, pads_begin, pads_end); + } + convolution::apply_padding(op, data_shape, filters_shape, pads_begin, pads_end); + + output_shape.reserve(util::spatial_dim_offset + num_spatial); + output_shape.emplace_back(data_rank.is_static() ? data_shape[0] : dim::inf_bound); + + if (filters_rank.is_static()) { + auto groups = filters_shape[0]; + + if (data_rank.is_static() && filters_shape[2].is_static()) { + NODE_VALIDATION_CHECK( + op, + groups.merge(groups, groups, (data_shape[1] / filters_shape[2].get_length())), + "Input channels dimension of data batch is incompatible with filter groups or input channels."); + } + + groups *= filters_shape[1]; + output_shape.push_back(std::move(groups)); + } else { + output_shape.emplace_back(dim::inf_bound); + } + + convolution::append_spatial_shape(op, data_shape, filters_shape, pads_begin, pads_end, output_shape); + } else { + output_shape = PartialShape::dynamic(); + } + + return {output_shape}; +} +} // namespace v1 +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/max_pool_shape_inference.hpp b/src/core/shape_inference/include/max_pool_shape_inference.hpp index 74dbd4962aecad..806cd3f3db3db2 100644 --- a/src/core/shape_inference/include/max_pool_shape_inference.hpp +++ b/src/core/shape_inference/include/max_pool_shape_inference.hpp @@ -12,44 +12,47 @@ namespace ov { namespace op { namespace v1 { -template -std::vector shape_infer(const MaxPool* op, const std::vector& input_shapes) { - NODE_VALIDATION_CHECK(op, input_shapes.size() == 1); +template +std::vector shape_infer(const MaxPool* op, + const std::vector& input_shapes, + TContainer& pads_begin, + TContainer& pads_end, + const std::map& constant_data = {}) { const auto& data_shape = input_shapes[0]; - const auto dilations = Strides(op->get_kernel().size(), 1); - pooling::update_and_validate_attributes(const_cast(op), data_shape, dilations); - - return {pooling::out_shape_infer(op, data_shape, dilations)}; -} + auto num_spatial = dilations.size(); + pooling::resize_empty_padding(num_spatial, pads_begin, pads_end); + pooling::validate::padding(op, pads_begin, pads_end); + pooling::validate::attributes(op, data_shape, dilations); + pooling::apply_padding(op, data_shape, dilations, pads_begin, pads_end); -template -void shape_infer(const MaxPool* op, const std::vector& input_shapes, std::vector& output_shapes) { - output_shapes = shape_infer(op, input_shapes); + return {pooling::out_shape_infer(op, data_shape, pads_begin, pads_end, dilations)}; } } // namespace v1 namespace v8 { -template -std::vector shape_infer(const MaxPool* op, const std::vector& input_shapes) { +template +std::vector shape_infer(const MaxPool* op, + const std::vector& input_shapes, + TContainer& pads_begin, + TContainer& pads_end, + const std::map& constant_data = {}) { NODE_VALIDATION_CHECK(op, input_shapes.size() == 1); const auto& data_shape = input_shapes[0]; + auto num_spatial = op->get_kernel().size(); auto dilations = op->get_dilations(); if (dilations.empty()) { - dilations.resize(op->get_kernel().size(), 1); + dilations.resize(num_spatial, 1); } - pooling::update_and_validate_attributes(const_cast(op), data_shape, dilations); - - auto output_shape = pooling::out_shape_infer(op, data_shape, dilations); - return {2, output_shape}; -} + pooling::resize_empty_padding(num_spatial, pads_begin, pads_end); + pooling::validate::padding(op, pads_begin, pads_end); + pooling::validate::attributes(op, data_shape, dilations); + pooling::apply_padding(op, data_shape, dilations, pads_begin, pads_end); -template -void shape_infer(const MaxPool* op, const std::vector& input_shapes, std::vector& output_shapes) { - output_shapes = shape_infer(op, input_shapes); + return {2, pooling::out_shape_infer(op, data_shape, pads_begin, pads_end, dilations)}; } } // namespace v8 } // namespace op diff --git a/src/core/shape_inference/include/one_hot_shape_inference.hpp b/src/core/shape_inference/include/one_hot_shape_inference.hpp index 93cd7c22a24769..3cc3d2c5e4a63c 100644 --- a/src/core/shape_inference/include/one_hot_shape_inference.hpp +++ b/src/core/shape_inference/include/one_hot_shape_inference.hpp @@ -48,7 +48,9 @@ inline bool get_data_as_shape_and_validate_sign( shape = PartialShape(data); return true; } else { + OPENVINO_SUPPRESS_DEPRECATED_START return ov::evaluate_as_partial_shape(op->input_value(idx), shape); + OPENVINO_SUPPRESS_DEPRECATED_END } } @@ -62,7 +64,9 @@ void inline resolve_axis(OneHot* op) { const auto& indices_shape = op->get_input_partial_shape(0); if (indices_shape.rank().is_static()) { const auto indices_rank = indices_shape.rank().get_length(); + OPENVINO_SUPPRESS_DEPRECATED_START op->m_axis = ov::normalize_axis(op, op->m_axis, indices_rank + 1, -indices_rank - 1, indices_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } } @@ -94,7 +98,9 @@ void shape_infer(const OneHot* op, if (indices_shape.rank().is_static()) { result_shape = indices_shape; const auto indices_rank = indices_shape.rank().get_length(); + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = ov::normalize_axis(op, op->get_axis(), indices_rank + 1, -indices_rank - 1, indices_rank); + OPENVINO_SUPPRESS_DEPRECATED_END T depth_dim_as_shape; if (utils::one_hot::get_data_as_shape_and_validate_sign(1, op, depth_dim_as_shape, constant_data) && diff --git a/src/core/shape_inference/include/pooling_shape_inference_util.hpp b/src/core/shape_inference/include/pooling_shape_inference_util.hpp index 8832f546f47e0e..baef60e624c1f4 100644 --- a/src/core/shape_inference/include/pooling_shape_inference_util.hpp +++ b/src/core/shape_inference/include/pooling_shape_inference_util.hpp @@ -12,64 +12,35 @@ namespace op { namespace pooling { constexpr size_t spatial_dim_offset = 2; -/** - * @brief Calculate dimension padding required by filter/kernel properties. - * - * Provides pair of padding values as left padding is total value of required padding divided by 2 and right as - * total required padding minus left padding. - * - * @param dim input dimension to calculate its padding. - * @param filter_size Kernel size for input dimension. - * @param dilation Kernel dilation. - * @param stride Kernel stride. - * @return Pair of left, right padding values for input dimension. - */ -template -inline std::pair dim_padding(const TDim& dim, const int64_t kernel_size, const int64_t dilation, int64_t stride) { - if (dim.is_static()) { - const auto dim_size = static_cast(dim.get_length()); - const auto dilated_kernel = ov::util::dim::dilated(kernel_size, dilation); - const int64_t tmp = (dim_size + stride - 1) / stride; - - const auto padding = std::max(0, (tmp - 1) * stride + dilated_kernel - dim_size); - const auto left_padding = padding / 2; - return {left_padding, padding - left_padding}; - } else { - // If input dimension is infinite or interval the padding will be set to 0 - // as operator cannot store paddings for both bounds. - return {0, 0}; - } +namespace validate { +template +void padding(const TOp* op, const TContainer& pads_begin, const TContainer& pads_end) { + const auto num_spatial = op->get_kernel().size(); + NODE_VALIDATION_CHECK(op, + pads_begin.size() == num_spatial, + "Expected pads_begin size to be equal to input size - 2. Got: ", + pads_begin.size()); + NODE_VALIDATION_CHECK(op, + pads_end.size() == num_spatial, + "Expected pads_end size to be equal to input size - 2. Got: ", + pads_end.size()); } template -void update_and_validate_attributes(TOp* op, const TShape& data_shape, const Strides& dilations) { +void attributes(const TOp* op, const TShape& data_shape, const Strides& dilations) { const auto& data_rank = data_shape.rank(); + OPENVINO_SUPPRESS_DEPRECATED_START NODE_VALIDATION_CHECK(op, is_rank_compatible_any_of(data_rank, {3, 4, 5}), "Expected a 3D, 4D or 5D tensor for the input. Got: ", data_shape); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& kernel = op->get_kernel(); - const auto& auto_pad = op->get_auto_pad(); const auto num_spatial = kernel.size(); const auto& strides = op->get_strides(); - if (auto_pad == PadType::VALID || op->get_pads_begin().empty()) { - op->set_pads_begin(Shape(num_spatial, 0)); - } - if (auto_pad == PadType::VALID || op->get_pads_end().empty()) { - op->set_pads_end(Shape(num_spatial, 0)); - } - - NODE_VALIDATION_CHECK(op, - op->get_pads_begin().size() == num_spatial, - "Expected pads_begin size to be equal to input size - 2. Got: ", - op->get_pads_begin().size()); - NODE_VALIDATION_CHECK(op, - op->get_pads_end().size() == num_spatial, - "Expected pads_end size to be equal to input size - 2. Got: ", - op->get_pads_end().size()); NODE_VALIDATION_CHECK(op, strides.size() == num_spatial, "Expected strides size to be equal to input size - 2. Got: ", @@ -79,31 +50,10 @@ void update_and_validate_attributes(TOp* op, const TShape& data_shape, const Str "Expected dilations size to be equal to kernel size. Got: ", dilations.size()); - if (data_rank.is_static()) { - NODE_VALIDATION_CHECK(op, - num_spatial == (data_shape.size() - spatial_dim_offset), - "Expected kernel size to be equal to input size - 2. Got: ", - num_spatial); - - if (auto_pad == PadType::SAME_UPPER || auto_pad == PadType::SAME_LOWER) { - Shape pads_begin, pads_end; - pads_begin.reserve(num_spatial); - pads_end.reserve(num_spatial); - - auto data_dim = data_shape.cbegin() + spatial_dim_offset; - auto pad_begin_ins = std::back_inserter(pads_begin); - auto pad_end_ins = std::back_inserter(pads_end); - auto& pad_left = auto_pad == PadType::SAME_UPPER ? pad_begin_ins : pad_end_ins; - auto& pad_right = auto_pad == PadType::SAME_UPPER ? pad_end_ins : pad_begin_ins; - - for (size_t i = 0; i < num_spatial; ++i, ++pad_left, ++pad_right, ++data_dim) { - std::tie(*pad_left, *pad_right) = dim_padding(*data_dim, kernel[i], dilations[i], strides[i]); - } - - op->set_pads_begin(pads_begin); - op->set_pads_end(std::move(pads_end)); - } - } + NODE_VALIDATION_CHECK(op, + data_rank.is_dynamic() || num_spatial == (data_shape.size() - spatial_dim_offset), + "Expected kernel size to be equal to input size - 2. Got: ", + num_spatial); constexpr auto is_zero = cmp::Equal(0); NODE_VALIDATION_CHECK(op, @@ -115,6 +65,65 @@ void update_and_validate_attributes(TOp* op, const TShape& data_shape, const Str "Kernel dilations has zero dimension(s). ", dilations); } +} // namespace validate + +/** + * @brief Resize paddings if empty to number of spatial dimensions. + * + * @param num_spatial Number of spatial dimensions. + * @param pads_begin Begin padding to resize. + * @param pads_end End padding to resize. + */ +template +void resize_empty_padding(const size_t num_spatial, TContainer& pads_begin, TContainer& pads_end) { + if (pads_begin.empty()) { + pads_begin.resize(num_spatial); + } + + if (pads_end.empty()) { + pads_end.resize(num_spatial); + } +} + +/** + * @brief Apply pooling operator padding depends on auto pad value. + * + * @param op Pointer to Pooling operator to apply padding. + * @param data_shape Shape infer data input shape. + * @param dilations Kernel dilations. + * @param pads_begin Padding begin to update. + * @param pads_end Padding end to update. + */ +template +void apply_padding(const TOp* op, + const TShape& data_shape, + const Strides& dilations, + TContainer& pads_begin, + TContainer& pads_end) { + const auto& auto_pad = op->get_auto_pad(); + if (data_shape.rank().is_static() && (auto_pad == PadType::SAME_UPPER || auto_pad == PadType::SAME_LOWER)) { + const auto& kernel = op->get_kernel(); + const auto& strides = op->get_strides(); + const auto num_spatial = kernel.size(); + pads_begin.reserve(num_spatial); + pads_end.reserve(num_spatial); + + auto data_dim = data_shape.cbegin() + spatial_dim_offset; + auto pad_b = auto_pad == PadType::SAME_UPPER ? pads_begin.begin() : pads_end.begin(); + auto pad_e = auto_pad == PadType::SAME_UPPER ? pads_end.begin() : pads_begin.begin(); + + for (size_t i = 0; i < num_spatial; ++i, ++pad_b, ++pad_e, ++data_dim) { + using namespace ov::util; + std::tie(*pad_b, *pad_e) = dim::padding(*data_dim, kernel[i], dilations[i], strides[i]); + } + } else if (auto_pad == PadType::VALID) { + std::fill_n(pads_begin.begin(), pads_begin.size(), 0); + std::fill_n(pads_end.begin(), pads_end.size(), 0); + } else if (op->get_auto_pad() == op::PadType::EXPLICIT) { + std::copy(op->get_pads_begin().begin(), op->get_pads_begin().end(), pads_begin.begin()); + std::copy(op->get_pads_end().begin(), op->get_pads_end().end(), pads_end.begin()); + } +} template void valid_dilated_kernel_with_dim(const TOp* op, const size_t kernel, const TDim& dim, const size_t axis) { @@ -144,8 +153,23 @@ void valid_dilated_kernel_with_padding(const TOp* op, const size_t pad_end, const size_t axis) {} -template -TShape spatial_shape_infer(const TOp* op, const TShape& data_shape, const Strides& dilations) { +/** + * @brief Append spatial shape to the end of output shape for pooling operator shape inference result. + * + * @param op Pointer to pooling operator. + * @param data_shape Shape inference input pooling data shape. + * @param pads_begin Pooling pads begin. + * @param pads_end Pooling pads end. + * @param dilations Kernel dilations. + * @param out_shape Output shape for appending the spatial shape of pooling + */ +template +void append_spatial_shape(const TOp* op, + const TShape& data_shape, + const TContainer& pads_begin, + const TContainer& pads_end, + const Strides& dilations, + TShape& out_shape) { using namespace ov::util; const auto spatial_num = data_shape.size() - spatial_dim_offset; const auto is_ceil_mode = op->get_rounding_type() == RoundingType::CEIL; @@ -154,12 +178,7 @@ TShape spatial_shape_infer(const TOp* op, const TShape& data_shape, const Stride using TDim = typename TShape::value_type; const auto& dim_divide = is_ceil_mode ? dim::ceil_div : dim::floor_div; - TShape out_shape; - out_shape.reserve(spatial_num); - auto data_dim = data_shape.cbegin() + spatial_dim_offset; - const auto& pads_begin = op->get_pads_begin(); - const auto& pads_end = op->get_pads_end(); const auto& kernel = op->get_kernel(); const auto& stride = op->get_strides(); @@ -183,15 +202,18 @@ TShape spatial_shape_infer(const TOp* op, const TShape& data_shape, const Stride out_shape.emplace_back(dim::inf_bound); } } - - return out_shape; } /** * @brief Shape inference helper used for pooling operators such Max Pool, Avg Pool. */ -template -TShape out_shape_infer(const TOp* op, const TShape& data_shape, const Strides& dilations) { +template +TShape out_shape_infer(const TOp* op, + const TShape& data_shape, + const TContainer& pads_begin, + const TContainer& pads_end, + const Strides& dilations) { + const auto out_rank_size = spatial_dim_offset + op->get_kernel().size(); TShape out_shape; if (data_shape.rank().is_static()) { const auto& batch_size = data_shape[0]; @@ -202,8 +224,9 @@ TShape out_shape_infer(const TOp* op, const TShape& data_shape, const Strides& d channel_count.is_dynamic() || channel_count.get_length() > 0, "Channel count is zero."); - out_shape = spatial_shape_infer(op, data_shape, dilations); - out_shape.insert(out_shape.begin(), data_shape.begin(), data_shape.begin() + spatial_dim_offset); + out_shape.reserve(out_rank_size); + std::copy_n(data_shape.begin(), spatial_dim_offset, std::back_inserter(out_shape)); + pooling::append_spatial_shape(op, data_shape, pads_begin, pads_end, dilations, out_shape); } else { out_shape.insert(out_shape.begin(), spatial_dim_offset + op->get_kernel().size(), Dimension::dynamic()); } @@ -228,10 +251,12 @@ TShape out_shape_infer(const TOp* op, const auto& data_rank = data_shape.rank(); + OPENVINO_SUPPRESS_DEPRECATED_START NODE_VALIDATION_CHECK(op, is_rank_compatible_any_of(data_rank, {3, 4, 5}), "Expected a 3D, 4D or 5D tensor for the input. Got: ", data_shape); + OPENVINO_SUPPRESS_DEPRECATED_END TShape output_shape; if (data_rank.is_static()) { diff --git a/src/core/shape_inference/include/psroi_pooling_shape_inference.hpp b/src/core/shape_inference/include/psroi_pooling_shape_inference.hpp new file mode 100644 index 00000000000000..c47b1211e669f5 --- /dev/null +++ b/src/core/shape_inference/include/psroi_pooling_shape_inference.hpp @@ -0,0 +1,99 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "compare.hpp" +#include "dimension_util.hpp" +#include "openvino/op/roi_pooling.hpp" +#include "roi_pooling_shape_inference.hpp" + +namespace ov { +namespace op { +namespace psroi_pooling { +namespace validate { +template +void feat_input_shape(const TROIPooling* op, const TShape feat_shape) { + using namespace ov::util; + + roi_pooling::validate::feat_intput_shape(op, feat_shape); + + if (feat_shape.rank().is_static()) { + const auto& mode = op->get_mode(); + const auto& num_channels = feat_shape[1]; + if (mode == "average") { + const auto group_area = op->get_group_size() * op->get_group_size(); + NODE_VALIDATION_CHECK( + op, + num_channels.compatible(group_area * op->get_output_dim()), + "Number of input's channels must be a multiply of output_dim * group_size * group_size"); + } else if (mode == "bilinear") { + const auto bins_area = op->get_spatial_bins_x() * op->get_spatial_bins_y(); + NODE_VALIDATION_CHECK( + op, + num_channels.compatible(bins_area * op->get_output_dim()), + "Number of input's channels must be a multiply of output_dim * spatial_bins_x * spatial_bins_y"); + } + } +} + +template +void output_group_attr(const TROIPooling* op) { + NODE_VALIDATION_CHECK(op, op->get_group_size() > 0, "group_size has to be greater than 0"); +} + +template +void bins_attr(const TROIPooling* op) { + if (op->get_mode() == "bilinear") { + NODE_VALIDATION_CHECK(op, op->get_spatial_bins_x() > 0, "spatial_bins_x has to be greater than 0"); + NODE_VALIDATION_CHECK(op, op->get_spatial_bins_y() > 0, "spatial_bins_y has to be greater than 0"); + } +} + +template +void mode_attr(const TROIPooling* op) { + const auto& mode = op->get_mode(); + NODE_VALIDATION_CHECK(op, + mode == "average" || mode == "bilinear", + "Expected 'average' or 'bilinear' mode. Got " + mode); +} +} // namespace validate +} // namespace psroi_pooling + +namespace v0 { +template +std::vector shape_infer(const PSROIPooling* op, const std::vector& input_shapes) { + NODE_VALIDATION_CHECK(op, input_shapes.size() == 2); + using namespace ov::util; + + const auto& feat_shape = input_shapes[0]; + const auto& rois_shape = input_shapes[1]; + + psroi_pooling::validate::mode_attr(op); + psroi_pooling::validate::output_group_attr(op); + psroi_pooling::validate::bins_attr(op); + roi_pooling::validate::scale_attr(op); + + psroi_pooling::validate::feat_input_shape(op, feat_shape); + roi_pooling::validate::rois_input_shape(op, rois_shape); + + TShape out_shape; + out_shape.reserve(4); + + out_shape.emplace_back(rois_shape.rank().is_static() ? rois_shape[0] : dim::inf_bound); + out_shape.emplace_back(op->get_output_dim()); + out_shape.insert(out_shape.end(), 2, op->get_group_size()); + + return {out_shape}; +} + +template +void shape_infer(const PSROIPooling* op, const std::vector& input_shapes, std::vector& output_shapes) { + output_shapes = shape_infer(op, input_shapes); +} +} // namespace v0 +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/range_shape_inference.hpp b/src/core/shape_inference/include/range_shape_inference.hpp index fa84a7a0a9393b..72da58cda67124 100644 --- a/src/core/shape_inference/include/range_shape_inference.hpp +++ b/src/core/shape_inference/include/range_shape_inference.hpp @@ -37,7 +37,9 @@ inline bool get_data_as_double( const std::map>& constant_data) { if (constant_data.count(idx)) { axes_value = ov::opset1::Constant(constant_data.at(idx)).cast_vector(); + OPENVINO_SUPPRESS_DEPRECATED_START } else if (const auto& constant = ov::get_constant_from_source(op->input_value(idx))) { + OPENVINO_SUPPRESS_DEPRECATED_END axes_value = constant->cast_vector(); } else { return false; diff --git a/src/core/shape_inference/include/reduce_shape_inference.hpp b/src/core/shape_inference/include/reduce_shape_inference.hpp index 5d26e1ca26f28d..e24b8d94d6efeb 100644 --- a/src/core/shape_inference/include/reduce_shape_inference.hpp +++ b/src/core/shape_inference/include/reduce_shape_inference.hpp @@ -32,7 +32,9 @@ void reduce_shape_infer(const ov::op::util::ReductionBase* op, bool axes_are_known = get_data_as_int64(1, op, axes_val, constant_data); if (data_rank.is_static() && axes_are_known) { + OPENVINO_SUPPRESS_DEPRECATED_START ov::normalize_axes(op, data_rank.get_length(), axes_val); + OPENVINO_SUPPRESS_DEPRECATED_END if (keep_dims) { output_shape = input_shape; diff --git a/src/core/shape_inference/include/region_yolo_shape_inference.hpp b/src/core/shape_inference/include/region_yolo_shape_inference.hpp index 23baea9a51e5ce..5363bf0362a9bd 100644 --- a/src/core/shape_inference/include/region_yolo_shape_inference.hpp +++ b/src/core/shape_inference/include/region_yolo_shape_inference.hpp @@ -31,7 +31,9 @@ void shape_infer(const RegionYolo* op, const std::vector& input_shapes, std:: if (op->m_do_softmax) { output_shape.resize(0); + OPENVINO_SUPPRESS_DEPRECATED_START auto axis = ov::normalize_axis(op, op->m_axis, input_rank); + OPENVINO_SUPPRESS_DEPRECATED_END DimType flat_dim = 1; for (int64_t i = 0; i < axis; i++) { output_shape.push_back(input_shape[i]); diff --git a/src/core/shape_inference/include/reverse_sequence_shape_inference.hpp b/src/core/shape_inference/include/reverse_sequence_shape_inference.hpp index f51b3265208e99..415f5e7f01f669 100644 --- a/src/core/shape_inference/include/reverse_sequence_shape_inference.hpp +++ b/src/core/shape_inference/include/reverse_sequence_shape_inference.hpp @@ -30,7 +30,9 @@ std::vector shape_infer(const ReverseSequence* op, const std::vectorget_origin_batch_axis(), data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END DimType merged_sequence_length; NODE_VALIDATION_CHECK( op, diff --git a/src/core/shape_inference/include/roi_pooling_shape_inference.hpp b/src/core/shape_inference/include/roi_pooling_shape_inference.hpp new file mode 100644 index 00000000000000..0de62e9c090724 --- /dev/null +++ b/src/core/shape_inference/include/roi_pooling_shape_inference.hpp @@ -0,0 +1,111 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "compare.hpp" +#include "dimension_util.hpp" +#include "openvino/op/roi_pooling.hpp" + +namespace ov { +namespace op { +namespace roi_pooling { +namespace validate { +template +void feat_intput_shape(const TROIPooling* op, const TShape& feat_shape) { + NODE_VALIDATION_CHECK(op, + feat_shape.rank().compatible(4), + "Expected a 4D tensor for the feature maps input. Got: ", + feat_shape); +} + +template +void rois_input_shape(const TROIPooling* op, const TShape rois_shape) { + if (rois_shape.rank().is_static()) { + NODE_VALIDATION_CHECK(op, + rois_shape.size() == 2, + "Expected a 2D tensor for the ROIs input with box coordinates. Got: ", + rois_shape); + + NODE_VALIDATION_CHECK(op, + rois_shape[1].compatible(5), + "The second dimension of ROIs input should contain batch id and box coordinates. ", + "This dimension is expected to be equal to 5. Got: ", + rois_shape[1]); + } +} + +template +void output_roi_attr(const TROIPooling* op) { + const auto& out_roi = op->get_output_roi(); + + NODE_VALIDATION_CHECK(op, + out_roi.size() == 2, + "The dimension of pooled size is expected to be equal to 2. Got: ", + out_roi.size()); + + NODE_VALIDATION_CHECK(op, + std::none_of(out_roi.cbegin(), out_roi.cend(), cmp::Less(1)), + "Pooled size attributes pooled_h and pooled_w should should be positive integers. Got: ", + out_roi[0], + " and: ", + out_roi[1], + "respectively"); +} + +template +void scale_attr(const TROIPooling* op) { + const auto scale = op->get_spatial_scale(); + NODE_VALIDATION_CHECK(op, + std::isnormal(scale) && !std::signbit(scale), + "The spatial scale attribute should be a positive floating point number. Got: ", + scale); +} + +template +void method_attr(const TROIPooling* op) { + const auto& method = op->get_method(); + NODE_VALIDATION_CHECK(op, + method == "max" || method == "bilinear", + "Pooling method attribute should be either \'max\' or \'bilinear\'. Got: ", + method); +} +} // namespace validate +} // namespace roi_pooling + +namespace v0 { +template +std::vector shape_infer(const ROIPooling* op, const std::vector& input_shapes) { + NODE_VALIDATION_CHECK(op, input_shapes.size() == 2); + using namespace ov::util; + + const auto& feat_shape = input_shapes[0]; + const auto& rois_shape = input_shapes[1]; + const auto& feat_rank = feat_shape.rank(); + + roi_pooling::validate::feat_intput_shape(op, feat_shape); + roi_pooling::validate::rois_input_shape(op, rois_shape); + roi_pooling::validate::output_roi_attr(op); + roi_pooling::validate::scale_attr(op); + roi_pooling::validate::method_attr(op); + + TShape out_shape; + out_shape.reserve(4); + + out_shape.emplace_back(rois_shape.rank().is_static() ? rois_shape[0] : dim::inf_bound); + out_shape.emplace_back(feat_rank.is_static() ? feat_shape[1] : dim::inf_bound); + std::copy(op->get_output_roi().cbegin(), op->get_output_roi().cend(), std::back_inserter(out_shape)); + + return {out_shape}; +} + +template +void shape_infer(const ROIPooling* op, const std::vector& input_shapes, std::vector& output_shapes) { + output_shapes = shape_infer(op, input_shapes); +} +} // namespace v0 +} // namespace op +} // namespace ov diff --git a/src/core/shape_inference/include/roll_shape_inference.hpp b/src/core/shape_inference/include/roll_shape_inference.hpp index 71052194150689..c985b2abff0b3b 100644 --- a/src/core/shape_inference/include/roll_shape_inference.hpp +++ b/src/core/shape_inference/include/roll_shape_inference.hpp @@ -40,7 +40,9 @@ std::vector shape_infer(const Roll* op, if (data_pshape.rank().is_static()) { if (const auto& axes = get_input_const_data_as(op, 2, constant_data)) { + OPENVINO_SUPPRESS_DEPRECATED_START ov::normalize_axes(op, data_pshape.size(), *axes); + OPENVINO_SUPPRESS_DEPRECATED_END } } diff --git a/src/core/shape_inference/include/scatter_elements_update_shape_inference.hpp b/src/core/shape_inference/include/scatter_elements_update_shape_inference.hpp index 8db69b87edcc7a..4233644354c339 100644 --- a/src/core/shape_inference/include/scatter_elements_update_shape_inference.hpp +++ b/src/core/shape_inference/include/scatter_elements_update_shape_inference.hpp @@ -23,11 +23,13 @@ std::vector shape_infer(const ScatterElementsUpdate* op, const auto& updates_shape = input_shapes[2]; const auto& axis_shape = input_shapes[3]; + OPENVINO_SUPPRESS_DEPRECATED_START NODE_VALIDATION_CHECK(op, is_rank_compatible_any_of(axis_shape.rank(), {0, 1}), "Axis input shape are required to be scalar or 1D tensor. ", "Got: ", axis_shape); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& data_rank = data_shape.rank(); const auto& indices_rank = indices_shape.rank(); @@ -50,7 +52,9 @@ std::vector shape_infer(const ScatterElementsUpdate* op, if (data_shape.rank().is_static()) { if (const auto axis_input = get_input_const_data_as(op, 3, constant_data)) { + OPENVINO_SUPPRESS_DEPRECATED_START ov::normalize_axis(op, (*axis_input)[0], data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } } return {data_shape}; diff --git a/src/core/shape_inference/include/shuffle_channels_shape_inference.hpp b/src/core/shape_inference/include/shuffle_channels_shape_inference.hpp index fd54069ea5c294..5067f97224b1af 100644 --- a/src/core/shape_inference/include/shuffle_channels_shape_inference.hpp +++ b/src/core/shape_inference/include/shuffle_channels_shape_inference.hpp @@ -26,7 +26,9 @@ std::vector shape_infer(const ShuffleChannels* op, const std::vector= 1, "The input tensor's shape is expected to be at least 1D."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis_zb = static_cast(normalize_axis(op, op->get_axis(), input_shape_rank)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& channel_dim = input_shape[axis_zb]; NODE_VALIDATION_CHECK(op, channel_dim.is_dynamic() || (channel_dim.get_length() % group) == 0, diff --git a/src/core/shape_inference/include/slice_shape_inference.hpp b/src/core/shape_inference/include/slice_shape_inference.hpp index 50fe6b9d42deff..4894b8d16866e8 100644 --- a/src/core/shape_inference/include/slice_shape_inference.hpp +++ b/src/core/shape_inference/include/slice_shape_inference.hpp @@ -107,7 +107,9 @@ void shape_infer(const Slice* op, "Slice `axes` input must have compatible shape with `start`, `stop`, `step` inputs."); if (auto axes = get_input_const_data_as(op, 4, constant_data)) { + OPENVINO_SUPPRESS_DEPRECATED_START ov::normalize_axes(op, input_shape.rank().get_length(), *axes); + OPENVINO_SUPPRESS_DEPRECATED_END axes_map.add(*axes); NODE_VALIDATION_CHECK(op, axes_map.is_valid, "Slice values in `axes` input must be unique."); } diff --git a/src/core/shape_inference/include/slice_shape_inference_utils.hpp b/src/core/shape_inference/include/slice_shape_inference_utils.hpp index 9b33900692b2e1..70eb8ea7299c63 100644 --- a/src/core/shape_inference/include/slice_shape_inference_utils.hpp +++ b/src/core/shape_inference/include/slice_shape_inference_utils.hpp @@ -4,11 +4,10 @@ #pragma once -#include -#include - +#include "openvino/op/constant.hpp" #include "sequnce_generator.hpp" #include "utils.hpp" +#include "validation_util.hpp" namespace ov { namespace internal { @@ -161,8 +160,8 @@ inline int64_t get_sliced_value(const int64_t& dim, const int64_t& start, const } lb = min_bound; } else { - lb = clip(normalize(start, norm_dim), min_bound, lower_max); - ub = clip(normalize(stop, norm_dim), upper_min, norm_dim); + lb = ov::util::clip(ov::util::normalize(start, norm_dim), min_bound, lower_max); + ub = ov::util::clip(ov::util::normalize(stop, norm_dim), upper_min, norm_dim); } // Calculate sliced value from bounds and step. @@ -195,7 +194,9 @@ inline element::Type get_input_const_element_type(const ov::Node* op, const std::map& constant_data = {}) { if (constant_data.count(idx)) { return constant_data.at(idx)->get_element_type(); + OPENVINO_SUPPRESS_DEPRECATED_START } else if (const auto& constant = ov::get_constant_from_source(op->input_value(idx))) { + OPENVINO_SUPPRESS_DEPRECATED_END return constant->get_element_type(); } else { return element::undefined; diff --git a/src/core/shape_inference/include/space_to_batch_shape_inference.hpp b/src/core/shape_inference/include/space_to_batch_shape_inference.hpp index 7cc04ec0ac34b0..9b239e00fc59a7 100644 --- a/src/core/shape_inference/include/space_to_batch_shape_inference.hpp +++ b/src/core/shape_inference/include/space_to_batch_shape_inference.hpp @@ -59,7 +59,7 @@ std::vector shape_infer(const SpaceToBatch* op, auto blocks = get_input_const_data_as(op, 1, constant_data); if (blocks) { - TVal block_prod = std::accumulate(begin(*blocks), end(*blocks), 1, std::multiplies()); + TVal block_prod = std::accumulate(begin(*blocks), end(*blocks), int64_t(1), std::multiplies()); out_shape.push_back(data_shape[0] * block_prod); } else { out_shape.emplace_back(dim::inf_bound); diff --git a/src/core/shape_inference/include/split_shape_inference.hpp b/src/core/shape_inference/include/split_shape_inference.hpp index a7e2b0635b5d3e..103415e4427a31 100644 --- a/src/core/shape_inference/include/split_shape_inference.hpp +++ b/src/core/shape_inference/include/split_shape_inference.hpp @@ -52,7 +52,9 @@ void shape_infer(const Split* op, axes_values.size(), " axes"); + OPENVINO_SUPPRESS_DEPRECATED_START auto axis = ov::normalize_axis(op, axes_values[0], data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END if (data_ps[axis].is_static()) { const auto dimension_at_axis = data_ps[axis].get_length(); diff --git a/src/core/shape_inference/include/squeeze_shape_inference.hpp b/src/core/shape_inference/include/squeeze_shape_inference.hpp index 7318de03d71e46..8c78c9e7f0cfeb 100644 --- a/src/core/shape_inference/include/squeeze_shape_inference.hpp +++ b/src/core/shape_inference/include/squeeze_shape_inference.hpp @@ -42,14 +42,18 @@ void shape_infer(const Squeeze* op, unique_axes.reset(new std::set()); } else if (number_of_inputs == 2) { const auto& axes_shape = input_shapes[1]; + OPENVINO_SUPPRESS_DEPRECATED_START NODE_VALIDATION_CHECK(op, axes_shape.is_dynamic() || is_rank_compatible_any_of(axes_shape.rank(), {0, 1}), "Second input (axes) should not be of rank higher than 1. Got: ", axes_shape.rank().get_length()); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector axes; if (arg_rank.is_static() && axes_shape.is_static() && get_data_as_int64(1, op, axes, constant_data)) { + OPENVINO_SUPPRESS_DEPRECATED_START normalize_axes(op, arg_rank.get_length(), axes); + OPENVINO_SUPPRESS_DEPRECATED_END unique_axes.reset(new std::set(axes.cbegin(), axes.cend())); } } else { diff --git a/src/core/shape_inference/include/tile_shape_inference.hpp b/src/core/shape_inference/include/tile_shape_inference.hpp index abc220b999ccd3..465e4a423974cf 100644 --- a/src/core/shape_inference/include/tile_shape_inference.hpp +++ b/src/core/shape_inference/include/tile_shape_inference.hpp @@ -11,6 +11,15 @@ namespace ov { namespace op { namespace v0 { +template +struct NegativeToZero { + NegativeToZero() = default; + template + T operator()(const U u) const { + return static_cast(std::max(0, ov::util::InTypeRange()(u))); + } +}; + template std::vector shape_infer(const Tile* op, const std::vector& input_shapes, @@ -28,9 +37,7 @@ std::vector shape_infer(const Tile* op, T output_shape; // Get repeats and pre process values - auto negative_repeats_to_zero = [](const TDimValue v) -> TDimValue { - return std::max(0, ov::util::InTypeRange()(v)); - }; + constexpr auto negative_repeats_to_zero = NegativeToZero(); auto repeats = get_input_const_data_as_shape(op, 1, constant_data, negative_repeats_to_zero); diff --git a/src/core/shape_inference/include/topk_shape_inference.hpp b/src/core/shape_inference/include/topk_shape_inference.hpp index 93893a450f1dcf..7fd02ec38e98ab 100644 --- a/src/core/shape_inference/include/topk_shape_inference.hpp +++ b/src/core/shape_inference/include/topk_shape_inference.hpp @@ -67,7 +67,9 @@ std::vector shape_infer(const util::TopKBase* op, auto output_shape = input_shape; if (input_shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_axis = ov::normalize_axis(op, op->get_provided_axis(), input_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END auto& dim_axis = output_shape[normalized_axis]; if (auto k_as_shape = get_input_const_data_as_shape(op, 1, constant_data, GetK(op))) { diff --git a/src/core/shape_inference/include/transpose_shape_inference.hpp b/src/core/shape_inference/include/transpose_shape_inference.hpp index 91060d69c108d2..c3c74d2c39d08e 100644 --- a/src/core/shape_inference/include/transpose_shape_inference.hpp +++ b/src/core/shape_inference/include/transpose_shape_inference.hpp @@ -26,14 +26,18 @@ T calc_output_shape(const Transpose* const op, const T& input_shape, std::vector const auto output_rank = input_shape.size(); if (axes_order.empty()) { + OPENVINO_SUPPRESS_DEPRECATED_START generate_transpose_default_order(axes_order, output_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else { + OPENVINO_SUPPRESS_DEPRECATED_START NODE_VALIDATION_CHECK(op, is_valid_axes_order(axes_order, output_rank), "Permutation ", AxisVector(axes_order.begin(), axes_order.end()), " is not valid for input shape ", input_shape); + OPENVINO_SUPPRESS_DEPRECATED_END } T output_shape; diff --git a/src/core/shape_inference/include/unsqueeze_shape_inference.hpp b/src/core/shape_inference/include/unsqueeze_shape_inference.hpp index 3bca49649d49bf..0ea9751f48871b 100644 --- a/src/core/shape_inference/include/unsqueeze_shape_inference.hpp +++ b/src/core/shape_inference/include/unsqueeze_shape_inference.hpp @@ -12,10 +12,12 @@ namespace v0 { template void check_unsqueeze_axes_rank(const TOp* op, const Rank& rank) { + OPENVINO_SUPPRESS_DEPRECATED_START NODE_VALIDATION_CHECK(op, is_rank_compatible_any_of(rank, {0, 1}), "Second input (axes) should not be of rank higher than 1. Got: ", rank); + OPENVINO_SUPPRESS_DEPRECATED_END } template @@ -40,7 +42,9 @@ void shape_infer(const Unsqueeze* op, const auto expanded_rank = arg_shape.rank().get_length() + unique_axes.size(); // Normalize then remove repeated axes after normalization. + OPENVINO_SUPPRESS_DEPRECATED_START normalize_axes(op, expanded_rank, unique_axes); + OPENVINO_SUPPRESS_DEPRECATED_END const std::set axes(unique_axes.begin(), unique_axes.end()); out_shape = arg_shape; diff --git a/src/core/shape_inference/include/utils.hpp b/src/core/shape_inference/include/utils.hpp index d8b85271c40943..0ceb89db413e11 100644 --- a/src/core/shape_inference/include/utils.hpp +++ b/src/core/shape_inference/include/utils.hpp @@ -281,7 +281,9 @@ std::unique_ptr get_input_const_data_as(const ov::Node* op, if (constant_data.count(idx)) { return std::unique_ptr( new TRes(get_tensor_data_as(constant_data.at(idx).get(), std::forward(func)))); + OPENVINO_SUPPRESS_DEPRECATED_START } else if (const auto& constant = ov::get_constant_from_source(op->input_value(idx))) { + OPENVINO_SUPPRESS_DEPRECATED_END const auto& et = constant->get_element_type(); const auto& shape = constant->get_shape(); return std::unique_ptr(new TRes(get_raw_data_as(et, @@ -325,7 +327,9 @@ std::unique_ptr get_input_const_data_as_shape(const ov::Node* op, return std::unique_ptr(new TShape(std::move(*d))); } else { PartialShape shape; + OPENVINO_SUPPRESS_DEPRECATED_START if (ov::evaluate_as_partial_shape(op->input_value(idx), shape)) { + OPENVINO_SUPPRESS_DEPRECATED_END return std::unique_ptr(new TShape(std::move(shape))); } } @@ -459,7 +463,9 @@ inline bool get_data_as_shape( shape = ov::PartialShape(ov::opset1::Constant(constant_data.at(idx)).cast_vector()); return true; } else { + OPENVINO_SUPPRESS_DEPRECATED_START return ov::evaluate_as_partial_shape(op->input_value(idx), shape); + OPENVINO_SUPPRESS_DEPRECATED_END } } diff --git a/src/core/shape_inference/include/variadic_split_shape_inference.hpp b/src/core/shape_inference/include/variadic_split_shape_inference.hpp index fa1a56c7739f08..59efd8da9feeed 100644 --- a/src/core/shape_inference/include/variadic_split_shape_inference.hpp +++ b/src/core/shape_inference/include/variadic_split_shape_inference.hpp @@ -52,7 +52,9 @@ void shape_infer(const VariadicSplit* op, " axes"); const auto axis_val = axis_values[0]; // Adjust split axis in case of negatives + OPENVINO_SUPPRESS_DEPRECATED_START const int64_t axis = ov::normalize_axis(op, axis_val, data_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END if (get_data_as_int64(2, op, split_lengths, constant_data)) { // Adjust split lengths in case of negatives diff --git a/src/core/src/any.cpp b/src/core/src/any.cpp index d1ecbd0716b27e..be1018dc8ea917 100644 --- a/src/core/src/any.cpp +++ b/src/core/src/any.cpp @@ -217,13 +217,13 @@ void Read>::operator()( } void Read::operator()(std::istream& is, AnyMap& map) const { - std::string key, value; char c; is >> c; OPENVINO_ASSERT(c == '{', "Failed to parse ov::AnyMap. Starting symbols is not '{', it's ", c); while (c != '}') { + std::string key, value; std::getline(is, key, ':'); size_t enclosed_container_level = 0; diff --git a/src/core/src/bound_evaluate.cpp b/src/core/src/bound_evaluate.cpp index 930d71b80cefee..a1ac9648b72c42 100644 --- a/src/core/src/bound_evaluate.cpp +++ b/src/core/src/bound_evaluate.cpp @@ -179,14 +179,26 @@ ov::Tensor or_tensor(const ov::Tensor& lhs, const ov::Tensor& rhs) { } struct TensorVectorCmp { + // Comparing Tensor vectors as numbers composed with pointers as digits. + // Indexed loop used to preserve order of comparison. bool operator()(const ov::TensorVector& lhs, const ov::TensorVector& rhs) const { - auto rhs_it = rhs.begin(); - return std::any_of(lhs.begin(), lhs.end(), [&rhs_it](const ov::Tensor& lhs) { - bool is_less = - (lhs && *rhs_it) ? lhs.data() < rhs_it->data() : static_cast(lhs) < static_cast(*rhs_it); - ++rhs_it; - return is_less; - }); + const auto lhs_size = lhs.size(); + const auto rhs_size = rhs.size(); + + if (lhs_size < rhs_size) + return true; + if (lhs_size > rhs_size) + return false; + + for (size_t i = 0; i < lhs_size; ++i) { + if (lhs[i].data() < rhs[i].data()) + return true; + if (lhs[i].data() > rhs[i].data()) + return false; + } + + // if all equals + return false; } }; @@ -281,17 +293,14 @@ bool ov::interval_bound_evaluator(const Node* node, auto low_1 = ov::evaluate_lower_bound(node->get_input_source_output(1)); auto up_0 = ov::evaluate_upper_bound(node->get_input_source_output(0)); auto up_1 = ov::evaluate_upper_bound(node->get_input_source_output(1)); + if (!low_0 || !low_1 || !up_0 || !up_1) + return false; std::set input_variants = {{low_0, low_1}, {low_0, up_1}, {up_0, low_1}, {up_0, up_1}}; - for (const auto& variant_of_input_vector : input_variants) - for (const auto& input_tensor : variant_of_input_vector) - if (!input_tensor) - return false; - if (input_variants.size() == 1) return node->evaluate(upper_output_values, *input_variants.begin()) && node->evaluate(lower_output_values, *input_variants.begin()); @@ -326,9 +335,10 @@ bool ov::interval_bound_evaluator(const Node* node, } unsqueezed_output_variants.push_back(vector_of_unsqueezed_output_variants); } - + OPENVINO_SUPPRESS_DEPRECATED_START auto input_0_maximum_value = ngraph::get_constant_max_of_type(low_0.get_element_type()); auto input_1_maximum_value = ngraph::get_constant_max_of_type(low_1.get_element_type()); + OPENVINO_SUPPRESS_DEPRECATED_END if (input_0_maximum_value == nullptr || input_1_maximum_value == nullptr) return false; @@ -433,7 +443,9 @@ bool ov::default_label_evaluator(const Node* node, for (size_t i = 0; i < inputs_count; ++i) { if (std::find(labeled_inputs.begin(), labeled_inputs.end(), i) != labeled_inputs.end()) { auto labels = node->get_input_tensor(i).get_value_label(); + OPENVINO_SUPPRESS_DEPRECATED_START if (!has_no_labels(labels) && !has_any_input_labels) { + OPENVINO_SUPPRESS_DEPRECATED_END has_any_input_labels = true; } diff --git a/src/core/src/descriptor/tensor.cpp b/src/core/src/descriptor/tensor.cpp index 5a7a4c24a69f34..18a024835bdd39 100644 --- a/src/core/src/descriptor/tensor.cpp +++ b/src/core/src/descriptor/tensor.cpp @@ -38,17 +38,13 @@ ov::descriptor::Tensor::Tensor(const element::Type& element_type, OPENVINO_SUPPRESS_DEPRECATED_START void ov::descriptor::Tensor::set_tensor_type(const element::Type& element_type, const PartialShape& pshape) { set_element_type(element_type); - set_partial_shape(pshape); + m_partial_shape = pshape; + m_shape_changed = true; } void ov::descriptor::Tensor::set_element_type(const element::Type& element_type) { m_element_type = element_type; } - -void ov::descriptor::Tensor::set_partial_shape(const PartialShape& partial_shape) { - m_partial_shape = partial_shape; - m_shape_changed = true; -} OPENVINO_SUPPRESS_DEPRECATED_END void ov::descriptor::Tensor::invalidate_values() { @@ -100,10 +96,8 @@ const ov::Shape& ov::descriptor::Tensor::get_shape() const { size_t ov::descriptor::Tensor::size() const { const bool bitwidth_less_than_byte = m_element_type.bitwidth() < 8; - if (bitwidth_less_than_byte) { - return static_cast(ceil((1.0 * shape_size(get_shape()) * m_element_type.bitwidth()) / 8)); - } - return shape_size(get_shape()) * m_element_type.size(); + return bitwidth_less_than_byte ? (shape_size(get_shape()) * m_element_type.bitwidth() + 7) >> 3 + : (shape_size(get_shape()) * m_element_type.size()); } const std::unordered_set& ov::descriptor::Tensor::get_names() const { diff --git a/src/core/src/model.cpp b/src/core/src/model.cpp index df540d81f5554b..d4fd18df72cc7c 100644 --- a/src/core/src/model.cpp +++ b/src/core/src/model.cpp @@ -318,6 +318,7 @@ std::vector> ov::Model::get_ordered_ops() const { m_cached_ordered_ops.clear(); for_each(order.cbegin(), order.cend(), [this](const shared_ptr& node) { m_cached_ordered_ops.push_back(node); + m_cached_ops.insert(node.get()); node->insert_info(m_shared_rt_info); }); m_cached_output_names.clear(); @@ -923,6 +924,9 @@ ov::Output ov::Model::add_output(const std::string& op_name, size_t ou } ov::Output ov::Model::add_output(const ov::Output& port) { + auto cache_valid = [&]() { + return m_cached_ops.count(port.get_node()); + }; if (ov::op::util::is_output(port.get_node())) return port; for (const auto& input : port.get_target_inputs()) { @@ -934,9 +938,14 @@ ov::Output ov::Model::add_output(const ov::Output& port) { auto result = std::make_shared(port); m_results.push_back(result); if (m_shared_rt_info->get_use_topological_cache()) { - // Full update of topological cache is not needed, 'result' can be just inserted to the end - m_cached_ordered_ops.push_back(result); - result->insert_info(m_shared_rt_info); // Just for consistency, not required for Result nodes + if (cache_valid()) { + // Full update of topological cache is not needed, 'result' can be just inserted to the end + m_cached_ordered_ops.push_back(result); + m_cached_ops.insert(result.get()); + result->insert_info(m_shared_rt_info); // Just for consistency, not required for Result nodes + } else { + m_shared_rt_info->set_use_topological_cache(false); + } } return result->output(0); } @@ -955,7 +964,7 @@ bool ov::Model::has_rt_info(const std::vector& args) const { return false; if (i == args.size() - 1) break; - const ov::Any& rt_attr = get_rt_arg(info, args[i]); + const ov::Any rt_attr = get_rt_arg(info, args[i]); info = get_map_from_attr(rt_attr); } return true; diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index 70b18e710f46e3..2a9baaeb90d144 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -385,9 +385,13 @@ std::ostream& ov::Node::write_description(std::ostream& out, uint32_t depth) con if (depth == 0) { out << get_friendly_name(); } else { - OPENVINO_SUPPRESS_DEPRECATED_START - out << "v" << get_type_info().version << "::" << get_type_info().name << " " << get_friendly_name() << " ("; - OPENVINO_SUPPRESS_DEPRECATED_END + auto version = get_type_info().version_id; + if (version) + out << "v" << version << "::" << get_type_info().name << " " << get_friendly_name() << " ("; + else + out << "v" + << " " + << "::" << get_type_info().name << " " << get_friendly_name() << " ("; string sep = ""; for (const auto& arg : input_values()) { out << sep << arg; diff --git a/src/core/src/op/adaptive_avg_pool.cpp b/src/core/src/op/adaptive_avg_pool.cpp index 775af472729a24..b49d2d68f21e12 100644 --- a/src/core/src/op/adaptive_avg_pool.cpp +++ b/src/core/src/op/adaptive_avg_pool.cpp @@ -19,7 +19,9 @@ op::v8::AdaptiveAvgPool::AdaptiveAvgPool(const Output& data, const Output< void op::v8::AdaptiveAvgPool::validate_and_infer_types() { OV_OP_SCOPE(v8_AdaptiveAvgPool_validate_and_infer_types); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); } diff --git a/src/core/src/op/adaptive_max_pool.cpp b/src/core/src/op/adaptive_max_pool.cpp index e3bced2b9a6f4a..ecc55b9ca838fb 100644 --- a/src/core/src/op/adaptive_max_pool.cpp +++ b/src/core/src/op/adaptive_max_pool.cpp @@ -31,7 +31,9 @@ void op::v8::AdaptiveMaxPool::validate_and_infer_types() { m_index_element_type == element::i64 || m_index_element_type == element::i32, "Index element type must be i32 or i64"); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shapes[0]); set_output_type(1, m_index_element_type, output_shapes[1]); diff --git a/src/core/src/op/avg_pool.cpp b/src/core/src/op/avg_pool.cpp index 8e63d1731441dc..007ad910bb752e 100644 --- a/src/core/src/op/avg_pool.cpp +++ b/src/core/src/op/avg_pool.cpp @@ -47,7 +47,9 @@ bool ov::op::v1::AvgPool::visit_attributes(AttributeVisitor& visitor) { void ov::op::v1::AvgPool::validate_and_infer_types() { OV_OP_SCOPE(v1_AvgPool_validate_and_infer_types); - const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_START + const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this), m_pads_begin, m_pads_end); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shapes.front()); } diff --git a/src/core/src/op/batch_norm.cpp b/src/core/src/op/batch_norm.cpp index 1741e1ea1bb2c3..129e1bc54f7743 100644 --- a/src/core/src/op/batch_norm.cpp +++ b/src/core/src/op/batch_norm.cpp @@ -42,6 +42,7 @@ void op::v0::BatchNormInference::validate_and_infer_types() { m_epsilon); set_output_size(1); + OPENVINO_SUPPRESS_DEPRECATED_START std::tie(result_et, result_batch_shape, result_channel_shape) = infer_batch_norm_forward(this, get_input_element_type(INPUT_DATA), @@ -54,7 +55,7 @@ void op::v0::BatchNormInference::validate_and_infer_types() { get_input_partial_shape(INPUT_BETA), get_input_partial_shape(INPUT_MEAN), get_input_partial_shape(INPUT_VARIANCE)); - + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, result_et, result_batch_shape); } @@ -98,6 +99,7 @@ void op::v5::BatchNormInference::validate_and_infer_types() { m_epsilon); set_output_size(1); + OPENVINO_SUPPRESS_DEPRECATED_START std::tie(result_et, result_batch_shape, result_channel_shape) = infer_batch_norm_forward(this, get_input_element_type(INPUT_DATA), @@ -110,7 +112,7 @@ void op::v5::BatchNormInference::validate_and_infer_types() { get_input_partial_shape(INPUT_BETA), get_input_partial_shape(INPUT_MEAN), get_input_partial_shape(INPUT_VARIANCE)); - + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, result_et, result_batch_shape); } diff --git a/src/core/src/op/batch_to_space.cpp b/src/core/src/op/batch_to_space.cpp index dfac266a0d00fe..cb52b92c68f1c8 100644 --- a/src/core/src/op/batch_to_space.cpp +++ b/src/core/src/op/batch_to_space.cpp @@ -60,7 +60,9 @@ void op::v1::BatchToSpace::validate_and_infer_types() { "block_shape and crops inputs must have integer element type. Got: ", inputs_integer_et); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, data_et, output_shape); } @@ -173,8 +175,10 @@ bool batch_to_space_evaluate(const HostTensorVector& outputs, const HostTensorVe bool ngraph::op::v1::BatchToSpace::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_BatchToSpace_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 4)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END if (outputs[0]->get_partial_shape().is_dynamic()) { std::map constant_data; diff --git a/src/core/src/op/binary_convolution.cpp b/src/core/src/op/binary_convolution.cpp index d6e5c8955995e1..ca89c851ba9cf6 100644 --- a/src/core/src/op/binary_convolution.cpp +++ b/src/core/src/op/binary_convolution.cpp @@ -2,15 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/binary_convolution.hpp" +#include "openvino/op/binary_convolution.hpp" +#include "binary_convolution_shape_inference.hpp" +#include "convolution_shape_inference.hpp" #include "itt.hpp" -#include "ngraph/attribute_visitor.hpp" -#include "ngraph/axis_vector.hpp" -#include "ngraph/coordinate_diff.hpp" -#include "ngraph/op/reshape.hpp" -#include "ngraph/util.hpp" -#include "ngraph/validation_util.hpp" +#include "openvino/core/attribute_visitor.hpp" +#include "openvino/core/axis_vector.hpp" +#include "openvino/core/coordinate_diff.hpp" using namespace std; @@ -23,14 +22,9 @@ ov::op::v1::BinaryConvolution::BinaryConvolution(const Output& data, BinaryConvolutionMode mode, float pad_value, const PadType& auto_pad) - : Op({data, kernel}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), + : ConvolutionFwdPropBase({data, kernel}, strides, pads_begin, pads_end, dilations, auto_pad), m_mode(mode), - m_pad_value(pad_value), - m_auto_pad(auto_pad) { + m_pad_value(pad_value) { constructor_validate_and_infer_types(); } @@ -43,23 +37,16 @@ ov::op::v1::BinaryConvolution::BinaryConvolution(const Output& data, const std::string& mode, float pad_value, const PadType& auto_pad) - : Op({data, kernel}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), + : ConvolutionFwdPropBase({data, kernel}, strides, pads_begin, pads_end, dilations, auto_pad), m_mode(mode_from_string(mode)), - m_pad_value(pad_value), - m_auto_pad(auto_pad) { + m_pad_value(pad_value) { constructor_validate_and_infer_types(); } void ov::op::v1::BinaryConvolution::validate_and_infer_types() { OV_OP_SCOPE(v1_BinaryConvolution_validate_and_infer_types); - const ov::PartialShape& data_batch_pshape = get_input_partial_shape(0); - element::Type data_batch_et = get_input_element_type(0); - const ov::PartialShape& filters_pshape = get_input_partial_shape(1); + const auto& data_batch_et = get_input_element_type(0); NODE_VALIDATION_CHECK(this, data_batch_et.is_real() || data_batch_et.is_integral_number(), "Data batch element type must be numeric. Got: ", @@ -67,25 +54,17 @@ void ov::op::v1::BinaryConvolution::validate_and_infer_types() { // TODO: Add NodeValidationCheck to filters et once u1 is supported in nGraph Python API // (#52715) + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END - Rank result_ps_rank; - NODE_VALIDATION_CHECK(this, - Rank::merge(result_ps_rank, data_batch_pshape.rank(), filters_pshape.rank()), - "Data batch and filters inputs must have same rank. Got: ", - data_batch_pshape, - " and ", - filters_pshape); + auto num_spatial = convolution::calculate_num_spatial(this, input_shapes); + if (num_spatial != util::num_spatial_undefined) { + resize_attributes(num_spatial); + } - ov::PartialShape result_shape = ngraph::validate_and_infer_convolution_forward_output_shape(this, - result_ps_rank, - data_batch_pshape, - filters_pshape, - m_auto_pad, - m_strides, - m_dilations, - m_pads_begin, - m_pads_end); - set_output_type(0, data_batch_et, result_shape); + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); + set_output_type(0, data_batch_et, output_shapes[0]); } shared_ptr ov::op::v1::BinaryConvolution::clone_with_new_inputs(const OutputVector& new_args) const { @@ -116,11 +95,11 @@ bool ov::op::v1::BinaryConvolution::visit_attributes(AttributeVisitor& visitor) namespace ov { template <> -NGRAPH_API EnumNames& -EnumNames::get() { - static auto enum_names = EnumNames( +OPENVINO_API EnumNames& +EnumNames::get() { + static auto enum_names = EnumNames( "op::v1::BinaryConvolution::BinaryConvolutionMode", - {{"xnor-popcount", ngraph::op::v1::BinaryConvolution::BinaryConvolutionMode::XNOR_POPCOUNT}}); + {{"xnor-popcount", op::v1::BinaryConvolution::BinaryConvolutionMode::XNOR_POPCOUNT}}); return enum_names; } } // namespace ov diff --git a/src/core/src/op/bucketize.cpp b/src/core/src/op/bucketize.cpp index b522db6c4ea984..d75710e926957b 100644 --- a/src/core/src/op/bucketize.cpp +++ b/src/core/src/op/bucketize.cpp @@ -47,7 +47,9 @@ void op::v3::Bucketize::validate_and_infer_types() { "Output type must be i32 or i64. Got: ", m_output_type); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END const auto output_shapes = shape_infer(this, input_shapes); if (get_input_partial_shape(0).is_dynamic()) { diff --git a/src/core/src/op/clamp.cpp b/src/core/src/op/clamp.cpp index ab4014cedd070e..797bddaa13525b 100644 --- a/src/core/src/op/clamp.cpp +++ b/src/core/src/op/clamp.cpp @@ -75,7 +75,9 @@ bool evaluate_clamp(const HostTensorPtr& arg, const HostTensorPtr& out, double m bool op::v0::Clamp::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Clamp_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return clamp::evaluate_clamp(inputs[0], outputs[0], get_min(), get_max()); } diff --git a/src/core/src/op/concat.cpp b/src/core/src/op/concat.cpp index 6f5cc09adb1ec4..00d85091d8dd52 100644 --- a/src/core/src/op/concat.cpp +++ b/src/core/src/op/concat.cpp @@ -12,7 +12,7 @@ #include "itt.hpp" #include "ngraph/attribute_visitor.hpp" #include "ngraph/runtime/reference/concat.hpp" -#include "ngraph/validation_util.hpp" +#include "validation_util.hpp" using namespace std; using namespace ngraph; @@ -104,8 +104,10 @@ bool evaluate_concat(const HostTensorVector& args, const HostTensorPtr& out, int bool op::Concat::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Concat_evaluate); NGRAPH_CHECK(!inputs.empty()); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, inputs.size())); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END auto concat_axis = get_axis() < 0 ? get_axis() + inputs[0]->get_shape().size() : get_axis(); return evaluate_concat(inputs, outputs[0], concat_axis); } @@ -115,7 +117,7 @@ bool op::Concat::evaluate(ov::TensorVector& outputs, const ov::TensorVector& inp OPENVINO_ASSERT(!inputs.empty()); OPENVINO_ASSERT(outputs.size() == 1); - auto concat_axis = ov::normalize(get_axis(), inputs.front().get_shape().size()); + auto concat_axis = ov::util::normalize(get_axis(), inputs.front().get_shape().size()); std::vector arg_bufs; std::vector arg_shapes; @@ -155,7 +157,9 @@ bool op::Concat::evaluate_label(TensorLabelVector& output_labels) const { const auto& inputs = input_values(); if (std::all_of(inputs.cbegin(), inputs.cend(), [](const Output& out) { const auto& labels = out.get_tensor().get_value_label(); + OPENVINO_SUPPRESS_DEPRECATED_START return has_no_labels(labels); + OPENVINO_SUPPRESS_DEPRECATED_END })) { return false; } diff --git a/src/core/src/op/convert.cpp b/src/core/src/op/convert.cpp index 08e5b1bded2cfa..c66a14c515a21a 100644 --- a/src/core/src/op/convert.cpp +++ b/src/core/src/op/convert.cpp @@ -143,8 +143,10 @@ bool evaluate_bound(const Node* node, ov::TensorVector& output_values, bool is_u } // constants for dynamic values translation + OPENVINO_SUPPRESS_DEPRECATED_START auto input_maximum_value = get_constant_max_of_type(input_element_type); auto output_maximum_value = get_constant_max_of_type(output_values[0].get_element_type()); + OPENVINO_SUPPRESS_DEPRECATED_END if (input_maximum_value == nullptr || output_maximum_value == nullptr) return false; @@ -171,8 +173,10 @@ bool evaluate_bound(const Node* node, ov::TensorVector& output_values, bool is_u } // namespace convert bool op::v0::Convert::evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const { OV_OP_SCOPE(v0_Convert_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(input_values, 1)); NGRAPH_CHECK(validate_host_tensor_vector(output_values, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return convert::evaluate_convert(input_values[0], output_values[0]); } diff --git a/src/core/src/op/convolution.cpp b/src/core/src/op/convolution.cpp index e87b4bb9b13261..7f613449505644 100644 --- a/src/core/src/op/convolution.cpp +++ b/src/core/src/op/convolution.cpp @@ -2,21 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/convolution.hpp" - -#include +#include "openvino/op/convolution.hpp" #include "bound_evaluate.hpp" +#include "convolution_backprop_shape_inference.hpp" +#include "convolution_shape_inference.hpp" #include "itt.hpp" -#include "ngraph/axis_vector.hpp" -#include "ngraph/coordinate_diff.hpp" -#include "ngraph/op/reshape.hpp" -#include "ngraph/util.hpp" #include "openvino/op/util/precision_sensitive_attribute.hpp" using namespace std; -using namespace ngraph; +namespace ov { // *** Convolution OP SET 1 *** op::v1::Convolution::Convolution(const Output& data_batch, const Output& filters, @@ -25,12 +21,7 @@ op::v1::Convolution::Convolution(const Output& data_batch, const CoordinateDiff& pads_end, const Strides& dilations, const PadType& auto_pad) - : Op({data_batch, filters}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad) { + : ConvolutionFwdPropBase({data_batch, filters}, strides, pads_begin, pads_end, dilations, auto_pad) { constructor_validate_and_infer_types(); } @@ -46,8 +37,8 @@ bool op::v1::Convolution::visit_attributes(AttributeVisitor& visitor) { void op::v1::Convolution::validate_and_infer_types() { OV_OP_SCOPE(v1_Convolution_validate_and_infer_types); - element::Type data_batch_et = get_input_element_type(0); - element::Type filters_et = get_input_element_type(1); + const auto& data_batch_et = get_input_element_type(0); + const auto& filters_et = get_input_element_type(1); element::Type result_et; NODE_VALIDATION_CHECK(this, @@ -62,21 +53,19 @@ void op::v1::Convolution::validate_and_infer_types() { result_et.is_real() || result_et.is_integral_number(), "Element types must be numeric. Got: ", result_et); - auto& data_shape = get_input_partial_shape(0); - auto& filter_shape = get_input_partial_shape(1); - - m_num_spatial = calculate_num_spatial(this, data_shape, filter_shape, 2, 2); - update_and_validate_attributes(this, m_num_spatial); - std::vector input_shapes = {data_shape, filter_shape}; - std::vector output_shapes = {ov::PartialShape::dynamic()}; + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END - if (m_num_spatial != -1) { - resolve_auto_pad_for_shape(this, m_pads_begin, m_pads_end, input_shapes, 2, 2); - shape_infer(this, m_pads_begin, m_pads_end, input_shapes, output_shapes); + auto num_spatial = convolution::calculate_num_spatial(this, input_shapes); + if (num_spatial != util::num_spatial_undefined) { + resize_attributes(num_spatial); } + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); set_output_type(0, result_et, output_shapes[0]); + set_num_spatial(num_spatial, input_shapes); } shared_ptr op::v1::Convolution::clone_with_new_inputs(const OutputVector& new_args) const { @@ -101,13 +90,13 @@ op::v1::ConvolutionBackpropData::ConvolutionBackpropData(const Output& dat const Strides& dilations, const PadType& auto_pad, const CoordinateDiff& output_padding) - : Op({data, filters, output_shape}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad), - m_output_padding(output_padding) { + : ConvolutionBackPropBase({data, filters, output_shape}, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + output_padding) { ov::mark_as_precision_sensitive(input(2)); constructor_validate_and_infer_types(); } @@ -131,44 +120,47 @@ op::v1::ConvolutionBackpropData::ConvolutionBackpropData(const Output& dat const Strides& dilations, const PadType& auto_pad, const CoordinateDiff& output_padding) - : Op({data, filters}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad), - m_output_padding(output_padding) { + : ConvolutionBackPropBase({data, filters}, strides, pads_begin, pads_end, dilations, auto_pad, output_padding) { constructor_validate_and_infer_types(); } bool op::v1::ConvolutionBackpropData::is_dynamic() const { - bool is_dynamic = Node::is_dynamic(); - if (inputs().size() == 3 && !is_dynamic) { - return !has_and_set_equal_bounds(input_value(2)); - } - return is_dynamic; + return Node::is_dynamic() || (get_input_size() == 3 && !has_and_set_equal_bounds(input_value(2))); } const ov::PartialShape op::v1::ConvolutionBackpropData::get_output_shape() const { - ov::PartialShape shape; - if (get_input_size() == 3 && evaluate_as_partial_shape(input_value(2), shape)) - return shape; - - auto data_pshape = get_input_partial_shape(0); - auto filter_pshape = get_input_partial_shape(1); - - if (data_pshape.rank().is_static()) - shape = ov::PartialShape::dynamic(data_pshape.rank().get_length() - 2); - else if (filter_pshape.rank().is_static()) - shape = ov::PartialShape::dynamic(filter_pshape.rank().get_length() - 2); - else - shape = ov::PartialShape::dynamic(); + auto shape = PartialShape::dynamic(); + + OPENVINO_SUPPRESS_DEPRECATED_START + if (get_input_size() < 3 || !evaluate_as_partial_shape(input_value(2), shape)) { + OPENVINO_SUPPRESS_DEPRECATED_END + const auto& data_rank = get_input_partial_shape(0).rank(); + const auto& filter_rank = get_input_partial_shape(1).rank(); + + if (data_rank.is_static()) { + shape.resize(data_rank.get_length() - convolution::spatial_dim_offset); + } else if (filter_rank.is_static()) { + shape.resize(filter_rank.get_length() - convolution::spatial_dim_offset); + } else if (get_input_size() == 3) { + const auto& out_spatial_shape = get_input_partial_shape(2); + if (out_spatial_shape.is_static()) { + shape.resize(out_spatial_shape[0].get_length()); + } + } + } + return shape; } void op::v1::ConvolutionBackpropData::set_output_shape(const ov::Shape& shape) { - this->input(2).replace_source_output( - op::v0::Constant::create(this->get_input_element_type(2), ov::Shape{shape.size()}, shape)->output(0)); + element::Type_t et = (get_input_size() == 3) ? get_input_element_type(2) : element::i64; + if (get_input_size() == 0) { + // Add dummy inputs when adding output shape and op has no inputs at all. + auto dummy = std::make_shared(et, ov::Shape{0}); + set_argument(0, dummy); + set_argument(1, dummy); + } + set_argument(2, v0::Constant::create(et, Shape{shape.size()}, shape)); } void op::v1::ConvolutionBackpropData::infer_conv_backprop_output_spatial_shape( @@ -200,8 +192,8 @@ void op::v1::ConvolutionBackpropData::infer_conv_backprop_output_spatial_shape( void op::v1::ConvolutionBackpropData::validate_and_infer_types() { OV_OP_SCOPE(v1_ConvolutionBackpropData_validate_and_infer_types); - element::Type delta_et = get_input_element_type(0); - element::Type filters_et = get_input_element_type(1); + const auto& delta_et = get_input_element_type(0); + const auto& filters_et = get_input_element_type(1); element::Type result_et; NODE_VALIDATION_CHECK(this, @@ -217,9 +209,8 @@ void op::v1::ConvolutionBackpropData::validate_and_infer_types() { "Element type of inputs must be numeric. Got: ", result_et); - bool is_output_shape_present = inputs().size() == 3; - if (is_output_shape_present) { - const element::Type output_shape_et = get_input_element_type(2); + if (get_input_size() == 3) { + const auto& output_shape_et = get_input_element_type(2); NODE_VALIDATION_CHECK(this, output_shape_et.is_integral_number(), "Element type for output shape should be of integer type ", @@ -228,26 +219,19 @@ void op::v1::ConvolutionBackpropData::validate_and_infer_types() { ")."); } - bool output_shape_input_present = get_input_size() == 3; - - const auto& data_shape = get_input_partial_shape(0); - const auto& filter_shape = get_input_partial_shape(1); - - auto& output_shapes_shape = output_shape_input_present ? get_input_partial_shape(2) : PartialShape::dynamic(); - m_num_spatial = calculate_num_spatial(this, data_shape, filter_shape, output_shapes_shape, 2, 2); - update_and_validate_attributes_back_prop(this, m_num_spatial); + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + const auto out_spatial_shape = get_output_shape(); + auto num_spatial = convolution::calculate_num_spatial(this, input_shapes, out_spatial_shape); - std::vector input_shapes = {data_shape, filter_shape}; - if (output_shape_input_present) - input_shapes.push_back(get_input_partial_shape(2)); - std::vector output_shapes = {ov::PartialShape::dynamic()}; - - if (m_num_spatial != -1) { - ov::PartialShape output_spatial_shape = get_output_shape(); - resolve_auto_pad_for_shape_back_prop(this, m_pads_begin, m_pads_end, input_shapes, output_spatial_shape, 2, 2); - shape_infer(this, m_pads_begin, m_pads_end, output_spatial_shape, input_shapes, output_shapes); + if (num_spatial != util::num_spatial_undefined) { + resize_attributes(num_spatial); } + + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); set_output_type(0, result_et, output_shapes[0]); + set_num_spatial(num_spatial, input_shapes); set_input_is_relevant_to_shape(0); set_input_is_relevant_to_shape(1); @@ -277,3 +261,4 @@ shared_ptr op::v1::ConvolutionBackpropData::clone_with_new_inputs(const Ou m_output_padding); } } +} // namespace ov diff --git a/src/core/src/op/cos.cpp b/src/core/src/op/cos.cpp index ce9542f259530f..824b653cf880c1 100644 --- a/src/core/src/op/cos.cpp +++ b/src/core/src/op/cos.cpp @@ -58,8 +58,10 @@ bool evaluate_cos(const HostTensorPtr& arg0, const HostTensorPtr& out, const siz bool op::Cos::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Cos_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); return cosop::evaluate_cos(inputs[0], outputs[0], shape_size(get_output_shape(0))); + OPENVINO_SUPPRESS_DEPRECATED_END } bool op::Cos::has_evaluate() const { diff --git a/src/core/src/op/cosh.cpp b/src/core/src/op/cosh.cpp index 9f40b4a9be10cf..573dfe9b135250 100644 --- a/src/core/src/op/cosh.cpp +++ b/src/core/src/op/cosh.cpp @@ -58,7 +58,9 @@ bool evaluate_cosh(const HostTensorPtr& arg0, const HostTensorPtr& out, const si bool op::Cosh::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Cosh_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return coshop::evaluate_cosh(inputs[0], outputs[0], shape_size(get_output_shape(0))); } diff --git a/src/core/src/op/ctc_loss.cpp b/src/core/src/op/ctc_loss.cpp index 89ef7f2260ae07..a6ce8b6bb8f7a2 100644 --- a/src/core/src/op/ctc_loss.cpp +++ b/src/core/src/op/ctc_loss.cpp @@ -62,7 +62,9 @@ void op::v4::CTCLoss::validate_and_infer_types() { input_et); } + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, ov::get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, logits_type, output_shape); } diff --git a/src/core/src/op/deformable_convolution.cpp b/src/core/src/op/deformable_convolution.cpp index a4bd3b65904134..ba9552759716a8 100644 --- a/src/core/src/op/deformable_convolution.cpp +++ b/src/core/src/op/deformable_convolution.cpp @@ -2,18 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/deformable_convolution.hpp" +#include "openvino/op/deformable_convolution.hpp" +#include "deformable_convolution_shape_inference.hpp" #include "itt.hpp" -#include "ngraph/axis_vector.hpp" -#include "ngraph/coordinate_diff.hpp" -#include "ngraph/runtime/reference/deformable_convolution.hpp" -#include "ngraph/util.hpp" -#include "ngraph/validation_util.hpp" using namespace std; -using namespace ngraph; - +namespace ov { op::v8::DeformableConvolution::DeformableConvolution(const Output& arg, const Output& offsets, const Output& filters, @@ -70,74 +65,47 @@ bool op::v8::DeformableConvolution::visit_attributes(AttributeVisitor& visitor) void op::v8::DeformableConvolution::validate_and_infer_types() { OV_OP_SCOPE(DeformableConvolution_v8_validate_and_infer_types); - DeformableConvolutionBase::validate_and_infer_types(); - if (inputs().size() == 4) { - const ov::PartialShape& data_pshape = get_input_partial_shape(0); - const ov::PartialShape& filters_pshape = get_input_partial_shape(2); - const ov::PartialShape& mask_pshape = get_input_partial_shape(3); + const auto& data_batch_et = get_input_element_type(0); + const auto& offsets_et = get_input_element_type(1); + const auto& filters_et = get_input_element_type(2); + + element::Type result_et; + NODE_VALIDATION_CHECK(this, + element::Type::merge(result_et, data_batch_et, offsets_et) && + element::Type::merge(result_et, result_et, filters_et), + "Element types of inputs do not match. Got: data batch (", + data_batch_et, + "), offsets (", + offsets_et, + ") and filters (", + filters_et, + ")"); + + NODE_VALIDATION_CHECK(this, + result_et.is_real() || result_et.is_integral_number(), + "Element type of inputs must be numeric. Got: ", + result_et); + + if (get_input_size() == 4) { element::Type mask_et = get_input_element_type(3); NODE_VALIDATION_CHECK(this, mask_et.is_real() || mask_et.is_integral_number(), "Element type of Mask input must be numeric. Got: ", mask_et); + } - NODE_VALIDATION_CHECK(this, - mask_pshape.rank().compatible(4), - "Mask input must be of rank 4. Got: ", - mask_pshape.rank()); - - if (mask_pshape.rank().is_static() && mask_pshape[1].is_static()) { - if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() && filters_pshape[3].is_static()) { - auto offsets_channels = - m_deformable_group * filters_pshape[2].get_length() * filters_pshape[3].get_length(); - NODE_VALIDATION_CHECK(this, - mask_pshape[1].get_length() == offsets_channels, - "The channels dimension of mask input is not " - "compatible with filters and 'deformable group' attribute. " - "Mask input shape: ", - mask_pshape, - ", deformable 'group' attribute value: ", - m_deformable_group, - ", filters shape: ", - filters_pshape); - } - // At least we can check if mask channels is evenly divisible by deformable - // group attribute - NODE_VALIDATION_CHECK(this, - mask_pshape[1].get_length() % m_deformable_group == 0, - "The channels dimension of mask input must be " - "evenly divisible by the 'deformable group' value along the " - "channels axis. Offsets input shape: ", - mask_pshape, - ", 'deformable group' attribute value: ", - m_deformable_group); - - if (data_pshape.rank().is_static()) { - NODE_VALIDATION_CHECK(this, - mask_pshape[0].compatible(data_pshape[0]), - "Data batch and mask batch dimension must be same value. Got: ", - mask_pshape[0], - " and ", - data_pshape[0]); - } - } - - ov::PartialShape result_pshape = get_output_partial_shape(0); - if (result_pshape.rank().is_static() && mask_pshape.rank().is_static()) { - NODE_VALIDATION_CHECK( - this, - result_pshape[2].compatible(mask_pshape[2]) && result_pshape[3].compatible(mask_pshape[3]), - "Spatial dimensions of mask and output must be equal. Got: ", - mask_pshape[2], - ", ", - mask_pshape[3], - " and ", - result_pshape[2], - ", ", - result_pshape[3]); - } + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + + auto num_spatial = deformable_conv::calculate_num_spatial(this, input_shapes); + if (num_spatial != convolution::num_spatial_undefined) { + resize_attributes(num_spatial); } + + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); + set_output_type(0, result_et, output_shapes[0]); } std::shared_ptr op::v8::DeformableConvolution::clone_with_new_inputs(const OutputVector& new_args) const { @@ -208,3 +176,41 @@ std::shared_ptr op::v1::DeformableConvolution::clone_with_new_inputs(const m_group, m_deformable_group); } + +void op::v1::DeformableConvolution::validate_and_infer_types() { + OV_OP_SCOPE(DeformableConvolution_v1_validate_and_infer_types); + + const auto& data_batch_et = get_input_element_type(0); + const auto& offsets_et = get_input_element_type(1); + const auto& filters_et = get_input_element_type(2); + + element::Type result_et; + NODE_VALIDATION_CHECK(this, + element::Type::merge(result_et, data_batch_et, offsets_et) && + element::Type::merge(result_et, result_et, filters_et), + "Element types of inputs do not match. Got: data batch (", + data_batch_et, + "), offsets (", + offsets_et, + ") and filters (", + filters_et, + ")"); + + NODE_VALIDATION_CHECK(this, + result_et.is_real() || result_et.is_integral_number(), + "Element type of inputs must be numeric. Got: ", + result_et); + + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + + auto num_spatial = deformable_conv::calculate_num_spatial(this, input_shapes); + if (num_spatial != convolution::num_spatial_undefined) { + resize_attributes(num_spatial); + } + + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); + set_output_type(0, result_et, output_shapes[0]); +} +} // namespace ov diff --git a/src/core/src/op/deformable_psroi_pooling.cpp b/src/core/src/op/deformable_psroi_pooling.cpp index b770941db1e8e7..d5126489c99ea6 100644 --- a/src/core/src/op/deformable_psroi_pooling.cpp +++ b/src/core/src/op/deformable_psroi_pooling.cpp @@ -72,7 +72,9 @@ bool op::v1::DeformablePSROIPooling::visit_attributes(AttributeVisitor& visitor) void op::v1::DeformablePSROIPooling::validate_and_infer_types() { OV_OP_SCOPE(v1_DeformablePSROIPooling_validate_and_infer_types); const auto& input_et = get_input_element_type(0); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, input_et, shape_infer(this, input_shapes)[0]); } diff --git a/src/core/src/op/depth_to_space.cpp b/src/core/src/op/depth_to_space.cpp index 385824ffc56d12..582a319bdda352 100644 --- a/src/core/src/op/depth_to_space.cpp +++ b/src/core/src/op/depth_to_space.cpp @@ -44,7 +44,9 @@ std::shared_ptr op::DepthToSpace::clone_with_new_inputs(const OutputVector void op::DepthToSpace::validate_and_infer_types() { OV_OP_SCOPE(v0_DepthToSpace_validate_and_infer_types); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); } diff --git a/src/core/src/op/detection_output.cpp b/src/core/src/op/detection_output.cpp index 05fe7588311d29..4dd8e806f0bd5e 100644 --- a/src/core/src/op/detection_output.cpp +++ b/src/core/src/op/detection_output.cpp @@ -36,7 +36,9 @@ void ov::op::v0::DetectionOutput::validate_and_infer_types() { NODE_VALIDATION_CHECK(this, m_attrs.num_classes > 0, "Number of classes must be greater than zero"); validate_base(m_attrs); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1); shape_infer(this, input_shapes, output_shapes); set_output_type(0, get_input_element_type(0), output_shapes[0]); @@ -94,7 +96,9 @@ void ov::op::v8::DetectionOutput::validate_and_infer_types() { OV_OP_SCOPE(v8_DetectionOutput_validate_and_infer_types); validate_base(m_attrs); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1); shape_infer(this, input_shapes, output_shapes); set_output_type(0, get_input_element_type(0), output_shapes[0]); diff --git a/src/core/src/op/divide.cpp b/src/core/src/op/divide.cpp index d90d9dc6b82f06..aba23215d48777 100644 --- a/src/core/src/op/divide.cpp +++ b/src/core/src/op/divide.cpp @@ -112,8 +112,10 @@ bool evaluate_bound(const Node* node, ov::TensorVector& output_values, bool is_u const auto zero_t = ov::Tensor(input2.get_element_type(), Shape{}); memcpy(zero_t.data(), zeros_const->get_data_ptr(), zero_t.get_byte_size()); + OPENVINO_SUPPRESS_DEPRECATED_START auto max_constant = get_constant_max_of_type(input2.get_element_type()); auto dynamic_mask = or_tensor(equality_mask(input1_up, max_constant), equality_mask(input2_up, max_constant)); + OPENVINO_SUPPRESS_DEPRECATED_END // mask to find out positive values for arg2 auto less_up_outputs = ov::TensorVector{{element::boolean, input2.get_shape()}}; @@ -157,7 +159,9 @@ bool evaluate_bound(const Node* node, ov::TensorVector& output_values, bool is_u return status; // replace values where zeros inside range of second arg to maximum values + OPENVINO_SUPPRESS_DEPRECATED_START auto output_minimum_value = get_constant_min_of_type(output_values[0].get_element_type()); + OPENVINO_SUPPRESS_DEPRECATED_END if (output_minimum_value == nullptr) return false; @@ -202,7 +206,9 @@ bool evaluate_bound(const Node* node, ov::TensorVector& output_values, bool is_u return status; // replace values where zeros were found in the second argument to maximum values + OPENVINO_SUPPRESS_DEPRECATED_START auto output_maximum_value = get_constant_max_of_type(output_values[0].get_element_type()); + OPENVINO_SUPPRESS_DEPRECATED_END if (output_maximum_value == nullptr) return false; diff --git a/src/core/src/op/einsum.cpp b/src/core/src/op/einsum.cpp index 4ec85e5321faa1..462c71b606b46d 100644 --- a/src/core/src/op/einsum.cpp +++ b/src/core/src/op/einsum.cpp @@ -190,7 +190,9 @@ void op::v7::Einsum::validate_and_infer_types() { "Inputs to Einsum operation must have the same type."); } + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes = {ov::PartialShape::dynamic()}; shape_infer(this, input_shapes, output_shapes); diff --git a/src/core/src/op/embedding_segments_sum.cpp b/src/core/src/op/embedding_segments_sum.cpp index cea947a07223d9..0058f99dc2a1fe 100644 --- a/src/core/src/op/embedding_segments_sum.cpp +++ b/src/core/src/op/embedding_segments_sum.cpp @@ -99,7 +99,9 @@ void op::v3::EmbeddingSegmentsSum::validate_and_infer_types() { ")"); } const auto& result_et = get_input_element_type(EMB_TABLE); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END const auto result_shapes = shape_infer(this, input_shapes); if (result_shapes[EMB_TABLE].rank().is_dynamic() || result_shapes[EMB_TABLE][0].is_dynamic()) { diff --git a/src/core/src/op/exp.cpp b/src/core/src/op/exp.cpp index 714d71913a155e..d262556110c3c5 100644 --- a/src/core/src/op/exp.cpp +++ b/src/core/src/op/exp.cpp @@ -60,7 +60,9 @@ bool evaluate_exp(const HostTensorPtr& arg0, const HostTensorPtr& out) { bool op::Exp::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Exp_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return expop::evaluate_exp(inputs[0], outputs[0]); } diff --git a/src/core/src/op/eye.cpp b/src/core/src/op/eye.cpp index f3fb9d7b121852..97e3582e8b7591 100644 --- a/src/core/src/op/eye.cpp +++ b/src/core/src/op/eye.cpp @@ -71,7 +71,9 @@ void ov::op::v9::Eye::validate_and_infer_types() { input_et); } + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_out_type(), output_shape); } @@ -112,8 +114,10 @@ bool ov::op::v9::Eye::has_evaluate() const { bool ov::op::v9::Eye::evaluate(const ov::HostTensorVector& outputs, const ov::HostTensorVector& inputs) const { OV_OP_SCOPE(v9_Eye_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(inputs, get_input_size()), "Invalid Eye input TensorVector."); OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(outputs, 1), "Invalid Eye output TensorVector."); + OPENVINO_SUPPRESS_DEPRECATED_END int64_t diagonal_index; diff --git a/src/core/src/op/gather.cpp b/src/core/src/op/gather.cpp index 24d12668ddc481..a5e69144829704 100644 --- a/src/core/src/op/gather.cpp +++ b/src/core/src/op/gather.cpp @@ -18,8 +18,11 @@ op::v1::Gather::Gather(const Output& params, const Output& indices, } int64_t ngraph::op::v1::Gather::get_axis() const { - if (!get_constant_from_source(input_value(2))) + OPENVINO_SUPPRESS_DEPRECATED_START + if (!get_constant_from_source(input_value(2))) { + OPENVINO_SUPPRESS_DEPRECATED_END return AXIS_NOT_SET_VALUE; + } return GatherBase::get_axis(); } diff --git a/src/core/src/op/gather_elements.cpp b/src/core/src/op/gather_elements.cpp index 391e4bb6e6ce94..cb519d2c9bce34 100644 --- a/src/core/src/op/gather_elements.cpp +++ b/src/core/src/op/gather_elements.cpp @@ -30,7 +30,9 @@ void op::v6::GatherElements::validate_and_infer_types() { "indices must be of int32 or int64 type. But instead got: ", indices_type); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1); shape_infer(this, input_shapes, output_shapes); set_output_type(0, data_type, output_shapes[0]); diff --git a/src/core/src/op/gather_tree.cpp b/src/core/src/op/gather_tree.cpp index f2fd41e6850239..67c41e792655c6 100644 --- a/src/core/src/op/gather_tree.cpp +++ b/src/core/src/op/gather_tree.cpp @@ -59,6 +59,8 @@ void op::v1::GatherTree::validate_and_infer_types() { "Element type of inputs must be numeric. Got: ", result_et); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, ov::get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, result_et, output_shape); } diff --git a/src/core/src/op/gelu.cpp b/src/core/src/op/gelu.cpp index 3dbf5086d95cdd..a2a4d2f313e464 100644 --- a/src/core/src/op/gelu.cpp +++ b/src/core/src/op/gelu.cpp @@ -132,7 +132,9 @@ bool evaluate_gelu(const HostTensorPtr& arg0, const HostTensorPtr& out, op::Gelu bool op::v7::Gelu::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v7_Gelu_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return gelu::evaluate_gelu(inputs[0], outputs[0], m_approximation_mode); } diff --git a/src/core/src/op/greater_eq.cpp b/src/core/src/op/greater_eq.cpp index 8c9158f9c78aca..cbb7a4494e7e8f 100644 --- a/src/core/src/op/greater_eq.cpp +++ b/src/core/src/op/greater_eq.cpp @@ -68,7 +68,9 @@ shared_ptr op::v1::GreaterEqual::clone_with_new_inputs(const OutputVector& bool op::v1::GreaterEqual::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_GreaterEqual_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END return greater_equalop::evaluate_greater_equal(inputs[0], inputs[1], outputs[0], get_autob()); } diff --git a/src/core/src/op/grid_sample.cpp b/src/core/src/op/grid_sample.cpp index b492cf744614b5..9262f115ab9024 100644 --- a/src/core/src/op/grid_sample.cpp +++ b/src/core/src/op/grid_sample.cpp @@ -127,8 +127,10 @@ bool evaluate_grid_sample(const HostTensorPtr& output, bool op::v9::GridSample::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v9_GridSample_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(inputs, 2), "Invalid GridSample input TensorVector."); OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(outputs, 1), "Invalid GridSample output TensorVector."); + OPENVINO_SUPPRESS_DEPRECATED_END return evaluate_grid_sample(outputs[0], inputs[0], inputs[1], m_attributes); } diff --git a/src/core/src/op/group_conv.cpp b/src/core/src/op/group_conv.cpp index 0945336c87595e..3a7b28108307c0 100644 --- a/src/core/src/op/group_conv.cpp +++ b/src/core/src/op/group_conv.cpp @@ -2,22 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/group_conv.hpp" - -#include +#include "openvino/op/group_conv.hpp" #include "bound_evaluate.hpp" +#include "group_convolution_backprop_shape_inference.hpp" +#include "group_convolution_shape_inference.hpp" #include "itt.hpp" -#include "ngraph/attribute_visitor.hpp" #include "openvino/op/util/precision_sensitive_attribute.hpp" using namespace std; -using namespace ngraph; //------------------------------------------------------------------------------ // v1::GroupConvolution //------------------------------------------------------------------------------ - +namespace ov { op::v1::GroupConvolution::GroupConvolution(const Output& data_batch, const Output& filters, const Strides& strides, @@ -25,16 +23,11 @@ op::v1::GroupConvolution::GroupConvolution(const Output& data_batch, const CoordinateDiff& pads_end, const Strides& dilations, const PadType& auto_pad) - : Op({data_batch, filters}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad) { + : ConvolutionFwdPropBase({data_batch, filters}, strides, pads_begin, pads_end, dilations, auto_pad) { constructor_validate_and_infer_types(); } -bool ngraph::op::v1::GroupConvolution::visit_attributes(AttributeVisitor& visitor) { +bool op::v1::GroupConvolution::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v1_GroupConvolution_visit_attributes); visitor.on_attribute("strides", m_strides); visitor.on_attribute("pads_begin", m_pads_begin); @@ -46,8 +39,8 @@ bool ngraph::op::v1::GroupConvolution::visit_attributes(AttributeVisitor& visito void op::v1::GroupConvolution::validate_and_infer_types() { OV_OP_SCOPE(v1_GroupConvolution_validate_and_infer_types); - element::Type data_batch_et = get_input_element_type(0); - element::Type filters_et = get_input_element_type(1); + const auto& data_batch_et = get_input_element_type(0); + const auto& filters_et = get_input_element_type(1); element::Type result_et; NODE_VALIDATION_CHECK(this, @@ -63,21 +56,18 @@ void op::v1::GroupConvolution::validate_and_infer_types() { "Element type of inputs must be numeric. Got: ", result_et); - auto& data_shape = get_input_partial_shape(0); - auto& filter_shape = get_input_partial_shape(1); - - m_num_spatial = calculate_num_spatial(this, data_shape, filter_shape, 2, 3); - update_and_validate_attributes(this, m_num_spatial); - - std::vector input_shapes = {data_shape, filter_shape}; - std::vector output_shapes = {ov::PartialShape::dynamic()}; + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END - if (m_num_spatial != -1) { - resolve_auto_pad_for_shape(this, m_pads_begin, m_pads_end, input_shapes, 2, 3); - shape_infer(this, m_pads_begin, m_pads_end, input_shapes, output_shapes); + auto num_spatial = convolution::calculate_num_spatial(this, input_shapes); + if (num_spatial != convolution::num_spatial_undefined) { + resize_attributes(num_spatial); } + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); set_output_type(0, result_et, output_shapes[0]); + set_num_spatial(num_spatial, input_shapes); } shared_ptr op::v1::GroupConvolution::clone_with_new_inputs(const OutputVector& new_args) const { @@ -96,14 +86,7 @@ shared_ptr op::v1::GroupConvolution::clone_with_new_inputs(const OutputVec // v1::GroupConvolutionBackpropData //------------------------------------------------------------------------------ -op::v1::GroupConvolutionBackpropData::GroupConvolutionBackpropData() - : Op(), - m_strides(), - m_dilations(), - m_pads_begin(), - m_pads_end(), - m_auto_pad(), - m_output_padding() {} +op::v1::GroupConvolutionBackpropData::GroupConvolutionBackpropData() : ConvolutionBackPropBase() {} op::v1::GroupConvolutionBackpropData::GroupConvolutionBackpropData(const Output& data, const Output& filters, @@ -114,13 +97,13 @@ op::v1::GroupConvolutionBackpropData::GroupConvolutionBackpropData(const Output< const Strides& dilations, const PadType& auto_pad, const CoordinateDiff& output_padding) - : Op({data, filters, output_shape}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad), - m_output_padding(output_padding) { + : ConvolutionBackPropBase({data, filters, output_shape}, + strides, + pads_begin, + pads_end, + dilations, + auto_pad, + output_padding) { ov::mark_as_precision_sensitive(input(2)); constructor_validate_and_infer_types(); } @@ -150,17 +133,11 @@ op::v1::GroupConvolutionBackpropData::GroupConvolutionBackpropData(const Output< const Strides& dilations, const PadType& auto_pad, const CoordinateDiff& output_padding) - : Op({data, filters}), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad), - m_output_padding(output_padding) { + : ConvolutionBackPropBase({data, filters}, strides, pads_begin, pads_end, dilations, auto_pad, output_padding) { constructor_validate_and_infer_types(); } -bool ngraph::op::v1::GroupConvolutionBackpropData::visit_attributes(AttributeVisitor& visitor) { +bool op::v1::GroupConvolutionBackpropData::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v1_GroupConvolutionBackpropData_visit_attributes); visitor.on_attribute("strides", m_strides); visitor.on_attribute("pads_begin", m_pads_begin); @@ -172,33 +149,42 @@ bool ngraph::op::v1::GroupConvolutionBackpropData::visit_attributes(AttributeVis } bool op::v1::GroupConvolutionBackpropData::is_dynamic() const { - bool is_dynamic = Node::is_dynamic(); - if (inputs().size() == 3 && !is_dynamic) { - return !has_and_set_equal_bounds(input_value(2)); - } - return is_dynamic; + return Node::is_dynamic() || (get_input_size() == 3 && !has_and_set_equal_bounds(input_value(2))); } const ov::PartialShape op::v1::GroupConvolutionBackpropData::get_convolution_output_shape() const { - ov::PartialShape shape; - if (get_input_size() == 3 && evaluate_as_partial_shape(input_value(2), shape)) - return shape; - - auto data_pshape = get_input_partial_shape(0); - auto filter_pshape = get_input_partial_shape(1); - - if (data_pshape.rank().is_static()) - shape = ov::PartialShape::dynamic(data_pshape.rank().get_length() - 2); - else if (filter_pshape.rank().is_static()) - shape = ov::PartialShape::dynamic(filter_pshape.rank().get_length() - 2); - else - shape = ov::PartialShape::dynamic(); + auto shape = PartialShape::dynamic(); + + OPENVINO_SUPPRESS_DEPRECATED_START + if (get_input_size() < 3 || !evaluate_as_partial_shape(input_value(2), shape)) { + OPENVINO_SUPPRESS_DEPRECATED_END + const auto& data_rank = get_input_partial_shape(0).rank(); + const auto& filter_rank = get_input_partial_shape(1).rank(); + + if (data_rank.is_static()) { + shape.resize(data_rank.get_length() - util::spatial_dim_offset); + } else if (filter_rank.is_static()) { + shape.resize(filter_rank.get_length() - util::spatial_dim_offset); + } else if (get_input_size() == 3) { + const auto& out_spatial_shape = get_input_partial_shape(2); + if (out_spatial_shape.is_static()) { + shape.resize(out_spatial_shape[0].get_length()); + } + } + } + return shape; } void op::v1::GroupConvolutionBackpropData::set_output_shape(const ov::Shape& shape) { - this->input(2).replace_source_output( - op::v0::Constant::create(this->get_input_element_type(2), ov::Shape{shape.size()}, shape)->output(0)); + element::Type_t et = (get_input_size() == 3) ? get_input_element_type(2) : element::i64; + if (get_input_size() == 0) { + // Add dummy inputs when adding output shape and op has no inputs at all. + auto dummy = std::make_shared(et, ov::Shape{0}); + set_argument(0, dummy); + set_argument(1, dummy); + } + set_argument(2, v0::Constant::create(et, Shape{shape.size()}, shape)); } void op::v1::GroupConvolutionBackpropData::infer_conv_backprop_output_spatial_shape( @@ -211,9 +197,9 @@ void op::v1::GroupConvolutionBackpropData::infer_conv_backprop_output_spatial_sh const CoordinateDiff& output_padding, vector& output_spatial_shape) { size_t num_spatial_dims = input_data_shape.size(); - NGRAPH_CHECK(filters_shape.size() == num_spatial_dims && strides.size() == num_spatial_dims && - dilations.size() == num_spatial_dims && pads_begin.size() == num_spatial_dims && - pads_end.size() == num_spatial_dims && output_padding.size() == num_spatial_dims); + OPENVINO_ASSERT(filters_shape.size() == num_spatial_dims && strides.size() == num_spatial_dims && + dilations.size() == num_spatial_dims && pads_begin.size() == num_spatial_dims && + pads_end.size() == num_spatial_dims && output_padding.size() == num_spatial_dims); for (size_t i = 0; i < num_spatial_dims; ++i) { if (input_data_shape[i].is_static() && filters_shape[i].is_static()) { @@ -229,8 +215,8 @@ void op::v1::GroupConvolutionBackpropData::infer_conv_backprop_output_spatial_sh void op::v1::GroupConvolutionBackpropData::validate_and_infer_types() { OV_OP_SCOPE(v1_GroupConvolutionBackpropData_validate_and_infer_types); - element::Type data_et = get_input_element_type(0); - element::Type filters_et = get_input_element_type(1); + const auto& data_et = get_input_element_type(0); + const auto& filters_et = get_input_element_type(1); element::Type result_et; NODE_VALIDATION_CHECK(this, @@ -257,25 +243,19 @@ void op::v1::GroupConvolutionBackpropData::validate_and_infer_types() { ")."); } - const auto& data_shape = get_input_partial_shape(0); - const auto& filter_shape = get_input_partial_shape(1); - - auto& output_shapes_shape = output_shape_input_present ? get_input_partial_shape(2) : PartialShape::dynamic(); + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + const auto out_spatial_shape = get_convolution_output_shape(); + auto num_spatial = convolution::calculate_num_spatial(this, input_shapes, out_spatial_shape); - m_num_spatial = calculate_num_spatial(this, data_shape, filter_shape, output_shapes_shape, 2, 3); - update_and_validate_attributes_back_prop(this, m_num_spatial); - - std::vector input_shapes = {data_shape, filter_shape}; - if (output_shape_input_present) - input_shapes.push_back(get_input_partial_shape(2)); - std::vector output_shapes = {ov::PartialShape::dynamic()}; - - if (m_num_spatial != -1) { - ov::PartialShape output_spatial_shape = get_convolution_output_shape(); - resolve_auto_pad_for_shape_back_prop(this, m_pads_begin, m_pads_end, input_shapes, output_spatial_shape, 2, 3); - shape_infer(this, m_pads_begin, m_pads_end, output_spatial_shape, input_shapes, output_shapes); + if (num_spatial != util::num_spatial_undefined) { + resize_attributes(num_spatial); } + + const auto output_shapes = shape_infer(this, input_shapes, m_pads_begin, m_pads_end); set_output_type(0, result_et, output_shapes[0]); + set_num_spatial(num_spatial, input_shapes); set_input_is_relevant_to_shape(0); set_input_is_relevant_to_shape(1); @@ -305,3 +285,4 @@ shared_ptr op::v1::GroupConvolutionBackpropData::clone_with_new_inputs(con m_output_padding); } } +} // namespace ov diff --git a/src/core/src/op/gru_cell.cpp b/src/core/src/op/gru_cell.cpp index cc1871dc76992f..038d783419fc42 100644 --- a/src/core/src/op/gru_cell.cpp +++ b/src/core/src/op/gru_cell.cpp @@ -97,7 +97,9 @@ void op::v3::GRUCell::validate_and_infer_types() { "Element types for X, initial_hidden_state, W, R and B inputs do not " "match."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes{ov::PartialShape::dynamic(2)}; shape_infer(this, input_shapes, output_shapes); diff --git a/src/core/src/op/gru_sequence.cpp b/src/core/src/op/gru_sequence.cpp index e26a778f75450b..e5c0b0321a3d59 100644 --- a/src/core/src/op/gru_sequence.cpp +++ b/src/core/src/op/gru_sequence.cpp @@ -58,7 +58,9 @@ void op::v5::GRUSequence::validate_and_infer_types() { "Element types for X, initial_hidden_state, W, R and B inputs do not " "match."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes = {ov::PartialShape::dynamic(4), ov::PartialShape::dynamic(3)}; shape_infer(this, input_shapes, output_shapes); diff --git a/src/core/src/op/hsigmoid.cpp b/src/core/src/op/hsigmoid.cpp index a5fa489921eced..296a2bf868d5af 100644 --- a/src/core/src/op/hsigmoid.cpp +++ b/src/core/src/op/hsigmoid.cpp @@ -57,7 +57,9 @@ bool evaluate_hsigmoid(const HostTensorPtr& arg, const HostTensorPtr& out) { bool op::v5::HSigmoid::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v5_HSigmoid_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return evaluate_hsigmoid(inputs[0], outputs[0]); } diff --git a/src/core/src/op/hswish.cpp b/src/core/src/op/hswish.cpp index 59c240de90f395..387c74aacceda5 100644 --- a/src/core/src/op/hswish.cpp +++ b/src/core/src/op/hswish.cpp @@ -58,7 +58,9 @@ bool evaluate_hswish(const HostTensorPtr& arg, const HostTensorPtr& out) { bool op::v4::HSwish::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v4_HSwish_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return hswish::evaluate_hswish(inputs[0], outputs[0]); } diff --git a/src/core/src/op/if.cpp b/src/core/src/op/if.cpp index f04a0a2dc05b4f..cd008e419e30ba 100644 --- a/src/core/src/op/if.cpp +++ b/src/core/src/op/if.cpp @@ -98,7 +98,9 @@ void ov::op::v8::If::validate_and_infer_types() { } // Trying to get cond as const value + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& cond_value = get_constant_from_source(if_condition)) { + OPENVINO_SUPPRESS_DEPRECATED_END // If cond is const shape and inference is run for one of bodies another body is skipped auto val = cond_value->cast_vector(); NODE_VALIDATION_CHECK(this, diff --git a/src/core/src/op/interpolate.cpp b/src/core/src/op/interpolate.cpp index 6bfd961fc35de8..a8946b78242cde 100644 --- a/src/core/src/op/interpolate.cpp +++ b/src/core/src/op/interpolate.cpp @@ -112,7 +112,9 @@ std::vector ov::op::v4::Interpolate::get_axes() const { return default_value; } + OPENVINO_SUPPRESS_DEPRECATED_START auto axes_node = get_constant_from_source(input_value(3)); + OPENVINO_SUPPRESS_DEPRECATED_END NODE_VALIDATION_CHECK(this, axes_node, "Input 'axes' should be Constant or foldable."); return axes_node->cast_vector(); @@ -186,6 +188,21 @@ void ov::op::v4::Interpolate::validate_and_infer_types() { input_shapes = {input_shape, target_spatial_shape, scales, axes}; } + const auto interpolation_mode_check = [](const op::util::InterpolateBase::InterpolateMode mode) { + constexpr std::array allowed_modes = { + op::util::InterpolateBase::InterpolateMode::NEAREST, + op::util::InterpolateBase::InterpolateMode::LINEAR, + op::util::InterpolateBase::InterpolateMode::LINEAR_ONNX, + op::util::InterpolateBase::InterpolateMode::CUBIC}; + + return std::find(std::begin(allowed_modes), std::end(allowed_modes), mode) != std::end(allowed_modes); + }; + + NODE_VALIDATION_CHECK(this, + interpolation_mode_check(m_attrs.mode), + "Unsupported interpolation mode used with version 4 of the Interpolate op: ", + as_string(m_attrs.mode)); + util::correct_pads_attr(this, m_attrs.pads_begin, m_attrs.pads_end, input_shapes); shape_infer(this, m_attrs.pads_begin, m_attrs.pads_end, input_shapes, output_shapes, {}); set_output_type(0, get_input_element_type(0), output_shapes[0]); diff --git a/src/core/src/op/logical_and.cpp b/src/core/src/op/logical_and.cpp index 44c6729b346800..8464df3ed10ced 100644 --- a/src/core/src/op/logical_and.cpp +++ b/src/core/src/op/logical_and.cpp @@ -65,7 +65,9 @@ bool evaluate_logand(const HostTensorPtr& arg0, bool op::v1::LogicalAnd::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_LogicalAnd_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END return logand::evaluate_logand(inputs[0], inputs[1], outputs[0], get_autob()); } diff --git a/src/core/src/op/logical_not.cpp b/src/core/src/op/logical_not.cpp index a3221ec973e937..a4e7a2a6688b0b 100644 --- a/src/core/src/op/logical_not.cpp +++ b/src/core/src/op/logical_not.cpp @@ -63,7 +63,9 @@ bool evaluate_not(const HostTensorPtr& arg0, const HostTensorPtr& out, const siz bool op::v1::LogicalNot::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_LogicalNot_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return notop::evaluate_not(inputs[0], outputs[0], inputs[0]->get_element_count()); } diff --git a/src/core/src/op/logical_or.cpp b/src/core/src/op/logical_or.cpp index 15dbed38051e41..e6814304e3a718 100644 --- a/src/core/src/op/logical_or.cpp +++ b/src/core/src/op/logical_or.cpp @@ -59,7 +59,9 @@ bool evaluate_logor(const HostTensorPtr& arg0, bool op::v1::LogicalOr::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_LogicalOr_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END return logor::evaluate_logor(inputs[0], inputs[1], outputs[0], get_autob()); } diff --git a/src/core/src/op/loop.cpp b/src/core/src/op/loop.cpp index 34ada949fd4988..ef1f3b6ccdb091 100644 --- a/src/core/src/op/loop.cpp +++ b/src/core/src/op/loop.cpp @@ -61,7 +61,9 @@ void op::v5::Loop::validate_and_infer_types() { loop_condition_rank.compatible(1) || loop_condition_rank.compatible(0), "Rank of ExecutionCondition input must be equal to 0 or 1"); } + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& cond_value = get_constant_from_source(loop_execution_condition)) { + OPENVINO_SUPPRESS_DEPRECATED_END auto val = cond_value->cast_vector(); NODE_VALIDATION_CHECK(this, val.size() == 1, @@ -85,7 +87,9 @@ void op::v5::Loop::validate_and_infer_types() { body_condition_rank.compatible(0) || body_condition_rank.compatible(1), "Rank of BodyExecutionCondition output must be equal to 0 or 1"); } + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& cond_value = get_constant_from_source(body_execution_condition)) { + OPENVINO_SUPPRESS_DEPRECATED_END auto val = cond_value->cast_vector(); NODE_VALIDATION_CHECK(this, val.size() == 1, @@ -101,7 +105,9 @@ void op::v5::Loop::validate_and_infer_types() { // Const(true or false) -> Loop (body: Parameter -> execution_condition output) for (const auto& desc : get_input_descriptions()) { if (m_bodies[0]->get_parameters().at(desc->m_body_parameter_index) == cond_param) { + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& cond_value = get_constant_from_source(input_value(desc->m_input_index))) { + OPENVINO_SUPPRESS_DEPRECATED_END auto val = cond_value->cast_vector(); NODE_VALIDATION_CHECK(this, val.size() == 1, @@ -124,7 +130,9 @@ void op::v5::Loop::validate_and_infer_types() { trip_count_rank.compatible(1) || trip_count_rank.compatible(0), "Rank of TripCount input must be equal to 0 or 1"); } + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& trip_count_val = get_constant_from_source(trip_count)) { + OPENVINO_SUPPRESS_DEPRECATED_END auto val = trip_count_val->cast_vector(); NODE_VALIDATION_CHECK(this, val.size() == 1, @@ -168,8 +176,10 @@ void op::v5::Loop::validate_and_infer_types() { body_parameter->set_partial_shape(ov::PartialShape::dynamic()); } else { auto out_shape = input_partial_shape; + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = ngraph::normalize_axis(this, slice_input_description->m_axis, input_partial_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END out_shape[axis] = slice_input_description->m_part_size; body_parameter->set_partial_shape(out_shape); } @@ -278,7 +288,9 @@ void op::v5::Loop::validate_and_infer_types() { if (zero_number_of_iter) { out_shape = ov::PartialShape{0}; } else if (out_shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = ngraph::normalize_axis(this, concat_output_description->m_axis, out_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END const auto rank = out_shape.rank().get_length(); if (rank == 0) { out_shape = ov::PartialShape{1}; diff --git a/src/core/src/op/lrn.cpp b/src/core/src/op/lrn.cpp index ff8b3ea4f92654..05b7ea406dae5c 100644 --- a/src/core/src/op/lrn.cpp +++ b/src/core/src/op/lrn.cpp @@ -29,8 +29,11 @@ op::LRN::LRN(const Output& arg, const Output& axes, double alpha, do AxisSet op::LRN::get_reduction_axes() const { AxisSet axes{1}; // channel axis as default auto axes_input_node = input_value(1).get_node_shared_ptr(); - if (const auto& const_op = get_constant_from_source(axes_input_node)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (const auto& const_op = get_constant_from_source(axes_input_node)) { + OPENVINO_SUPPRESS_DEPRECATED_END axes = const_op->get_axis_set_val(); + } return axes; } diff --git a/src/core/src/op/max_pool.cpp b/src/core/src/op/max_pool.cpp index 13d98688c4d2cd..7651ea2979e332 100644 --- a/src/core/src/op/max_pool.cpp +++ b/src/core/src/op/max_pool.cpp @@ -41,7 +41,9 @@ bool ngraph::op::v1::MaxPool::visit_attributes(AttributeVisitor& visitor) { void op::v1::MaxPool::validate_and_infer_types() { OV_OP_SCOPE(v1_MaxPool_validate_and_infer_types); - const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_START + const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this), m_pads_begin, m_pads_end); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shapes.front()); } @@ -108,7 +110,9 @@ bool evaluate_maxpool(const HostTensorPtr& arg, bool op::v1::MaxPool::evaluate_maxpool(const HostTensorVector& outputs, const HostTensorVector& inputs) const { const auto input_shapes = std::vector{inputs[0]->get_partial_shape()}; - auto out_shape = shape_infer(this, input_shapes).front(); + auto pads_begin = m_pads_begin; + auto pads_end = m_pads_end; + auto out_shape = shape_infer(this, input_shapes, pads_begin, pads_end).front(); return maxpool::evaluate_maxpool(inputs[0], outputs[0], @@ -273,10 +277,14 @@ void op::v8::MaxPool::validate_and_infer_types() { const auto input_shape = get_input_partial_shape(0); if (input_shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START m_axis = ngraph::normalize_axis(this, m_axis, input_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END } - const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_START + const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this), m_pads_begin, m_pads_end); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shapes[0]); set_output_type(1, m_index_element_type, output_shapes[1]); } @@ -318,7 +326,9 @@ bool op::v8::MaxPool::evaluate(const HostTensorVector& outputs, const HostTensor OV_OP_SCOPE(v8_MaxPool_evaluate); const auto input_shapes = std::vector{inputs[0]->get_partial_shape()}; - auto out_shape = shape_infer(this, input_shapes).front(); + auto pads_begin = m_pads_begin; + auto pads_end = m_pads_end; + auto out_shape = shape_infer(this, input_shapes, pads_begin, pads_end).front(); return maxpool_v8::evaluate_maxpool(inputs[0], outputs[0], diff --git a/src/core/src/op/mish.cpp b/src/core/src/op/mish.cpp index 51b47e157c9c44..3c8f9479ad3d53 100644 --- a/src/core/src/op/mish.cpp +++ b/src/core/src/op/mish.cpp @@ -71,7 +71,9 @@ bool evaluate_mish(const HostTensorPtr& arg0, const HostTensorPtr& out) { bool op::v4::Mish::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v4_Mish_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return mish::evaluate_mish(inputs[0], outputs[0]); } diff --git a/src/core/src/op/negative.cpp b/src/core/src/op/negative.cpp index 64351efb0cbf15..e7f7461afdf6b4 100644 --- a/src/core/src/op/negative.cpp +++ b/src/core/src/op/negative.cpp @@ -57,8 +57,10 @@ bool evaluate_negative(const HostTensorPtr& arg0, const HostTensorPtr& out, cons bool op::Negative::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Negative_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 1)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return negativeop::evaluate_negative(inputs[0], outputs[0], shape_size(outputs[0]->get_shape())); } diff --git a/src/core/src/op/non_max_suppression.cpp b/src/core/src/op/non_max_suppression.cpp index 60ce03c95faf39..19d923019ac259 100644 --- a/src/core/src/op/non_max_suppression.cpp +++ b/src/core/src/op/non_max_suppression.cpp @@ -154,7 +154,9 @@ void op::v1::NonMaxSuppression::validate_and_infer_types() { "The last dimension of the 'boxes' input must be equal to 4. Got:", boxes_ps[2]); + OPENVINO_SUPPRESS_DEPRECATED_START const auto& max_output_boxes_input = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END if (num_boxes_boxes.is_static() && scores_ps[1].is_static() && max_output_boxes_input) { const auto num_boxes = num_boxes_boxes.get_length(); const auto max_output_boxes_per_class = max_output_boxes_input->cast_vector().at(0); @@ -168,7 +170,9 @@ void op::v1::NonMaxSuppression::validate_and_infer_types() { int64_t op::v1::NonMaxSuppression::max_boxes_output_from_input() const { int64_t max_output_boxes{0}; + OPENVINO_SUPPRESS_DEPRECATED_START const auto max_output_boxes_input = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END max_output_boxes = max_output_boxes_input->cast_vector().at(0); return max_output_boxes; @@ -336,7 +340,9 @@ void op::v3::NonMaxSuppression::validate_and_infer_types() { if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) { const auto num_boxes_boxes = boxes_ps[1]; + OPENVINO_SUPPRESS_DEPRECATED_START const auto max_output_boxes_input = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END if (num_boxes_boxes.is_static() && scores_ps[1].is_static() && max_output_boxes_input) { const auto num_boxes = num_boxes_boxes.get_length(); const auto num_classes = scores_ps[1].get_length(); @@ -351,7 +357,9 @@ void op::v3::NonMaxSuppression::validate_and_infer_types() { int64_t op::v3::NonMaxSuppression::max_boxes_output_from_input() const { int64_t max_output_boxes{0}; + OPENVINO_SUPPRESS_DEPRECATED_START const auto max_output_boxes_input = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END max_output_boxes = max_output_boxes_input->cast_vector().at(0); return max_output_boxes; @@ -444,7 +452,9 @@ void op::v4::NonMaxSuppression::validate_and_infer_types() { if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) { const auto num_boxes_boxes = boxes_ps[1]; + OPENVINO_SUPPRESS_DEPRECATED_START const auto max_output_boxes_input = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END if (num_boxes_boxes.is_static() && scores_ps[0].is_static() && scores_ps[1].is_static() && max_output_boxes_input) { const auto num_boxes = num_boxes_boxes.get_length(); @@ -713,7 +723,9 @@ int64_t op::v5::NonMaxSuppression::max_boxes_output_from_input() const { return 0; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto max_output_boxes_input = get_constant_from_source(input_value(max_output_boxes_port)); + OPENVINO_SUPPRESS_DEPRECATED_END max_output_boxes = max_output_boxes_input->cast_vector().at(0); return max_output_boxes; @@ -726,7 +738,9 @@ float op::v5::NonMaxSuppression::iou_threshold_from_input() const { return iou_threshold; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto iou_threshold_input = get_constant_from_source(input_value(iou_threshold_port)); + OPENVINO_SUPPRESS_DEPRECATED_END iou_threshold = iou_threshold_input->cast_vector().at(0); return iou_threshold; @@ -739,7 +753,9 @@ float op::v5::NonMaxSuppression::score_threshold_from_input() const { return score_threshold; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto score_threshold_input = get_constant_from_source(input_value(score_threshold_port)); + OPENVINO_SUPPRESS_DEPRECATED_END score_threshold = score_threshold_input->cast_vector().at(0); return score_threshold; @@ -752,7 +768,9 @@ float op::v5::NonMaxSuppression::soft_nms_sigma_from_input() const { return soft_nms_sigma; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto soft_nms_sigma_input = get_constant_from_source(input_value(soft_nms_sigma_port)); + OPENVINO_SUPPRESS_DEPRECATED_END soft_nms_sigma = soft_nms_sigma_input->cast_vector().at(0); return soft_nms_sigma; @@ -1021,7 +1039,9 @@ int64_t op::v9::NonMaxSuppression::max_boxes_output_from_input() const { return 0; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto max_output_boxes_input = get_constant_from_source(input_value(max_output_boxes_port)); + OPENVINO_SUPPRESS_DEPRECATED_END max_output_boxes = max_output_boxes_input->cast_vector().at(0); return max_output_boxes; @@ -1034,7 +1054,9 @@ float op::v9::NonMaxSuppression::iou_threshold_from_input() const { return iou_threshold; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto iou_threshold_input = get_constant_from_source(input_value(iou_threshold_port)); + OPENVINO_SUPPRESS_DEPRECATED_END iou_threshold = iou_threshold_input->cast_vector().at(0); return iou_threshold; @@ -1047,7 +1069,9 @@ float op::v9::NonMaxSuppression::score_threshold_from_input() const { return score_threshold; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto score_threshold_input = get_constant_from_source(input_value(score_threshold_port)); + OPENVINO_SUPPRESS_DEPRECATED_END score_threshold = score_threshold_input->cast_vector().at(0); return score_threshold; @@ -1060,7 +1084,9 @@ float op::v9::NonMaxSuppression::soft_nms_sigma_from_input() const { return soft_nms_sigma; } + OPENVINO_SUPPRESS_DEPRECATED_START const auto soft_nms_sigma_input = get_constant_from_source(input_value(soft_nms_sigma_port)); + OPENVINO_SUPPRESS_DEPRECATED_END soft_nms_sigma = soft_nms_sigma_input->cast_vector().at(0); return soft_nms_sigma; diff --git a/src/core/src/op/non_zero.cpp b/src/core/src/op/non_zero.cpp index c98b145808a221..16caf890782fa0 100644 --- a/src/core/src/op/non_zero.cpp +++ b/src/core/src/op/non_zero.cpp @@ -56,7 +56,9 @@ void op::v3::NonZero::validate_and_infer_types() { set_input_is_relevant_to_shape(0); + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& input_constant = get_constant_from_source(input_value(0))) { + OPENVINO_SUPPRESS_DEPRECATED_END // input_value is available to calculate output shape const auto& input_data = std::make_shared(input_constant); auto output = std::make_shared(m_output_type, get_output_partial_shape(0)); diff --git a/src/core/src/op/normalize_l2.cpp b/src/core/src/op/normalize_l2.cpp index 58e616e2a19bad..a714836c7f214f 100644 --- a/src/core/src/op/normalize_l2.cpp +++ b/src/core/src/op/normalize_l2.cpp @@ -66,10 +66,14 @@ void op::v0::NormalizeL2::validate_and_infer_types() { AxisSet op::v0::NormalizeL2::get_reduction_axes() const { AxisSet axes; + OPENVINO_SUPPRESS_DEPRECATED_START if (auto const_op = get_constant_from_source(input_value(1))) { + OPENVINO_SUPPRESS_DEPRECATED_END const auto const_data = const_op->cast_vector(); const auto input_data_rank = get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_axes = ov::normalize_axes(get_friendly_name(), const_data, input_data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END axes = AxisSet{normalized_axes}; } return axes; diff --git a/src/core/src/op/not_equal.cpp b/src/core/src/op/not_equal.cpp index bfc07058bfe340..f0971c1e5a371d 100644 --- a/src/core/src/op/not_equal.cpp +++ b/src/core/src/op/not_equal.cpp @@ -65,7 +65,9 @@ shared_ptr op::v1::NotEqual::clone_with_new_inputs(const OutputVector& new bool op::v1::NotEqual::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_NotEqual_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END return not_equalop::evaluate_not_equal(inputs[0], inputs[1], outputs[0], get_autob()); } diff --git a/src/core/src/op/one_hot.cpp b/src/core/src/op/one_hot.cpp index adc6373075d7b0..5c90ec2539bc20 100644 --- a/src/core/src/op/one_hot.cpp +++ b/src/core/src/op/one_hot.cpp @@ -106,8 +106,10 @@ bool evaluate_onehot(const HostTensorVector& output_values, const HostTensorVect bool op::v1::OneHot::evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const { OV_OP_SCOPE(v1_OneHot_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(input_values, 4)); NGRAPH_CHECK(validate_host_tensor_vector(output_values, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& ind_Pshape = input_values[0]->get_partial_shape(); const auto& out_Pshape = output_values[0]->get_partial_shape(); diff --git a/src/core/src/op/pad.cpp b/src/core/src/op/pad.cpp index f601ff142357a8..1669b8882611e7 100644 --- a/src/core/src/op/pad.cpp +++ b/src/core/src/op/pad.cpp @@ -45,7 +45,9 @@ op::v1::Pad::Pad(const Output& arg, CoordinateDiff op::v1::Pad::get_pads_begin() const { CoordinateDiff pads_begin_coord{}; + OPENVINO_SUPPRESS_DEPRECATED_START if (auto pads_begin_const = get_constant_from_source(input_value(1))) { + OPENVINO_SUPPRESS_DEPRECATED_END pads_begin_coord = pads_begin_const->cast_vector(); } return pads_begin_coord; @@ -53,7 +55,9 @@ CoordinateDiff op::v1::Pad::get_pads_begin() const { CoordinateDiff op::v1::Pad::get_pads_end() const { CoordinateDiff pads_end_coord{}; + OPENVINO_SUPPRESS_DEPRECATED_START if (auto pads_end_const = get_constant_from_source(input_value(2))) { + OPENVINO_SUPPRESS_DEPRECATED_END pads_end_coord = pads_end_const->cast_vector(); } return pads_end_coord; @@ -104,7 +108,9 @@ void op::v1::Pad::validate_and_infer_types() { pads_end_element_type, ")."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, result_et, output_shapes[0]); } @@ -183,5 +189,7 @@ bool op::v1::Pad::evaluate_upper(ov::TensorVector& output_values) const { bool op::v1::Pad::evaluate_label(ov::TensorLabelVector& output_labels) const { OV_OP_SCOPE(v1_Pad_evaluate_label); + OPENVINO_SUPPRESS_DEPRECATED_START return ov::default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/prelu.cpp b/src/core/src/op/prelu.cpp index 6aadeb877206d4..6044ee722d8d5e 100644 --- a/src/core/src/op/prelu.cpp +++ b/src/core/src/op/prelu.cpp @@ -64,7 +64,9 @@ bool evaluate_prelu(const ngraph::HostTensorPtr& arg, bool ov::op::v0::PRelu::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_PRelu_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(ngraph::validate_host_tensor_vector(outputs, 1) && ngraph::validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END return prelu::evaluate_prelu(inputs[0], inputs[1], outputs[0]); } diff --git a/src/core/src/op/prior_box.cpp b/src/core/src/op/prior_box.cpp index cef4a6f1f542c3..f84806a9e72b15 100644 --- a/src/core/src/op/prior_box.cpp +++ b/src/core/src/op/prior_box.cpp @@ -50,7 +50,9 @@ void op::v0::PriorBox::validate_and_infer_types() { set_input_is_relevant_to_shape(0); PartialShape spatials; + OPENVINO_SUPPRESS_DEPRECATED_START if (evaluate_as_partial_shape(input_value(0), spatials)) { + OPENVINO_SUPPRESS_DEPRECATED_END NODE_VALIDATION_CHECK(this, spatials.rank().is_static() && spatials.size() == 2, "Layer shape must have rank 2", @@ -237,7 +239,9 @@ void op::v8::PriorBox::validate_and_infer_types() { set_input_is_relevant_to_shape(0); PartialShape spatials; + OPENVINO_SUPPRESS_DEPRECATED_START if (evaluate_as_partial_shape(input_value(0), spatials)) { + OPENVINO_SUPPRESS_DEPRECATED_END NODE_VALIDATION_CHECK(this, spatials.rank().is_static() && spatials.size() == 2, "Layer shape must have rank 2", diff --git a/src/core/src/op/prior_box_clustered.cpp b/src/core/src/op/prior_box_clustered.cpp index c06c8a80f47d29..61e298a7f642f4 100644 --- a/src/core/src/op/prior_box_clustered.cpp +++ b/src/core/src/op/prior_box_clustered.cpp @@ -55,7 +55,9 @@ void ov::op::v0::PriorBoxClustered::validate_and_infer_types() { set_input_is_relevant_to_shape(0); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto const_shape = get_constant_from_source(input_value(0).get_node_shared_ptr())) { + OPENVINO_SUPPRESS_DEPRECATED_END NODE_VALIDATION_CHECK(this, shape_size(const_shape->get_shape()) == 2, "Layer shape must have rank 2", diff --git a/src/core/src/op/psroi_pooling.cpp b/src/core/src/op/psroi_pooling.cpp index 1b211567980159..7b4e2748960398 100644 --- a/src/core/src/op/psroi_pooling.cpp +++ b/src/core/src/op/psroi_pooling.cpp @@ -2,22 +2,27 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/psroi_pooling.hpp" +#include "openvino/op/psroi_pooling.hpp" #include "itt.hpp" -#include "ngraph/attribute_visitor.hpp" +#include "openvino/core/attribute_visitor.hpp" +#include "openvino/core/validation_util.hpp" +#include "psroi_pooling_shape_inference.hpp" using namespace std; -using namespace ngraph; -ov::op::v0::PSROIPooling::PSROIPooling(const Output& input, - const Output& coords, - const size_t output_dim, - const size_t group_size, - const float spatial_scale, - int spatial_bins_x, - int spatial_bins_y, - const string& mode) +namespace ov { +namespace op { +namespace v0 { + +PSROIPooling::PSROIPooling(const Output& input, + const Output& coords, + const size_t output_dim, + const size_t group_size, + const float spatial_scale, + int spatial_bins_x, + int spatial_bins_y, + const string& mode) : Op({input, coords}), m_output_dim(output_dim), m_group_size(group_size), @@ -28,7 +33,7 @@ ov::op::v0::PSROIPooling::PSROIPooling(const Output& input, constructor_validate_and_infer_types(); } -bool ngraph::op::v0::PSROIPooling::visit_attributes(AttributeVisitor& visitor) { +bool PSROIPooling::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v0_PSROIPooling_visit_attributes); visitor.on_attribute("output_dim", m_output_dim); visitor.on_attribute("group_size", m_group_size); @@ -39,70 +44,24 @@ bool ngraph::op::v0::PSROIPooling::visit_attributes(AttributeVisitor& visitor) { return true; } -void ov::op::v0::PSROIPooling::validate_and_infer_types() { +void PSROIPooling::validate_and_infer_types() { OV_OP_SCOPE(v0_PSROIPooling_validate_and_infer_types); - auto feat_maps_et = get_input_element_type(0); - auto coords_et = get_input_element_type(1); + const auto& feat_maps_et = get_input_element_type(0); + const auto& coords_et = get_input_element_type(1); NODE_VALIDATION_CHECK(this, feat_maps_et.is_real(), "Feature maps' data type must be floating point. Got " + feat_maps_et.get_type_name()); NODE_VALIDATION_CHECK(this, coords_et.is_real(), "Coords' data type must be floating point. Got " + coords_et.get_type_name()); - NODE_VALIDATION_CHECK(this, - m_mode == "average" || m_mode == "bilinear", - "Expected 'average' or 'bilinear' mode. Got " + m_mode); - NODE_VALIDATION_CHECK(this, m_group_size > 0, "group_size has to be greater than 0"); - if (m_mode == "bilinear") { - NODE_VALIDATION_CHECK(this, m_spatial_bins_x > 0, "spatial_bins_x has to be greater than 0"); - NODE_VALIDATION_CHECK(this, m_spatial_bins_y > 0, "spatial_bins_y has to be greater than 0"); - } - const ov::PartialShape& feat_map_pshape = get_input_partial_shape(0); - const ov::PartialShape& coords_pshape = get_input_partial_shape(1); - if (feat_map_pshape.rank().is_dynamic() || coords_pshape.rank().is_dynamic()) { - set_output_type(0, feat_maps_et, ov::PartialShape::dynamic()); - } else { - NODE_VALIDATION_CHECK(this, - feat_map_pshape.rank().get_length() == 4, - "PSROIPooling expects 4 dimensions for input. Got ", - feat_map_pshape.rank().get_length()); - NODE_VALIDATION_CHECK(this, - coords_pshape.rank().get_length() == 2, - "PSROIPooling expects 2 dimensions for box coordinates. Got ", - coords_pshape.rank().get_length()); - - if (feat_map_pshape[1].is_static()) { - auto num_input_channels = feat_map_pshape[1].get_interval().get_min_val(); - if (m_mode == "average") { - NODE_VALIDATION_CHECK(this, - num_input_channels % (m_group_size * m_group_size) == 0, - "Number of input's channels must be a multiply of group_size * group_size"); - NODE_VALIDATION_CHECK(this, - m_output_dim == num_input_channels / (m_group_size * m_group_size), - "output_dim must be equal to input channels divided by " - "group_size * group_size"); - } else if (m_mode == "bilinear") { - NODE_VALIDATION_CHECK(this, - num_input_channels % (m_spatial_bins_x * m_spatial_bins_y) == 0, - "Number of input's channels must be a multiply of " - "spatial_bins_x * spatial_bins_y"); - NODE_VALIDATION_CHECK( - this, - m_output_dim == static_cast(num_input_channels / (m_spatial_bins_x * m_spatial_bins_y)), - "output_dim must be equal to input channels divided by " - "spatial_bins_x * spatial_bins_y"); - } - } - std::vector output_shape{coords_pshape[0], static_cast(m_output_dim)}; - for (int64_t i = 2; i < feat_map_pshape.rank().get_length(); i++) { - output_shape.emplace_back(m_group_size); - } - set_output_type(0, feat_maps_et, output_shape); - } + OPENVINO_SUPPRESS_DEPRECATED_START + const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_END + set_output_type(0, feat_maps_et, output_shapes[0]); } -shared_ptr ov::op::v0::PSROIPooling::clone_with_new_inputs(const OutputVector& new_args) const { +shared_ptr PSROIPooling::clone_with_new_inputs(const OutputVector& new_args) const { OV_OP_SCOPE(v0_PSROIPooling_clone_with_new_inputs); check_new_args_count(this, new_args); return make_shared(new_args.at(0), @@ -114,3 +73,30 @@ shared_ptr ov::op::v0::PSROIPooling::clone_with_new_inputs(const OutputVec m_spatial_bins_y, m_mode); } + +void PSROIPooling::set_output_dim(size_t output_dim) { + m_output_dim = output_dim; +} + +void PSROIPooling::set_group_size(size_t group_size) { + m_group_size = group_size; +} + +void PSROIPooling::set_spatial_scale(float scale) { + m_spatial_scale = scale; +} + +void PSROIPooling::set_spatial_bins_x(int x) { + m_spatial_bins_x = x; +} + +void PSROIPooling::set_spatial_bins_y(int y) { + m_spatial_bins_y = y; +} + +void PSROIPooling::set_mode(std::string mode) { + m_mode = std::move(mode); +} +} // namespace v0 +} // namespace op +} // namespace ov diff --git a/src/core/src/op/random_uniform.cpp b/src/core/src/op/random_uniform.cpp index 602fe00ea562b6..b756d529c292a4 100644 --- a/src/core/src/op/random_uniform.cpp +++ b/src/core/src/op/random_uniform.cpp @@ -40,7 +40,9 @@ void op::v8::RandomUniform::validate_and_infer_types() { input_shape.rank() == 1, "The rank of the tensor defining output shape must be equal to 1."); + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& const_shape = get_constant_from_source(input_value(0))) { + OPENVINO_SUPPRESS_DEPRECATED_END output_shape = ov::PartialShape(const_shape->cast_vector()); } else { output_shape = ov::PartialShape::dynamic(input_shape[0]); @@ -76,8 +78,10 @@ void op::v8::RandomUniform::validate_and_infer_types() { min_element_type == get_out_type(), "'min_val' and 'max_val' should have the same type as 'out_type' attribute."); + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& const_min = get_constant_from_source(input_value(1))) { if (const auto& const_max = get_constant_from_source(input_value(2))) { + OPENVINO_SUPPRESS_DEPRECATED_END if (get_out_type() == ngraph::element::Type_t::i64 || get_out_type() == ngraph::element::Type_t::i32) { int64_t min_val = const_min->cast_vector()[0]; int64_t max_val = const_max->cast_vector()[0]; diff --git a/src/core/src/op/range.cpp b/src/core/src/op/range.cpp index 5532a4c0f2aab9..77d11edcb28ebe 100644 --- a/src/core/src/op/range.cpp +++ b/src/core/src/op/range.cpp @@ -272,9 +272,11 @@ adjust_for_step_and_sign(T span, T step) { template static ov::PartialShape infer_output_shape(const op::v0::Range* node, const element::Type& /* et */) { + OPENVINO_SUPPRESS_DEPRECATED_START auto const_start = get_constant_from_source(node->input_value(0)); auto const_stop = get_constant_from_source(node->input_value(1)); auto const_step = get_constant_from_source(node->input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END T start = static_cast(0); T stop = static_cast(0); diff --git a/src/core/src/op/reduce_l1.cpp b/src/core/src/op/reduce_l1.cpp index 7e727a9e71fa16..622bc457851282 100644 --- a/src/core/src/op/reduce_l1.cpp +++ b/src/core/src/op/reduce_l1.cpp @@ -55,8 +55,10 @@ bool evaluate_sum(const HostTensorPtr& arg, const HostTensorPtr& out, const Axis bool op::v4::ReduceL1::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v4_ReduceL1_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/reduce_l2.cpp b/src/core/src/op/reduce_l2.cpp index 5a37d7d8883ca4..6a59a62885ad1a 100644 --- a/src/core/src/op/reduce_l2.cpp +++ b/src/core/src/op/reduce_l2.cpp @@ -53,8 +53,10 @@ bool evaluate_reduce_l2(const HostTensorPtr& arg, const HostTensorPtr& out, cons bool op::v4::ReduceL2::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v4_ReduceL2_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/reduce_logical_and.cpp b/src/core/src/op/reduce_logical_and.cpp index 5e2a7267c40ac4..7033f76a78fe37 100644 --- a/src/core/src/op/reduce_logical_and.cpp +++ b/src/core/src/op/reduce_logical_and.cpp @@ -51,8 +51,11 @@ bool evaluate_reduce_logical_and(const HostTensorPtr& data, bool op::v1::ReduceLogicalAnd::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceLogicalAnd_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END + const auto& data = inputs[0]; const auto& axes = inputs[1]; const auto& out = outputs[0]; diff --git a/src/core/src/op/reduce_logical_or.cpp b/src/core/src/op/reduce_logical_or.cpp index fecd760fbabb1e..e401d7d27a5622 100644 --- a/src/core/src/op/reduce_logical_or.cpp +++ b/src/core/src/op/reduce_logical_or.cpp @@ -51,8 +51,11 @@ bool evaluate_reduce_logical_or(const HostTensorPtr& data, bool op::v1::ReduceLogicalOr::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceLogicalOr_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END + const auto& data = inputs[0]; const auto& axes = inputs[1]; const auto& out = outputs[0]; diff --git a/src/core/src/op/reduce_max.cpp b/src/core/src/op/reduce_max.cpp index 0fa1c4f30a80db..d6faa03bac7529 100644 --- a/src/core/src/op/reduce_max.cpp +++ b/src/core/src/op/reduce_max.cpp @@ -57,8 +57,10 @@ shared_ptr op::v1::ReduceMax::clone_with_new_inputs(const OutputVector& ne bool op::v1::ReduceMax::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceMax_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/reduce_mean.cpp b/src/core/src/op/reduce_mean.cpp index 9f7f7bf04299db..48885583431782 100644 --- a/src/core/src/op/reduce_mean.cpp +++ b/src/core/src/op/reduce_mean.cpp @@ -57,8 +57,10 @@ bool evaluate_mean(const HostTensorPtr& arg, const HostTensorPtr& out, const Axi bool op::v1::ReduceMean::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceMean_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/reduce_min.cpp b/src/core/src/op/reduce_min.cpp index 6066ace0c26647..0bc74d92232833 100644 --- a/src/core/src/op/reduce_min.cpp +++ b/src/core/src/op/reduce_min.cpp @@ -57,8 +57,10 @@ shared_ptr op::v1::ReduceMin::clone_with_new_inputs(const OutputVector& ne bool op::v1::ReduceMin::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceMin_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/reduce_prod.cpp b/src/core/src/op/reduce_prod.cpp index f015a2dcf56d8c..fd7a67dc3701c3 100644 --- a/src/core/src/op/reduce_prod.cpp +++ b/src/core/src/op/reduce_prod.cpp @@ -55,8 +55,10 @@ bool evaluate_product(const HostTensorPtr& arg, const HostTensorPtr& out, const bool op::v1::ReduceProd::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceProd_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/reduce_sum.cpp b/src/core/src/op/reduce_sum.cpp index df1f7aaaf881f2..e945b7ecafc9fd 100644 --- a/src/core/src/op/reduce_sum.cpp +++ b/src/core/src/op/reduce_sum.cpp @@ -58,8 +58,10 @@ bool evaluate_sum(const HostTensorPtr& arg, const HostTensorPtr& out, const Axis bool op::v1::ReduceSum::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_ReduceSum_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto reduction_axes = get_normalized_axes_from_tensor(inputs[1], inputs[0]->get_partial_shape().rank(), get_friendly_name()); diff --git a/src/core/src/op/relu.cpp b/src/core/src/op/relu.cpp index 6d862359c84c1e..f2ab8190a309d7 100644 --- a/src/core/src/op/relu.cpp +++ b/src/core/src/op/relu.cpp @@ -56,7 +56,9 @@ bool evaluate_relu(const HostTensorPtr& arg0, const HostTensorPtr& out) { bool op::Relu::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Relu_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return relu::evaluate_relu(inputs[0], outputs[0]); } diff --git a/src/core/src/op/reshape.cpp b/src/core/src/op/reshape.cpp index 4182e7aaad462d..5bfd9993d76bcf 100644 --- a/src/core/src/op/reshape.cpp +++ b/src/core/src/op/reshape.cpp @@ -190,8 +190,10 @@ bool op::v1::Reshape::evaluate_reshape(const HostTensorVector& outputs, const Ho bool op::v1::Reshape::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_Reshape_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return evaluate_reshape(outputs, inputs); } @@ -224,7 +226,9 @@ bool op::v1::Reshape::evaluate_upper(ov::TensorVector& output_values) const { bool op::v1::Reshape::evaluate_label(TensorLabelVector& output_labels) const { if (!get_input_tensor(1).has_and_set_bound()) return false; + OPENVINO_SUPPRESS_DEPRECATED_START return default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } bool op::v1::Reshape::constant_fold(OutputVector& output_values, const OutputVector& inputs_values) { diff --git a/src/core/src/op/reverse.cpp b/src/core/src/op/reverse.cpp index af1d6ac8b06100..b9afee30fe63d3 100644 --- a/src/core/src/op/reverse.cpp +++ b/src/core/src/op/reverse.cpp @@ -50,7 +50,9 @@ void op::v1::Reverse::validate_and_infer_types() { "In 'index' mode the second input must contain integer values."); } + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); } diff --git a/src/core/src/op/reverse_sequence.cpp b/src/core/src/op/reverse_sequence.cpp index dc476232a27ffb..af6ab100e627fc 100644 --- a/src/core/src/op/reverse_sequence.cpp +++ b/src/core/src/op/reverse_sequence.cpp @@ -41,10 +41,14 @@ void op::ReverseSequence::validate_and_infer_types() { "Sequence lengths element type must be numeric type. Got: ", seq_lengths_et); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); + OPENVINO_SUPPRESS_DEPRECATED_START m_normalized_seq_axis = ov::normalize_axis(this, m_seq_axis, get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END } shared_ptr op::ReverseSequence::clone_with_new_inputs(const OutputVector& new_args) const { @@ -59,10 +63,14 @@ void op::ReverseSequence::set_batch_axis(int64_t batch_axis) { size_t op::ReverseSequence::get_batch_axis() const { const auto& data_rank = get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START return static_cast(ov::normalize_axis(this, m_batch_axis, data_rank)); + OPENVINO_SUPPRESS_DEPRECATED_END } void op::ReverseSequence::set_sequence_axis(int64_t sequence_axis) { m_seq_axis = sequence_axis; + OPENVINO_SUPPRESS_DEPRECATED_START m_normalized_seq_axis = ov::normalize_axis(this, m_seq_axis, get_input_partial_shape(0).rank()); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/roi_pooling.cpp b/src/core/src/op/roi_pooling.cpp index d0baa803933db5..7b508e46a14338 100644 --- a/src/core/src/op/roi_pooling.cpp +++ b/src/core/src/op/roi_pooling.cpp @@ -2,18 +2,22 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/roi_pooling.hpp" +#include "openvino/op/roi_pooling.hpp" #include "itt.hpp" +#include "openvino/core/validation_util.hpp" +#include "roi_pooling_shape_inference.hpp" using namespace std; -using namespace ngraph; -op::ROIPooling::ROIPooling(const Output& input, - const Output& coords, - const ov::Shape& output_size, - const float spatial_scale, - const string& method) +namespace ov { +namespace op { +namespace v0 { +ROIPooling::ROIPooling(const Output& input, + const Output& coords, + const ov::Shape& output_size, + const float spatial_scale, + const string& method) : Op({input, coords}), m_output_size(output_size), m_spatial_scale(spatial_scale), @@ -21,10 +25,10 @@ op::ROIPooling::ROIPooling(const Output& input, constructor_validate_and_infer_types(); } -void op::ROIPooling::validate_and_infer_types() { +void ROIPooling::validate_and_infer_types() { OV_OP_SCOPE(v0_ROIPooling_validate_and_infer_types); - auto feat_maps_et = get_input_element_type(0); - auto coords_et = get_input_element_type(1); + const auto& feat_maps_et = get_input_element_type(0); + const auto& coords_et = get_input_element_type(1); NODE_VALIDATION_CHECK(this, feat_maps_et.is_real() && coords_et.is_real(), "The data type for input and ROIs is expected to be a floating point type. Got: ", @@ -34,72 +38,18 @@ void op::ROIPooling::validate_and_infer_types() { NODE_VALIDATION_CHECK(this, feat_maps_et == coords_et, - "Type of feature maps (inputs) and rois is expected to be the same. Got: ", + "Type of feature maps (inputs) and ROIs is expected to be the same. Got: ", feat_maps_et, " and: ", coords_et); - NODE_VALIDATION_CHECK(this, - m_output_size.size() == 2, - "The dimension of pooled size is expected to be equal to 2. Got: ", - m_output_size.size()); - - NODE_VALIDATION_CHECK(this, - m_output_size[0] > 0 && m_output_size[1] > 0, - "Pooled size attributes pooled_h and pooled_w should should be " - "non-negative integers. Got: ", - m_output_size[0], - " and: ", - m_output_size[1], - "respectively"); - - NODE_VALIDATION_CHECK(this, - m_spatial_scale > 0, - "The spatial scale attribute should be a positive floating point number. Got: ", - m_spatial_scale); - - NODE_VALIDATION_CHECK(this, - m_method == "max" || m_method == "bilinear", - "Pooling method attribute should be either \'max\' or \'bilinear\'. Got: ", - m_method); + OPENVINO_SUPPRESS_DEPRECATED_START + const auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_END + set_output_type(0, feat_maps_et, output_shapes[0]); const auto& feat_maps_ps = get_input_partial_shape(0); - NODE_VALIDATION_CHECK(this, - feat_maps_ps.rank().compatible(4), - "Expected a 4D tensor for the feature maps input. Got: ", - feat_maps_ps); - const auto& coords_ps = get_input_partial_shape(1); - NODE_VALIDATION_CHECK(this, - coords_ps.rank().compatible(2), - "Expected a 2D tensor for the ROIs input with box coordinates. Got: ", - coords_ps); - - if (coords_ps.rank().is_static()) { - const auto coords_second_dim = coords_ps[1]; - NODE_VALIDATION_CHECK(this, - coords_second_dim.compatible(5), - "The second dimension of ROIs input should contain batch id and box coordinates. ", - "This dimension is expected to be equal to 5. Got: ", - coords_second_dim); - } - - // output shape should be {NUM_ROIS, C, pooled_h, pooled_w} - auto output_shape = ov::PartialShape{{Dimension::dynamic(), - Dimension::dynamic(), - Dimension{static_cast(m_output_size[0])}, - Dimension{static_cast(m_output_size[1])}}}; - - if (coords_ps.rank().is_static()) { - output_shape[0] = coords_ps[0]; - } - - if (feat_maps_ps.rank().is_static()) { - output_shape[1] = feat_maps_ps[1]; - } - - set_output_size(1); - set_output_type(0, feat_maps_et, output_shape); // if channel dimension, C, not known // feature maps input is used by shape specialization pass @@ -114,13 +64,13 @@ void op::ROIPooling::validate_and_infer_types() { } } -shared_ptr op::ROIPooling::clone_with_new_inputs(const OutputVector& new_args) const { +shared_ptr ROIPooling::clone_with_new_inputs(const OutputVector& new_args) const { OV_OP_SCOPE(v0_ROIPooling_clone_with_new_inputs); check_new_args_count(this, new_args); return make_shared(new_args.at(0), new_args.at(1), m_output_size, m_spatial_scale, m_method); } -bool op::ROIPooling::visit_attributes(AttributeVisitor& visitor) { +bool ROIPooling::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v0_ROIPooling_visit_attributes); visitor.on_attribute("output_size", m_output_size); visitor.on_attribute("pooled_h", m_output_size[0]); @@ -129,3 +79,21 @@ bool op::ROIPooling::visit_attributes(AttributeVisitor& visitor) { visitor.on_attribute("method", m_method); return true; } + +void ROIPooling::set_output_roi(Shape output_size) { + m_output_size = std::move(output_size); +} +const Shape& ROIPooling::get_output_roi() const { + return m_output_size; +} + +void ROIPooling::set_spatial_scale(float scale) { + m_spatial_scale = scale; +} + +void ROIPooling::set_method(std::string method_name) { + m_method = std::move(method_name); +} +} // namespace v0 +} // namespace op +} // namespace ov diff --git a/src/core/src/op/roll.cpp b/src/core/src/op/roll.cpp index f68be96300c216..01fc204d152dcc 100644 --- a/src/core/src/op/roll.cpp +++ b/src/core/src/op/roll.cpp @@ -29,7 +29,9 @@ void Roll::validate_and_infer_types() { axes_et.is_dynamic() || axes_et == element::i32 || axes_et == element::i64, "Axes must have int32 or int64 element type."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); } diff --git a/src/core/src/op/scatter_elements_update.cpp b/src/core/src/op/scatter_elements_update.cpp index 365745255332e2..721ef2255ce911 100644 --- a/src/core/src/op/scatter_elements_update.cpp +++ b/src/core/src/op/scatter_elements_update.cpp @@ -52,7 +52,9 @@ void op::v3::ScatterElementsUpdate::validate_and_infer_types() { " and: ", updates_et); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, data_et, output_shape); if (output_shape.is_dynamic()) set_input_is_relevant_to_shape(0); @@ -198,9 +200,13 @@ bool op::v3::ScatterElementsUpdate::evaluate_scatter_element_update(const HostTe if (normalized_axis < 0) { if (input_rank.is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START normalized_axis = ngraph::normalize_axis(this, axis, input_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else { + OPENVINO_SUPPRESS_DEPRECATED_START normalized_axis = ngraph::normalize_axis(this, axis, static_cast(inputs[0]->get_shape().size())); + OPENVINO_SUPPRESS_DEPRECATED_END } } @@ -261,5 +267,7 @@ bool op::v3::ScatterElementsUpdate::evaluate_upper(ov::TensorVector& output_valu bool op::v3::ScatterElementsUpdate::evaluate_label(TensorLabelVector& output_labels) const { OV_OP_SCOPE(v3_ScatterNDUpdate_evaluate_label); + OPENVINO_SUPPRESS_DEPRECATED_START return ov::default_label_evaluator(this, {0, 2}, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/scatter_nd_update.cpp b/src/core/src/op/scatter_nd_update.cpp index e67a4e992c6e1d..8ba94140f5e4e4 100644 --- a/src/core/src/op/scatter_nd_update.cpp +++ b/src/core/src/op/scatter_nd_update.cpp @@ -82,8 +82,10 @@ bool evaluate_scatter(const HostTensorPtr& arg0, bool op::v3::ScatterNDUpdate::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v3_ScatterNDUpdate_evaluate); NGRAPH_CHECK(!inputs.empty()); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 3)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return scatter::evaluate_scatter(inputs[0], inputs[1], inputs[2], outputs[0]); } @@ -126,5 +128,7 @@ bool op::v3::ScatterNDUpdate::evaluate_upper(ov::TensorVector& output_values) co bool op::v3::ScatterNDUpdate::evaluate_label(TensorLabelVector& output_labels) const { OV_OP_SCOPE(v3_ScatterNDUpdate_evaluate_label); + OPENVINO_SUPPRESS_DEPRECATED_START return ov::default_label_evaluator(this, {0, 2}, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/scatter_update.cpp b/src/core/src/op/scatter_update.cpp index 1b754dc0a22143..2d3cb0c28eb3b6 100644 --- a/src/core/src/op/scatter_update.cpp +++ b/src/core/src/op/scatter_update.cpp @@ -58,7 +58,9 @@ bool op::v3::ScatterUpdate::evaluate_scatter_update(const HostTensorVector& outp int64_t axis_val = host_tensor_2_vector(axis)[0]; if (axis_val < 0) { + OPENVINO_SUPPRESS_DEPRECATED_START axis_val = ngraph::normalize_axis(this, axis_val, static_cast(data->get_shape().size())); + OPENVINO_SUPPRESS_DEPRECATED_END } std::vector indices_casted_vector; @@ -126,5 +128,7 @@ bool op::v3::ScatterUpdate::has_evaluate() const { bool op::v3::ScatterUpdate::evaluate_label(TensorLabelVector& output_labels) const { OV_OP_SCOPE(v3_ScatterUpdate_evaluate_label); + OPENVINO_SUPPRESS_DEPRECATED_START return ov::default_label_evaluator(this, {0, 2}, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/select.cpp b/src/core/src/op/select.cpp index da9bb36d17b1e8..c7df95bbd138ec 100644 --- a/src/core/src/op/select.cpp +++ b/src/core/src/op/select.cpp @@ -39,7 +39,9 @@ void op::v1::Select::validate_and_infer_types() { element::Type::merge(result_et, get_input_element_type(1), get_input_element_type(2)), "Argument 1 and 2 element types must match."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1); shape_infer(this, input_shapes, output_shapes); @@ -115,8 +117,10 @@ bool evaluate_select(const HostTensorVector& output_values, bool op::v1::Select::evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const { OV_OP_SCOPE(v1_Select_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(input_values, 3)); NGRAPH_CHECK(validate_host_tensor_vector(output_values, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto autob = get_auto_broadcast(); return detail::evaluate_select(output_values, input_values, autob, output_values[0]->get_element_type()); } diff --git a/src/core/src/op/shape_of.cpp b/src/core/src/op/shape_of.cpp index 5d499a56b250b3..d8b796acd1f655 100644 --- a/src/core/src/op/shape_of.cpp +++ b/src/core/src/op/shape_of.cpp @@ -173,8 +173,10 @@ bool evaluate_label(const Node* shape_of_node, TensorLabelVector& output_labels) bool op::v3::ShapeOf::evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const { OV_OP_SCOPE(v3_ShapeOf_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(input_values, 1)); NGRAPH_CHECK(validate_host_tensor_vector(output_values, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return shape_of::evaluate_shape_of(output_values[0], input_values[0]); } @@ -251,8 +253,10 @@ shared_ptr op::v0::ShapeOf::clone_with_new_inputs(const OutputVector& new_ bool op::v0::ShapeOf::evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const { OV_OP_SCOPE(v0_ShapeOf_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(input_values, 1)); NGRAPH_CHECK(validate_host_tensor_vector(output_values, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return shape_of::evaluate_shape_of(output_values[0], input_values[0]); } diff --git a/src/core/src/op/shuffle_channels.cpp b/src/core/src/op/shuffle_channels.cpp index 51b057c9f7c3b3..d6c7afe64971f8 100644 --- a/src/core/src/op/shuffle_channels.cpp +++ b/src/core/src/op/shuffle_channels.cpp @@ -37,7 +37,9 @@ bool ngraph::op::v0::ShuffleChannels::visit_attributes(AttributeVisitor& visitor size_t op::ShuffleChannels::get_zero_based_axis() const { const auto input_rank = get_input_partial_shape(0).rank(); if (input_rank.is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START return ov::normalize_axis(this, m_axis, input_rank); + OPENVINO_SUPPRESS_DEPRECATED_END } else { throw ngraph_error("Cannot request zero-based axis with a input of unknown rank"); } @@ -46,7 +48,9 @@ size_t op::ShuffleChannels::get_zero_based_axis() const { void op::ShuffleChannels::validate_and_infer_types() { OV_OP_SCOPE(v0_ShuffleChannels_validate_and_infer_types); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); } diff --git a/src/core/src/op/sigmoid.cpp b/src/core/src/op/sigmoid.cpp index f90d52c7306b74..12f7e9cb75b4e7 100644 --- a/src/core/src/op/sigmoid.cpp +++ b/src/core/src/op/sigmoid.cpp @@ -57,7 +57,9 @@ bool evaluate_sigmoid(const HostTensorPtr& arg0, const HostTensorPtr& out) { bool ov::op::v0::Sigmoid::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Sigmoid_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return sigmoid::evaluate_sigmoid(inputs[0], outputs[0]); } diff --git a/src/core/src/op/sign.cpp b/src/core/src/op/sign.cpp index b46225dcdf0f92..c202c1f2acd2c5 100644 --- a/src/core/src/op/sign.cpp +++ b/src/core/src/op/sign.cpp @@ -58,7 +58,9 @@ bool evaluate_sign(const HostTensorPtr& arg0, const HostTensorPtr& out, const si bool op::Sign::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Sign_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return signop::evaluate_sign(inputs[0], outputs[0], shape_size(get_output_shape(0))); } diff --git a/src/core/src/op/sinh.cpp b/src/core/src/op/sinh.cpp index d945387df00ddb..3594be082b0552 100644 --- a/src/core/src/op/sinh.cpp +++ b/src/core/src/op/sinh.cpp @@ -58,7 +58,9 @@ bool evaluate_sinh(const HostTensorPtr& arg0, const HostTensorPtr& out, const si bool op::Sinh::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Sinh_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return sinhop::evaluate_sinh(inputs[0], outputs[0], shape_size(get_output_shape(0))); } diff --git a/src/core/src/op/slice.cpp b/src/core/src/op/slice.cpp index 0c2a489693e34d..4d313913ff4ddb 100644 --- a/src/core/src/op/slice.cpp +++ b/src/core/src/op/slice.cpp @@ -87,7 +87,9 @@ void op::v8::Slice::validate_and_infer_types() { set_input_is_relevant_to_shape(i); } + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes = {ov::PartialShape::dynamic()}; shape_infer(this, input_shapes, output_shapes); @@ -216,5 +218,7 @@ bool op::v8::Slice::evaluate_upper(ov::TensorVector& output_values) const { bool op::v8::Slice::evaluate_label(TensorLabelVector& output_labels) const { if (!slice_input_check(this)) return false; + OPENVINO_SUPPRESS_DEPRECATED_START return default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/softmax.cpp b/src/core/src/op/softmax.cpp index ef02e146e623c1..ab189fc9184324 100644 --- a/src/core/src/op/softmax.cpp +++ b/src/core/src/op/softmax.cpp @@ -74,7 +74,9 @@ shared_ptr op::v1::Softmax::clone_with_new_inputs(const OutputVector& new_ bool op::v1::Softmax::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_Softmax_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END outputs[0]->set_unary(inputs[0]); return evaluate_softmax(inputs[0], outputs[0], AxisSet{m_axis}); } @@ -129,7 +131,9 @@ shared_ptr op::v8::Softmax::clone_with_new_inputs(const OutputVector& new_ bool op::v8::Softmax::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v8_Softmax_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END outputs[0]->set_unary(inputs[0]); auto rank = static_cast(inputs[0]->get_shape().size()); NGRAPH_CHECK(-rank <= m_axis && m_axis < rank, @@ -138,7 +142,9 @@ bool op::v8::Softmax::evaluate(const HostTensorVector& outputs, const HostTensor ") is out of bounds (argument shape: ", inputs[0]->get_shape(), ")."); + OPENVINO_SUPPRESS_DEPRECATED_START size_t axis = static_cast(ov::normalize_axis(this->description(), m_axis, rank)); + OPENVINO_SUPPRESS_DEPRECATED_END return evaluate_softmax(inputs[0], outputs[0], AxisSet{axis}); } diff --git a/src/core/src/op/softplus.cpp b/src/core/src/op/softplus.cpp index e0419291dc70ab..e0f8cf9a5a33ae 100644 --- a/src/core/src/op/softplus.cpp +++ b/src/core/src/op/softplus.cpp @@ -71,7 +71,9 @@ bool evaluate_softplus(const HostTensorPtr& arg, const HostTensorPtr& out) { bool op::v4::SoftPlus::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v4_SoftPlus_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return softplus::evaluate_softplus(inputs[0], outputs[0]); } diff --git a/src/core/src/op/space_to_batch.cpp b/src/core/src/op/space_to_batch.cpp index 92c9d95f20a3ae..57721c5e120472 100644 --- a/src/core/src/op/space_to_batch.cpp +++ b/src/core/src/op/space_to_batch.cpp @@ -58,7 +58,9 @@ void op::v1::SpaceToBatch::validate_and_infer_types() { "pads_end must be an integral number but got (", pads_end_type, ")."); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, data_type, output_shape); } diff --git a/src/core/src/op/space_to_depth.cpp b/src/core/src/op/space_to_depth.cpp index 3faa4074f8ba27..cb10742d9df2df 100644 --- a/src/core/src/op/space_to_depth.cpp +++ b/src/core/src/op/space_to_depth.cpp @@ -46,7 +46,9 @@ std::shared_ptr ov::op::v0::SpaceToDepth::clone_with_new_inputs(const Outp void ngraph::op::v0::SpaceToDepth::validate_and_infer_types() { OV_OP_SCOPE(v0_SpaceToDepth_validate_and_infer_types); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shape = shape_infer(this, get_node_input_partial_shapes(*this)).front(); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shape); } diff --git a/src/core/src/op/split.cpp b/src/core/src/op/split.cpp index 9e437748d3f40f..e13ac15a098c95 100644 --- a/src/core/src/op/split.cpp +++ b/src/core/src/op/split.cpp @@ -46,7 +46,9 @@ void op::v1::Split::validate_and_infer_types() { "Attribute 'num_splits' must be greater than zero. Got: ", m_num_splits); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes; shape_infer(this, input_shapes, output_shapes); @@ -65,7 +67,9 @@ shared_ptr op::v1::Split::clone_with_new_inputs(const OutputVector& new_ar bool op::v1::Split::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_Split_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_ASSERT(validate_host_tensor_vector(outputs, m_num_splits) && validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END if (has_evaluate()) { const auto& data_tensor = inputs[0]; @@ -85,7 +89,9 @@ bool op::v1::Split::evaluate(const HostTensorVector& outputs, const HostTensorVe } auto axis = host_tensor_2_vector(axis_tensor)[0]; + OPENVINO_SUPPRESS_DEPRECATED_START axis = normalize_axis(this, axis, data_tensor->get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END ngraph::runtime::reference::split(data_tensor->get_data_ptr(), data_tensor->get_shape(), @@ -118,5 +124,7 @@ bool op::v1::Split::evaluate_upper(ov::TensorVector& output_values) const { bool op::v1::Split::evaluate_label(TensorLabelVector& output_labels) const { OPENVINO_ASSERT(output_labels.size() == get_num_splits()); + OPENVINO_SUPPRESS_DEPRECATED_START return input(1).get_tensor().has_and_set_bound() && default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/squeeze.cpp b/src/core/src/op/squeeze.cpp index 179966599f98f6..78e37b140f148d 100644 --- a/src/core/src/op/squeeze.cpp +++ b/src/core/src/op/squeeze.cpp @@ -31,7 +31,9 @@ op::Squeeze::Squeeze(const Output& data) : Op({data}) { void op::Squeeze::validate_and_infer_types() { OV_OP_SCOPE(v0_Squeeze_validate_and_infer_types); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1); shape_infer(this, input_shapes, output_shapes); @@ -57,8 +59,10 @@ shared_ptr op::Squeeze::clone_with_new_inputs(const OutputVector& new_args bool op::v0::Squeeze::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Squeeze_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, inputs.size())); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END if (has_evaluate()) { auto output_shapes = std::vector{outputs[0]->get_partial_shape()}; @@ -126,7 +130,9 @@ bool op::v0::Squeeze::evaluate_upper(ov::TensorVector& output_values) const { bool op::v0::Squeeze::evaluate_label(TensorLabelVector& output_labels) const { if (get_input_size() > 1 && !get_input_tensor(1).has_and_set_bound()) return false; + OPENVINO_SUPPRESS_DEPRECATED_START return default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } bool op::v0::Squeeze::constant_fold(OutputVector& output_values, const OutputVector& inputs_values) { diff --git a/src/core/src/op/strided_slice.cpp b/src/core/src/op/strided_slice.cpp index b5e26295d4e8e5..1ba48d885df4a3 100644 --- a/src/core/src/op/strided_slice.cpp +++ b/src/core/src/op/strided_slice.cpp @@ -137,7 +137,9 @@ void op::v1::StridedSlice::validate_and_infer_types() { set_input_is_relevant_to_shape(2); set_input_is_relevant_to_shape(3); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1, PartialShape::dynamic()); shape_infer(this, input_shapes, output_shapes); @@ -214,8 +216,10 @@ bool evaluate_strided_slice(const HostTensorPtr& in, bool op::v1::StridedSlice::evaluate(const HostTensorVector& output_values, const HostTensorVector& input_values) const { OV_OP_SCOPE(v1_StridedSlice_evaluate); // FIXME: 4th input is optional, but it is required by the following code + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(input_values, 4)); NGRAPH_CHECK(validate_host_tensor_vector(output_values, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return strided_slice::evaluate_strided_slice(input_values[0], input_values[1], input_values[2], @@ -253,5 +257,7 @@ bool op::v1::StridedSlice::evaluate_upper(ov::TensorVector& output_values) const bool op::v1::StridedSlice::evaluate_label(TensorLabelVector& output_labels) const { if (!strided_slice_input_check(this)) return false; + OPENVINO_SUPPRESS_DEPRECATED_START return default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/swish.cpp b/src/core/src/op/swish.cpp index f21faeb2182a2e..d41fcb64368060 100644 --- a/src/core/src/op/swish.cpp +++ b/src/core/src/op/swish.cpp @@ -112,8 +112,10 @@ bool evaluate_swish(const HostTensorVector& inputs, const HostTensorPtr& out) { bool op::v4::Swish::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v4_Swish_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && (validate_host_tensor_vector(inputs, 2) || validate_host_tensor_vector(inputs, 1))); + OPENVINO_SUPPRESS_DEPRECATED_END return swish::evaluate_swish(inputs, outputs[0]); } diff --git a/src/core/src/op/tile.cpp b/src/core/src/op/tile.cpp index e7c15a36d6fd85..203998b483fe53 100644 --- a/src/core/src/op/tile.cpp +++ b/src/core/src/op/tile.cpp @@ -35,7 +35,9 @@ void op::v0::Tile::validate_and_infer_types() { "Tile repeats must have any integer element type, but has ", repeats_et); + OPENVINO_SUPPRESS_DEPRECATED_START auto output_shapes = shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shapes[0]); set_input_is_relevant_to_shape(0); @@ -126,5 +128,7 @@ bool op::v0::Tile::evaluate_label(TensorLabelVector& output_labels) const { OV_OP_SCOPE(v0_Tile_evaluate_label); OPENVINO_ASSERT(output_labels.size() == 1); + OPENVINO_SUPPRESS_DEPRECATED_START return get_input_tensor(1).has_and_set_bound() && default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/topk.cpp b/src/core/src/op/topk.cpp index 7a856a2c590cb2..9bc85add54b3ac 100644 --- a/src/core/src/op/topk.cpp +++ b/src/core/src/op/topk.cpp @@ -107,7 +107,9 @@ bool TopK_evaluate(const ov::op::util::TopKBase* const node, const HostTensorVector& outputs, const HostTensorVector& inputs) { const auto& arg_shape = inputs[0]->get_shape(); + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = normalize_axis(node, node->get_provided_axis(), arg_shape.size()); + OPENVINO_SUPPRESS_DEPRECATED_END const auto compute_max = node->get_mode() == ov::op::TopKMode::MAX; const auto sort_type = node->get_sort_type(); diff --git a/src/core/src/op/transpose.cpp b/src/core/src/op/transpose.cpp index c0cdd5f0d47a9b..a161552b0de343 100644 --- a/src/core/src/op/transpose.cpp +++ b/src/core/src/op/transpose.cpp @@ -93,5 +93,7 @@ bool op::v1::Transpose::evaluate_upper(ov::TensorVector& output_values) const { } bool op::v1::Transpose::evaluate_label(TensorLabelVector& output_labels) const { + OPENVINO_SUPPRESS_DEPRECATED_START return get_input_tensor(ORDER).has_and_set_bound() && default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/unique.cpp b/src/core/src/op/unique.cpp index 5d57b1d4b2893d..0ddff3c854c399 100644 --- a/src/core/src/op/unique.cpp +++ b/src/core/src/op/unique.cpp @@ -179,7 +179,9 @@ void op::v10::Unique::validate_and_infer_types() { extract_axis(std::dynamic_pointer_cast(input_value(1).get_node_shared_ptr())); if (input_shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_axis = ngraph::normalize_axis(this, axis, input_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END const auto dim_at_axis = input_shape[normalized_axis]; Dimension output_dim_at_axis; diff --git a/src/core/src/op/unsqueeze.cpp b/src/core/src/op/unsqueeze.cpp index c2c00a17b32ac1..4bd98e4cc01870 100644 --- a/src/core/src/op/unsqueeze.cpp +++ b/src/core/src/op/unsqueeze.cpp @@ -23,7 +23,9 @@ op::v0::Unsqueeze::Unsqueeze(const Output& data, const Output& axes) void op::v0::Unsqueeze::validate_and_infer_types() { OV_OP_SCOPE(v0_Unsqueeze_validate_and_infer_types); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END auto output_shapes = std::vector(1); shape_infer(this, input_shapes, output_shapes); @@ -70,7 +72,9 @@ bool evaluate_unsqueeze(const Node* node, // Get axes and normalize auto axes = read_index_vector(arg1); + OPENVINO_SUPPRESS_DEPRECATED_START normalize_axes(node, out_rank, axes); + OPENVINO_SUPPRESS_DEPRECATED_END // Sort in increasing order std::set axes_set(axes.begin(), axes.end()); @@ -103,8 +107,10 @@ bool evaluate_unsqueeze(const Node* node, bool op::v0::Unsqueeze::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v0_Unsqueeze_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(inputs, 2)); NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1)); + OPENVINO_SUPPRESS_DEPRECATED_END return unsqueeze::evaluate_unsqueeze(this, inputs[0], inputs[1], outputs[0]); } @@ -137,7 +143,9 @@ bool op::v0::Unsqueeze::evaluate_upper(ov::TensorVector& output_values) const { bool op::v0::Unsqueeze::evaluate_label(TensorLabelVector& output_labels) const { if (!get_input_tensor(1).has_and_set_bound()) return false; + OPENVINO_SUPPRESS_DEPRECATED_START return default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } bool op::v0::Unsqueeze::constant_fold(OutputVector& output_values, const OutputVector& inputs_values) { diff --git a/src/core/src/op/util/broadcast_base.cpp b/src/core/src/op/util/broadcast_base.cpp index b0a4394c0680a7..8463562d974d5a 100644 --- a/src/core/src/op/util/broadcast_base.cpp +++ b/src/core/src/op/util/broadcast_base.cpp @@ -194,7 +194,9 @@ void ov::op::util::BroadcastBase::validate_and_infer_types() { } PartialShape output_shape; + OPENVINO_SUPPRESS_DEPRECATED_START bool output_shape_defined = ngraph::evaluate_as_partial_shape(get_input_source_output(1), output_shape); + OPENVINO_SUPPRESS_DEPRECATED_END if (auto concat = ov::as_type_ptr(input_value(1).get_node_shared_ptr())) { auto concat_inputs = concat->inputs(); @@ -235,7 +237,9 @@ void ov::op::util::BroadcastBase::validate_and_infer_types() { input_rank); if (output_shape_defined && has_and_set_equal_bounds(input_value(2))) { + OPENVINO_SUPPRESS_DEPRECATED_START auto axes_mapping_val = get_constant_from_source(input_value(2))->get_axis_vector_val(); + OPENVINO_SUPPRESS_DEPRECATED_END validate_target_shape_none(arg_shape, axes_mapping_val, output_shape); } } @@ -292,7 +296,9 @@ std::pair ov::op::util::BroadcastBase::get_broadcast_axes() c bool axes_known = false; if (m_mode.m_type == BroadcastType::NONE) { + OPENVINO_SUPPRESS_DEPRECATED_START const auto axes_mapping_constant = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END if (get_input_partial_shape(1).is_static() && axes_mapping_constant) { auto axes_mapping_val = axes_mapping_constant->get_axis_vector_val(); auto target_shape = get_input_shape(1); @@ -443,9 +449,10 @@ ov::Shape ov::op::util::BroadcastBase::get_target_shape(const HostTensorPtr& inp bool ov::op::util::BroadcastBase::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(util_BroadcastBase_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(ngraph::validate_host_tensor_vector(inputs, 2) || ngraph::validate_host_tensor_vector(inputs, 3)); NGRAPH_CHECK(ngraph::validate_host_tensor_vector(outputs, 1)); - + OPENVINO_SUPPRESS_DEPRECATED_END Shape target_shape = get_target_shape(inputs[1]); PartialShape result_shape; diff --git a/src/core/src/op/util/deformable_convolution_base.cpp b/src/core/src/op/util/deformable_convolution_base.cpp index 8e8515c8ccdf48..32be72c2efba9d 100644 --- a/src/core/src/op/util/deformable_convolution_base.cpp +++ b/src/core/src/op/util/deformable_convolution_base.cpp @@ -2,14 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/util/deformable_convolution_base.hpp" +#include "openvino/op/util/deformable_convolution_base.hpp" #include "itt.hpp" -#include "ngraph/axis_vector.hpp" -#include "ngraph/coordinate_diff.hpp" -#include "ngraph/op/reshape.hpp" -#include "ngraph/util.hpp" -#include "ngraph/validation_util.hpp" using namespace std; @@ -21,12 +16,7 @@ ov::op::util::DeformableConvolutionBase::DeformableConvolutionBase(const OutputV const PadType& auto_pad, const int64_t group, const int64_t deformable_group) - : Op(arguments), - m_strides(strides), - m_dilations(dilations), - m_pads_begin(pads_begin), - m_pads_end(pads_end), - m_auto_pad(auto_pad), + : ConvolutionBase(arguments, strides, pads_begin, pads_end, dilations, auto_pad), m_group(group), m_deformable_group(deformable_group) {} @@ -41,154 +31,3 @@ bool ov::op::util::DeformableConvolutionBase::visit_attributes(AttributeVisitor& visitor.on_attribute("deformable_group", m_deformable_group); return true; } - -void ov::op::util::DeformableConvolutionBase::validate_and_infer_types() { - OV_OP_SCOPE(util_DeformableConvolutionBase_validate_and_infer_types); - const PartialShape& data_batch_pshape = get_input_partial_shape(0); - const PartialShape& offsets_pshape = get_input_partial_shape(1); - const PartialShape& filters_pshape = get_input_partial_shape(2); - - element::Type data_batch_et = get_input_element_type(0); - element::Type offsets_et = get_input_element_type(1); - element::Type filters_et = get_input_element_type(2); - - element::Type result_et; - NODE_VALIDATION_CHECK(this, - element::Type::merge(result_et, data_batch_et, offsets_et) && - element::Type::merge(result_et, result_et, filters_et), - "Element types of inputs do not match. Got: data batch (", - data_batch_et, - "), offsets (", - offsets_et, - ") and filters (", - filters_et, - ")"); - - NODE_VALIDATION_CHECK(this, - result_et.is_real() || result_et.is_integral_number(), - "Element type of inputs must be numeric. Got: ", - result_et); - - Rank result_ps_rank{}; - NODE_VALIDATION_CHECK(this, - Rank::merge(result_ps_rank, data_batch_pshape.rank(), offsets_pshape.rank()) && - Rank::merge(result_ps_rank, result_ps_rank, filters_pshape.rank()), - "Ranks of inputs do not match. Got: data batch shape ", - data_batch_pshape, - ", offsets shape ", - offsets_pshape, - ", filters shape ", - filters_pshape); - - NODE_VALIDATION_CHECK(this, result_ps_rank.compatible(4), "Inputs must be of rank 4. Got: ", result_ps_rank); - - NODE_VALIDATION_CHECK(this, m_group > 0, "Attribute 'group' must be any value starting from 1. Got: ", m_group); - - NODE_VALIDATION_CHECK(this, - m_deformable_group > 0, - "Attribute 'deformable group' must be any value starting from 1. Got: ", - m_deformable_group); - - if (offsets_pshape.rank().is_static()) { - if (offsets_pshape[1].is_static()) { - if (filters_pshape.rank().is_static() && filters_pshape[2].is_static() && filters_pshape[3].is_static()) { - auto offsets_channels = - m_deformable_group * filters_pshape[2].get_length() * filters_pshape[3].get_length() * 2; - NODE_VALIDATION_CHECK(this, - offsets_pshape[1].get_length() == offsets_channels, - "The channels dimension of offsets input is not " - "compatible with filters and 'deformable group' attribute. " - "Offsets input shape: ", - offsets_pshape, - ", deformable 'group' attribute value: ", - m_deformable_group, - ", filters shape: ", - filters_pshape); - } else { - // At least we can check if offsets channels is evenly divisible by deformable - // group attribute - NODE_VALIDATION_CHECK(this, - offsets_pshape[1].get_length() % m_deformable_group == 0, - "The channels dimension of offsets input must be " - "evenly divisible by the 'deformable group' value along the " - "channels axis. Offsets input shape: ", - offsets_pshape, - ", 'deformable group' attribute value: ", - m_deformable_group); - } - } - - if (data_batch_pshape.rank().is_static()) { - NODE_VALIDATION_CHECK(this, - offsets_pshape[0].compatible(data_batch_pshape[0]), - "Data batch and offsets batch dimension must be same value. Got: ", - offsets_pshape[0], - " and ", - data_batch_pshape[0]); - } - } - - if (data_batch_pshape.rank().is_static() && data_batch_pshape[1].is_static()) { - NODE_VALIDATION_CHECK(this, - data_batch_pshape[1].get_length() % m_group == 0, - "The input data shape must be evenly divisible by the 'group' value " - "along the channels axis. Current input shape: ", - data_batch_pshape, - ", 'group' attribute value: ", - m_group); - } - - if (filters_pshape.rank().is_static() && filters_pshape[0].is_static()) { - NODE_VALIDATION_CHECK(this, - filters_pshape[0].get_length() % m_group == 0, - "The filters shape must be evenly divisible by the 'group' value along " - "the channels axis. Current filters shape: ", - filters_pshape, - ", 'group' attribute value: ", - m_group); - } - - // adjust filter shape to reuse regular infer_convolution_forward() - const auto new_filters_pshape = [&](int64_t groups) { - auto new_shape(filters_pshape); - if (new_shape.rank().is_static()) { - new_shape[1] *= groups; - } - return new_shape; - }(m_group); - PartialShape result_shape = ngraph::validate_and_infer_convolution_forward_output_shape(this, - result_ps_rank, - data_batch_pshape, - new_filters_pshape, - m_auto_pad, - m_strides, - m_dilations, - m_pads_begin, - m_pads_end); - - if (result_shape.rank().is_static() && offsets_pshape.rank().is_static()) { - PartialShape result_spatial_shape = [&result_shape]() { - vector result_spatial_dims{result_shape}; - result_spatial_dims.erase(result_spatial_dims.begin(), result_spatial_dims.begin() + 2); - return PartialShape{result_spatial_dims}; - }(); - - PartialShape offsets_spatial_shape = [&offsets_pshape]() { - vector offsets_spatial_dims{offsets_pshape}; - offsets_spatial_dims.erase(offsets_spatial_dims.begin(), offsets_spatial_dims.begin() + 2); - return PartialShape{offsets_spatial_dims}; - }(); - - NODE_VALIDATION_CHECK(this, - offsets_spatial_shape.compatible(result_spatial_shape), - "Spatial dimensions of offsets and output must be equal. Got: ", - offsets_spatial_shape, - " and ", - result_spatial_shape); - - if (result_shape[0].is_dynamic()) { - result_shape[0] = offsets_pshape[0]; // batch size - } - } - set_output_type(0, result_et, result_shape); -} diff --git a/src/core/src/op/util/embeddingbag_offsets_base.cpp b/src/core/src/op/util/embeddingbag_offsets_base.cpp index 2276df55b90374..6a58f2cb5b205b 100644 --- a/src/core/src/op/util/embeddingbag_offsets_base.cpp +++ b/src/core/src/op/util/embeddingbag_offsets_base.cpp @@ -80,7 +80,9 @@ void ov::op::util::EmbeddingBagOffsetsBase::validate_and_infer_types() { } const auto& result_et = get_input_element_type(EMB_TABLE); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, result_et, shape_infer(this, input_shapes)[0]); } diff --git a/src/core/src/op/util/embeddingbag_packed_base.cpp b/src/core/src/op/util/embeddingbag_packed_base.cpp index b3cdaab7e16d08..7fce2be82e4ceb 100644 --- a/src/core/src/op/util/embeddingbag_packed_base.cpp +++ b/src/core/src/op/util/embeddingbag_packed_base.cpp @@ -41,7 +41,9 @@ void ov::op::util::EmbeddingBagPackedBase::validate_and_infer_types() { } const auto& emb_et = get_input_element_type(EMB_TABLE); + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, emb_et, shape_infer(this, input_shapes)[0]); } diff --git a/src/core/src/op/util/evaluate_helpers.cpp b/src/core/src/op/util/evaluate_helpers.cpp index 06481a1aa1d3aa..80e62644e05eff 100644 --- a/src/core/src/op/util/evaluate_helpers.cpp +++ b/src/core/src/op/util/evaluate_helpers.cpp @@ -9,7 +9,9 @@ AxisSet get_normalized_axes_from_tensor(const HostTensorPtr tensor, const ngraph::Rank& rank, const std::string& node_description) { const auto axes_vector = host_tensor_2_vector(tensor); + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_axes = ngraph::normalize_axes(node_description, axes_vector, rank); + OPENVINO_SUPPRESS_DEPRECATED_END return AxisSet{normalized_axes}; } } // namespace ngraph diff --git a/src/core/src/op/util/gather_base.cpp b/src/core/src/op/util/gather_base.cpp index bb57a196407da4..f61bffd1e345e4 100644 --- a/src/core/src/op/util/gather_base.cpp +++ b/src/core/src/op/util/gather_base.cpp @@ -39,7 +39,9 @@ void ov::op::util::GatherBase::validate_and_infer_types() { } int64_t ov::op::util::GatherBase::get_axis() const { + OPENVINO_SUPPRESS_DEPRECATED_START const auto& const_op = get_constant_from_source(input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_ASSERT(const_op, "axis value is not set"); int64_t axis = const_op->cast_vector()[0]; @@ -192,9 +194,10 @@ bool cf_gather_with_subgraph(ov::OutputVector& output_values, bool ov::op::util::GatherBase::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(util_GatherBase_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(ngraph::validate_host_tensor_vector(inputs, 3)); NGRAPH_CHECK(ngraph::validate_host_tensor_vector(outputs, 1)); - + OPENVINO_SUPPRESS_DEPRECATED_END int64_t axis = 0; switch (inputs[2]->get_element_type()) { case element::Type_t::i32: @@ -255,7 +258,9 @@ bool ov::op::util::GatherBase::evaluate_upper(ov::TensorVector& output_values) c bool ov::op::util::GatherBase::evaluate_label(TensorLabelVector& output_labels) const { if (!get_input_tensor(1).has_and_set_bound() || !get_input_tensor(2).has_and_set_bound()) return false; + OPENVINO_SUPPRESS_DEPRECATED_START return default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } bool ov::op::util::GatherBase::constant_fold(OutputVector& output_values, const OutputVector& input_values) { diff --git a/src/core/src/op/util/reduction_base.cpp b/src/core/src/op/util/reduction_base.cpp index fde2f105dd5584..72e3a4c2e8a36c 100644 --- a/src/core/src/op/util/reduction_base.cpp +++ b/src/core/src/op/util/reduction_base.cpp @@ -26,10 +26,14 @@ bool ov::op::util::ReductionBase::reduction_axes_constant() const { const ov::AxisSet ov::op::util::ReductionBase::get_reduction_axes() const { AxisSet axes; + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& const_op = get_constant_from_source(input_value(1))) { + OPENVINO_SUPPRESS_DEPRECATED_END const auto const_data = const_op->cast_vector(); const auto input_data_rank = get_input_partial_shape(0).rank(); + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_axes = ov::normalize_axes(get_friendly_name(), const_data, input_data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END axes = AxisSet{normalized_axes}; } return axes; diff --git a/src/core/src/op/util/scatter_base.cpp b/src/core/src/op/util/scatter_base.cpp index 9f833eb8014b35..643ce1a99d54cc 100644 --- a/src/core/src/op/util/scatter_base.cpp +++ b/src/core/src/op/util/scatter_base.cpp @@ -74,11 +74,15 @@ void ov::op::util::ScatterBase::validate_and_infer_types() { return; // Get axis value if possible. + OPENVINO_SUPPRESS_DEPRECATED_START if (const auto& axis_const_input = get_constant_from_source(input_value(AXIS))) { + OPENVINO_SUPPRESS_DEPRECATED_END bool compatible = true; int64_t axis = axis_const_input->cast_vector().at(0); int64_t data_rank = data_shape.rank().get_length(); + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(this, axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END if (indices_shape.rank().is_static() && updates_shape.rank().is_static()) { int64_t indices_rank = indices_shape.rank().get_length(); diff --git a/src/core/src/op/util/topk_base.cpp b/src/core/src/op/util/topk_base.cpp index 3bc07aa0b7ba5b..7461ef977e8d19 100644 --- a/src/core/src/op/util/topk_base.cpp +++ b/src/core/src/op/util/topk_base.cpp @@ -45,7 +45,9 @@ void ov::op::util::TopKBase::validate_and_infer_types() { set_axis(get_input_partial_shape(0).rank(), get_provided_axis()); + OPENVINO_SUPPRESS_DEPRECATED_START const auto output_shapes = op::util::shape_infer(this, get_node_input_partial_shapes(*this)); + OPENVINO_SUPPRESS_DEPRECATED_END set_output_type(0, get_input_element_type(0), output_shapes[0]); set_output_type(1, m_index_element_type, output_shapes[1]); @@ -149,7 +151,9 @@ void ov::op::util::TopKBase::set_axis(const int64_t axis) { } void ov::op::util::TopKBase::set_axis(const Rank& input_rank, const int64_t axis) { + OPENVINO_SUPPRESS_DEPRECATED_START m_normalized_axis = input_rank.is_static() ? normalize_axis(this, axis, input_rank) : UNKNOWN_NORMALIZED_AXIS; + OPENVINO_SUPPRESS_DEPRECATED_END m_axis = axis; } diff --git a/src/core/src/op/variadic_split.cpp b/src/core/src/op/variadic_split.cpp index 4c1f7ad1427601..91c9069e6ec5f8 100644 --- a/src/core/src/op/variadic_split.cpp +++ b/src/core/src/op/variadic_split.cpp @@ -34,7 +34,9 @@ void ngraph::op::v1::VariadicSplit::validate_and_infer_types() { set_input_is_relevant_to_value(i); } + OPENVINO_SUPPRESS_DEPRECATED_START const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector output_shapes; shape_infer(this, input_shapes, output_shapes); @@ -85,7 +87,9 @@ bool op::v1::VariadicSplit::evaluate_variadic_split(const HostTensorVector& inpu "split_lengths element type is not integral data type"); int64_t axis = host_tensor_2_vector(axis_tensor)[0]; + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(this, axis, data_tensor->get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END std::vector input_shapes = {data_tensor->get_partial_shape(), axis_tensor->get_partial_shape(), @@ -141,5 +145,7 @@ bool op::v1::VariadicSplit::evaluate_upper(ov::TensorVector& output_values) cons } bool op::v1::VariadicSplit::evaluate_label(TensorLabelVector& output_labels) const { + OPENVINO_SUPPRESS_DEPRECATED_START return has_axis_and_splits_bound_set() && default_label_evaluator(this, output_labels); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/src/op/xor.cpp b/src/core/src/op/xor.cpp index 39c0926b98f1a6..d43277cec3b936 100644 --- a/src/core/src/op/xor.cpp +++ b/src/core/src/op/xor.cpp @@ -60,7 +60,9 @@ bool evaluate_logxor(const HostTensorPtr& arg0, bool op::v1::LogicalXor::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { OV_OP_SCOPE(v1_LogicalXor_evaluate); + OPENVINO_SUPPRESS_DEPRECATED_START NGRAPH_CHECK(validate_host_tensor_vector(outputs, 1) && validate_host_tensor_vector(inputs, 2)); + OPENVINO_SUPPRESS_DEPRECATED_END return logxor::evaluate_logxor(inputs[0], inputs[1], outputs[0], get_autob()); } diff --git a/src/core/src/pass/low_latency.cpp b/src/core/src/pass/low_latency.cpp index 455adc024cd382..5364b08fd8a88e 100644 --- a/src/core/src/pass/low_latency.cpp +++ b/src/core/src/pass/low_latency.cpp @@ -18,7 +18,7 @@ #include NGRAPH_SUPPRESS_DEPRECATED_START -NGRAPH_RTTI_DEFINITION(ngraph::pass::LowLatency, "LowLatency", 0); +NGRAPH_RTTI_DEFINITION(ngraph::pass::LowLatency, "LowLatency"); using namespace std; diff --git a/src/core/src/pass/pass.cpp b/src/core/src/pass/pass.cpp index 01f24f05fba1d3..6b246959ac3f13 100644 --- a/src/core/src/pass/pass.cpp +++ b/src/core/src/pass/pass.cpp @@ -57,6 +57,6 @@ ov::pass::ModelPass::~ModelPass() = default; OPENVINO_SUPPRESS_DEPRECATED_START -NGRAPH_RTTI_DEFINITION(ngraph::pass::NodePass, "ngraph::pass::NodePass", 0); +NGRAPH_RTTI_DEFINITION(ngraph::pass::NodePass, "ngraph::pass::NodePass"); ngraph::pass::NodePass::~NodePass() = default; diff --git a/src/core/src/pass/serialize.cpp b/src/core/src/pass/serialize.cpp index 81d2618e122084..dc7f634488bf5d 100644 --- a/src/core/src/pass/serialize.cpp +++ b/src/core/src/pass/serialize.cpp @@ -1002,11 +1002,9 @@ void ngfunction_2_ir(pugi::xml_node& netXml, // WA for LSTMCellv0, peephole input shall not be serialized if (e.to_port == 6) { const auto& type_info = ordered_ops[e.to_layer]->get_type_info(); - OPENVINO_SUPPRESS_DEPRECATED_START - if (!strcmp(type_info.name, "LSTMCell") && type_info.version == 0) { + if (!strcmp(type_info.name, "LSTMCell")) { continue; } - OPENVINO_SUPPRESS_DEPRECATED_END } pugi::xml_node edge = edges.append_child("edge"); edge.append_attribute("from-layer").set_value(e.from_layer); diff --git a/src/core/src/pass/visualize_tree.cpp b/src/core/src/pass/visualize_tree.cpp index 70ee298b547e5e..c89decb3f42121 100644 --- a/src/core/src/pass/visualize_tree.cpp +++ b/src/core/src/pass/visualize_tree.cpp @@ -503,7 +503,9 @@ string pass::VisualizeTree::get_node_name(shared_ptr node) { if (node->get_friendly_name() != node->get_name()) { rc += "\\n" + (nvtmn ? string("name: ") : "") + node->get_name(); } - rc += "\\n" + (nvtmn ? string("type_name: ") : "") + std::string(node->get_type_name()); + const auto type_info = node->get_type_info(); + rc += "\\n" + (nvtmn ? string("type_name: ") : "") + std::string(type_info.version_id) + + "::" + std::string(type_info.name); static const bool nvttn = getenv_bool("OV_VISUALIZE_TREE_TENSORS_NAME"); if (nvttn) { diff --git a/src/core/src/pattern/op/pattern.cpp b/src/core/src/pattern/op/pattern.cpp index 00e80ca563fe28..793d62dc9f1772 100644 --- a/src/core/src/pattern/op/pattern.cpp +++ b/src/core/src/pattern/op/pattern.cpp @@ -50,6 +50,12 @@ std::function)> consumers_count(size_t n) { }; } +std::function)> consumers_more_than(size_t n) { + return [=](Output output) -> bool { + return output.get_target_inputs().size() > n; + }; +} + std::function)> has_static_dim(size_t pos) { return [=](Output output) -> bool { const auto& shape = output.get_partial_shape(); diff --git a/src/core/src/runtime/host_tensor.cpp b/src/core/src/runtime/host_tensor.cpp index 1a0b5248c23506..90bc88e968dc89 100644 --- a/src/core/src/runtime/host_tensor.cpp +++ b/src/core/src/runtime/host_tensor.cpp @@ -143,9 +143,8 @@ void runtime::HostTensor::set_shape(const Shape& shape) { shape, " must be compatible with the partial shape: ", get_partial_shape()); - OPENVINO_SUPPRESS_DEPRECATED_START - m_descriptor->set_partial_shape(shape); - OPENVINO_SUPPRESS_DEPRECATED_END + m_descriptor->m_partial_shape = shape; + m_descriptor->m_shape_changed = true; } void runtime::HostTensor::set_unary(const HostTensorPtr& arg) { diff --git a/src/core/src/type.cpp b/src/core/src/type.cpp index 3def4c0adde5f2..be48b68f1feb40 100644 --- a/src/core/src/type.cpp +++ b/src/core/src/type.cpp @@ -18,12 +18,9 @@ size_t DiscreteTypeInfo::hash() const { if (hash_value != 0) return hash_value; size_t name_hash = name ? std::hash()(std::string(name)) : 0; - OPENVINO_SUPPRESS_DEPRECATED_START - size_t version_hash = std::hash()(version); - OPENVINO_SUPPRESS_DEPRECATED_END size_t version_id_hash = version_id ? std::hash()(std::string(version_id)) : 0; - return ov::util::hash_combine(std::vector{name_hash, version_hash, version_id_hash}); + return ov::util::hash_combine(std::vector{name_hash, version_id_hash}); } size_t DiscreteTypeInfo::hash() { @@ -40,9 +37,7 @@ std::string DiscreteTypeInfo::get_version() const { if (version_id) { return std::string(version_id); } - OPENVINO_SUPPRESS_DEPRECATED_START - return std::to_string(version); - OPENVINO_SUPPRESS_DEPRECATED_END + return nullptr; } DiscreteTypeInfo::operator std::string() const { @@ -51,10 +46,7 @@ DiscreteTypeInfo::operator std::string() const { std::ostream& operator<<(std::ostream& s, const DiscreteTypeInfo& info) { std::string version_id = info.version_id ? info.version_id : "(empty)"; - OPENVINO_SUPPRESS_DEPRECATED_START - s << "DiscreteTypeInfo{name: " << info.name << ", version_id: " << version_id << ", old_version: " << info.version - << ", parent: "; - OPENVINO_SUPPRESS_DEPRECATED_END + s << "DiscreteTypeInfo{name: " << info.name << ", version_id: " << version_id << ", parent: "; if (!info.parent) s << info.parent; else @@ -66,10 +58,7 @@ std::ostream& operator<<(std::ostream& s, const DiscreteTypeInfo& info) { // parent is commented to fix type relaxed operations bool DiscreteTypeInfo::operator<(const DiscreteTypeInfo& b) const { - OPENVINO_SUPPRESS_DEPRECATED_START - if (version < b.version) - return true; - if (version == b.version && name != nullptr && b.name != nullptr) { + if (name != nullptr && b.name != nullptr) { int cmp_status = strcmp(name, b.name); if (cmp_status < 0) return true; @@ -81,15 +70,20 @@ bool DiscreteTypeInfo::operator<(const DiscreteTypeInfo& b) const { } } - OPENVINO_SUPPRESS_DEPRECATED_END return false; } bool DiscreteTypeInfo::operator==(const DiscreteTypeInfo& b) const { if (hash_value != 0 && b.hash_value != 0) return hash() == b.hash(); - OPENVINO_SUPPRESS_DEPRECATED_START - return version == b.version && strcmp(name, b.name) == 0; - OPENVINO_SUPPRESS_DEPRECATED_END + if (name != nullptr && b.name != nullptr) { + if (strcmp(name, b.name) == 0) { + std::string v_id(version_id == nullptr ? "" : version_id); + std::string bv_id(b.version_id == nullptr ? "" : b.version_id); + if (v_id == bv_id) + return true; + } + } + return false; } bool DiscreteTypeInfo::operator<=(const DiscreteTypeInfo& b) const { return *this == b || *this < b; diff --git a/src/core/src/type/element_type.cpp b/src/core/src/type/element_type.cpp index 9dbccd1095d408..3a04c9cc1771d6 100644 --- a/src/core/src/type/element_type.cpp +++ b/src/core/src/type/element_type.cpp @@ -74,6 +74,48 @@ inline TypeInfo get_type_info(ov::element::Type_t type) { OPENVINO_THROW("ov::element::Type_t not supported: ", type); } }; + +ov::element::Type type_from_string(const std::string& type) { + if (type == "f16" || type == "FP16") { + return ::ov::element::Type(::ov::element::Type_t::f16); + } else if (type == "f32" || type == "FP32") { + return ::ov::element::Type(::ov::element::Type_t::f32); + } else if (type == "bf16" || type == "BF16") { + return ::ov::element::Type(::ov::element::Type_t::bf16); + } else if (type == "f64" || type == "FP64") { + return ::ov::element::Type(::ov::element::Type_t::f64); + } else if (type == "i4" || type == "I4") { + return ::ov::element::Type(::ov::element::Type_t::i4); + } else if (type == "i8" || type == "I8") { + return ::ov::element::Type(::ov::element::Type_t::i8); + } else if (type == "i16" || type == "I16") { + return ::ov::element::Type(::ov::element::Type_t::i16); + } else if (type == "i32" || type == "I32") { + return ::ov::element::Type(::ov::element::Type_t::i32); + } else if (type == "i64" || type == "I64") { + return ::ov::element::Type(::ov::element::Type_t::i64); + } else if (type == "u1" || type == "U1" || type == "BIN" || type == "bin") { + return ::ov::element::Type(::ov::element::Type_t::u1); + } else if (type == "u4" || type == "U4") { + return ::ov::element::Type(::ov::element::Type_t::u4); + } else if (type == "u8" || type == "U8") { + return ::ov::element::Type(::ov::element::Type_t::u8); + } else if (type == "u16" || type == "U16") { + return ::ov::element::Type(::ov::element::Type_t::u16); + } else if (type == "u32" || type == "U32") { + return ::ov::element::Type(::ov::element::Type_t::u32); + } else if (type == "u64" || type == "U64") { + return ::ov::element::Type(::ov::element::Type_t::u64); + } else if (type == "boolean" || type == "BOOL") { + return ::ov::element::Type(::ov::element::Type_t::boolean); + } else if (type == "undefined" || type == "UNSPECIFIED") { + return ::ov::element::Type(::ov::element::Type_t::undefined); + } else if (type == "dynamic") { + return ::ov::element::Type(::ov::element::Type_t::dynamic); + } else { + OPENVINO_THROW("Incorrect type: ", type); + } +} } // namespace std::vector ov::element::Type::get_known_types() { @@ -133,6 +175,8 @@ ov::element::Type::Type(size_t bitwidth, } } +ov::element::Type::Type(const std::string& type) : Type(type_from_string(type)) {} + std::string ov::element::Type::c_type_string() const { return get_type_info(m_type).m_cname; } diff --git a/src/core/src/validation_util.cpp b/src/core/src/validation_util.cpp index 34d1c616910b19..0559275e632b86 100644 --- a/src/core/src/validation_util.cpp +++ b/src/core/src/validation_util.cpp @@ -27,6 +27,7 @@ #include "ngraph/util.hpp" #include "openvino/op/ops.hpp" #include "sequnce_generator.hpp" +#include "validation_util.hpp" NGRAPH_SUPPRESS_DEPRECATED_START using namespace std; @@ -872,7 +873,7 @@ std::string normalize_axis_error_msg(const int64_t& axis, const int64_t& lower, } } // namespace -int64_t ov::normalize(const int64_t& value, const int64_t& max) { +int64_t ov::util::normalize(const int64_t& value, const int64_t& max) { return (value < 0) ? value + max : value; }; @@ -937,7 +938,7 @@ int64_t ov::normalize_axis(const std::string& node_description, OPENVINO_ASSERT((axis_range_min <= axis) && (axis <= axis_range_max), node_description, normalize_axis_error_msg(axis, axis_range_min, axis_range_max)); - return normalize(axis, tensor_rank); + return util::normalize(axis, tensor_rank); } void ngraph::opset1::infer_conv_backprop_auto_padding(const Shape& input_data_shape, @@ -1312,7 +1313,7 @@ void ov::generate_transpose_default_order(std::vector& axes_order, cons } bool ov::is_valid_axes_order(const std::vector& axes_order, const size_t size) { - return are_unique(axes_order) && + return util::are_unique(axes_order) && std::all_of(axes_order.cbegin(), axes_order.cend(), ov::cmp::Between(0, size)); } @@ -1331,16 +1332,16 @@ bool ov::is_rank_compatible_any_of(const ov::Rank& rank, const std::vector }); } -bool ov::are_unique(const std::vector& data) { +bool ov::util::are_unique(const std::vector& data) { return std::unordered_set(data.begin(), data.cend()).size() == data.size(); } // clip value to min, max -int64_t ov::clip(const int64_t& value, const int64_t& min, const int64_t& max) { +int64_t ov::util::clip(const int64_t& value, const int64_t& min, const int64_t& max) { return std::min(std::max(value, min), max); }; -std::shared_ptr ov::constantfold_subgraph(const Output& subgraph_sink) { +std::shared_ptr ov::util::constantfold_subgraph(const Output& subgraph_sink) { if (const auto& c = ov::as_type_ptr(subgraph_sink.get_node_shared_ptr())) return c; diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index e1dd3e60b7c1d1..829f1822a21488 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -2,11 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - # 'argument': conversion from 'size_t' to 'int', possible loss of data - ie_add_compiler_flags(/wd4267) - ie_add_compiler_flags(/wd4244) -endif() set(TARGET_NAME ov_core_unit_tests) @@ -16,11 +11,6 @@ ov_try_use_gold_linker() add_definitions(-DSERIALIZED_ZOO=\"${TEST_MODEL_ZOO}/core/models\") -if(NOT ENABLE_OV_CORE_UNIT_TESTS) - message(STATUS "OpenVINO Core unit tests disabled") - return() -endif() - message(STATUS "OpenVINO Core unit tests enabled") # For type relaxed types @@ -37,13 +27,6 @@ list(APPEND UNIT_TESTS_DEPENDENCIES openvino_template_extension) list(APPEND UNIT_TESTS_DEPENDENCIES template_extension) list(APPEND EXCLUDE_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/dnnl.cpp) -if (ENABLE_TEMPLATE) - list(APPEND UNIT_TESTS_DEPENDENCIES openvino_template_plugin) -else() - list(APPEND EXCLUDE_TESTS - # It should be a part of template plugin - ${CMAKE_CURRENT_SOURCE_DIR}/op_eval/memory.cpp) -endif() ov_add_test_target( NAME ${TARGET_NAME} diff --git a/src/core/tests/any.cpp b/src/core/tests/any.cpp index 6b009cd15f3a6a..f66ae9720f0511 100644 --- a/src/core/tests/any.cpp +++ b/src/core/tests/any.cpp @@ -181,8 +181,8 @@ TEST_F(AnyTests, AnyAsMapOfMapOfAnys) { ASSERT_NE(testMap.find("refMap1"), testMap.end()); auto testMap1 = testMap.at("refMap1").as>(); - ASSERT_NE(testMap1.find("testParamInt"), testMap.end()); - ASSERT_NE(testMap1.find("testParamString"), testMap.end()); + ASSERT_NE(testMap1.find("testParamInt"), testMap1.end()); + ASSERT_NE(testMap1.find("testParamString"), testMap1.end()); int testInt1 = testMap1["testParamInt"].as(); std::string testString1 = testMap1["testParamString"].as(); @@ -192,8 +192,8 @@ TEST_F(AnyTests, AnyAsMapOfMapOfAnys) { ASSERT_NE(testMap.find("refMap2"), testMap.end()); auto testMap2 = testMap.at("refMap2").as>(); - ASSERT_NE(testMap2.find("testParamInt"), testMap.end()); - ASSERT_NE(testMap2.find("testParamString"), testMap.end()); + ASSERT_NE(testMap2.find("testParamInt"), testMap2.end()); + ASSERT_NE(testMap2.find("testParamString"), testMap2.end()); int testInt2 = testMap2["testParamInt"].as(); std::string testString2 = testMap2["testParamString"].as(); diff --git a/src/core/tests/bound_evaluate.cpp b/src/core/tests/bound_evaluate.cpp index cb855ddfa76e16..664f8ebcb810f7 100644 --- a/src/core/tests/bound_evaluate.cpp +++ b/src/core/tests/bound_evaluate.cpp @@ -51,3 +51,31 @@ TEST_F(EvaluateBoundTest, no_exception_when_node_has_output_with_dynamic_element EXPECT_NO_THROW(evaluate_both_bounds(fn_op)); } + +using BoundEvaluatorTest = ::testing::Test; +TEST(BoundEvaluatorTest, no_exception_on_single_bound) { + constexpr auto et = element::i32; + const auto s = Shape{1, 1}; + const auto a = std::make_shared(et, PartialShape{s}); + const auto b = Constant::create(et, s, {1}); + const auto sub = std::make_shared(a, b); + + int32_t a_l[1] = {1}; + a->get_output_tensor(0).set_lower_value(Tensor{et, s, a_l}); + + int32_t o_[1] = {INT32_MIN}; // initial value of output tensor is not needed, it's set to check whether changed + TensorVector output{{et, s, o_}}; + // evaluations won't be performed due to missing upper bound tensor of parameter a + ASSERT_NO_THROW(sub->evaluate_lower(output)); + EXPECT_EQ(o_[0], INT32_MIN); + ASSERT_NO_THROW(sub->evaluate_upper(output)); + EXPECT_EQ(o_[0], INT32_MIN); + + int32_t a_u[1] = {11}; + a->get_output_tensor(0).set_upper_value(Tensor{et, s, a_u}); + // now both bounds of sub node can be calculated + ASSERT_NO_THROW(sub->evaluate_lower(output)); + EXPECT_EQ(o_[0], 0); + ASSERT_NO_THROW(sub->evaluate_upper(output)); + EXPECT_EQ(o_[0], 10); +} diff --git a/src/core/tests/element_type.cpp b/src/core/tests/element_type.cpp index 4c3ec1941f1022..c19d17ea301dd1 100644 --- a/src/core/tests/element_type.cpp +++ b/src/core/tests/element_type.cpp @@ -7,6 +7,7 @@ #include #include "gtest/gtest.h" +#include "openvino/core/except.hpp" using namespace ngraph; @@ -25,6 +26,52 @@ TEST(element_type, from) { EXPECT_EQ(element::from(), element::u64); } +TEST(element_type, from_string) { + EXPECT_EQ(element::Type("boolean"), element::boolean); + EXPECT_EQ(element::Type("BOOL"), element::boolean); + + EXPECT_EQ(element::Type("bf16"), element::bf16); + EXPECT_EQ(element::Type("BF16"), element::bf16); + EXPECT_EQ(element::Type("f16"), element::f16); + EXPECT_EQ(element::Type("FP16"), element::f16); + EXPECT_EQ(element::Type("f32"), element::f32); + EXPECT_EQ(element::Type("FP32"), element::f32); + EXPECT_EQ(element::Type("f64"), element::f64); + EXPECT_EQ(element::Type("FP64"), element::f64); + + EXPECT_EQ(element::Type("i4"), element::i4); + EXPECT_EQ(element::Type("I4"), element::i4); + EXPECT_EQ(element::Type("i8"), element::i8); + EXPECT_EQ(element::Type("I8"), element::i8); + EXPECT_EQ(element::Type("i16"), element::i16); + EXPECT_EQ(element::Type("I16"), element::i16); + EXPECT_EQ(element::Type("i32"), element::i32); + EXPECT_EQ(element::Type("I32"), element::i32); + EXPECT_EQ(element::Type("i64"), element::i64); + EXPECT_EQ(element::Type("I64"), element::i64); + + EXPECT_EQ(element::Type("bin"), element::u1); + EXPECT_EQ(element::Type("BIN"), element::u1); + EXPECT_EQ(element::Type("u1"), element::u1); + EXPECT_EQ(element::Type("U1"), element::u1); + EXPECT_EQ(element::Type("u4"), element::u4); + EXPECT_EQ(element::Type("U4"), element::u4); + EXPECT_EQ(element::Type("u8"), element::u8); + EXPECT_EQ(element::Type("U8"), element::u8); + EXPECT_EQ(element::Type("u16"), element::u16); + EXPECT_EQ(element::Type("U16"), element::u16); + EXPECT_EQ(element::Type("u32"), element::u32); + EXPECT_EQ(element::Type("U32"), element::u32); + EXPECT_EQ(element::Type("u64"), element::u64); + EXPECT_EQ(element::Type("U64"), element::u64); + + EXPECT_EQ(element::Type("undefined"), element::undefined); + EXPECT_EQ(element::Type("UNSPECIFIED"), element::undefined); + EXPECT_EQ(element::Type("dynamic"), element::dynamic); + + EXPECT_THROW(element::Type("some_string"), ov::Exception); +} + TEST(element_type, mapable) { std::map test_map; diff --git a/src/core/tests/eval.cpp b/src/core/tests/eval.cpp index 00b72d098fce57..4961d7cdb39f7c 100644 --- a/src/core/tests/eval.cpp +++ b/src/core/tests/eval.cpp @@ -101,14 +101,18 @@ TEST(eval, bad_get_data_ptr) { TEST(eval, max_eval_parameter) { auto p = make_shared(element::i64, Shape{}); + OPENVINO_SUPPRESS_DEPRECATED_START auto result = maximum_value(p); + OPENVINO_SUPPRESS_DEPRECATED_END EXPECT_FALSE(result.first); EXPECT_EQ(result.second, numeric_limits::max()); } TEST(eval, max_eval_constant) { auto c = op::Constant::create(element::i64, Shape{}, {27}); + OPENVINO_SUPPRESS_DEPRECATED_START auto result = maximum_value(c); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result.first); EXPECT_EQ(result.second, 27); } @@ -117,7 +121,9 @@ TEST(eval, max_eval_minimum_constant) { auto c = op::Constant::create(element::i64, Shape{}, {27}); auto p = make_shared(element::i64, Shape{}); auto m = make_shared(c, p); + OPENVINO_SUPPRESS_DEPRECATED_START auto result = maximum_value(m); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result.first); EXPECT_EQ(result.second, 27); } @@ -134,7 +140,9 @@ TEST(eval, max_eval_reduce_min) { auto squeezes = make_shared( make_shared(reduce, make_shared(element::i32, Shape{1}, 0)), make_shared(element::i64, Shape{1}, 0)); + OPENVINO_SUPPRESS_DEPRECATED_START EXPECT_EQ(maximum_value(squeezes).second, 37); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(eval, evaluate_shape_of) { @@ -173,24 +181,6 @@ TEST(eval, evaluate_dynamic_range_sum) { ASSERT_EQ(cval, seq); } -TEST(eval, interpret_dynamic_range_sum) { - auto p_start = make_shared(element::f32, PartialShape{}); - auto p_stop = make_shared(element::f32, PartialShape{}); - auto p_step = make_shared(element::f32, PartialShape{}); - auto p1 = make_shared(element::f32, PartialShape{}); - auto range = make_shared(p_start, p_stop, p_step); - auto add = make_shared(range, p1); - auto fun = make_shared(OutputVector{add}, ParameterVector{p_start, p_stop, p_step, p1}); - auto test_case = test::TestCase(fun); - test_case.add_input(std::vector{1.0f}); - test_case.add_input(std::vector{10.0f}); - test_case.add_input(std::vector{3.0f}); - test_case.add_input(std::vector{7.0f}); - vector seq{8.0f, 11.0f, 14.0f}; - test_case.add_expected_output({3}, seq); - test_case.run(); -} - TEST(eval, evaluate_broadcast_v3_bidirectional) { Shape shape_a{4, 1}; auto A = make_shared(element::f32, shape_a); diff --git a/src/core/tests/frontend/mock_frontend.cpp b/src/core/tests/frontend/mock_frontend.cpp index 257c464af36114..642943805119ec 100644 --- a/src/core/tests/frontend/mock_frontend.cpp +++ b/src/core/tests/frontend/mock_frontend.cpp @@ -134,7 +134,9 @@ class FrontEndMock : public FrontEnd { } bool supported_impl(const std::vector& variants) const override { - if (variants.size() == 1 && variants[0].is()) { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + if (variants.size() == 1 + extra_variants_num && variants[0].is()) { std::string command = variants[0].as(); FRONT_END_GENERAL_CHECK(command != "throw_now", "Test exception"); } @@ -146,8 +148,10 @@ class FrontEndMock : public FrontEnd { } InputModel::Ptr load_impl(const std::vector& variants) const override { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; auto input_model = std::make_shared(); - if (variants.size() == 1 && variants[0].is()) { + if (variants.size() == 1 + extra_variants_num && variants[0].is()) { std::string command = variants[0].as(); if (command == "throw_now") { OPENVINO_THROW("Test throw load input model"); diff --git a/src/core/tests/graph_rewrite.cpp b/src/core/tests/graph_rewrite.cpp index c7fb67d243c4c3..d85146b1ffd2d0 100644 --- a/src/core/tests/graph_rewrite.cpp +++ b/src/core/tests/graph_rewrite.cpp @@ -55,9 +55,9 @@ class Anchor : public ngraph::pass::GraphRewrite { Anchor() : GraphRewrite() {} }; -NGRAPH_RTTI_DEFINITION(TestPass, "TestPass", 0); -NGRAPH_RTTI_DEFINITION(Anchor, "Anchor", 0); -NGRAPH_RTTI_DEFINITION(GatherNodesPass, "GatherNodesPass", 0); +NGRAPH_RTTI_DEFINITION(TestPass, "TestPass"); +NGRAPH_RTTI_DEFINITION(Anchor, "Anchor"); +NGRAPH_RTTI_DEFINITION(GatherNodesPass, "GatherNodesPass"); std::shared_ptr get_function() { auto data = std::make_shared(ngraph::element::f32, ngraph::Shape{3, 1, 2}); @@ -165,7 +165,7 @@ class PrivateDivide : public ngraph::opset3::Divide { using ngraph::opset3::Divide::Divide; }; -NGRAPH_RTTI_DEFINITION(PrivateDivide, "PrivateDivide", 0, ngraph::opset3::Divide); +NGRAPH_RTTI_DEFINITION(PrivateDivide, "PrivateDivide", ngraph::opset3::Divide); std::shared_ptr get_derived_function() { auto data = std::make_shared(ngraph::element::f32, ngraph::Shape{3, 1, 2}); @@ -434,7 +434,7 @@ class CheckConsumers : public ngraph::pass::MatcherPass { } }; -NGRAPH_RTTI_DEFINITION(CheckConsumers, "CheckConsumers", 0); +NGRAPH_RTTI_DEFINITION(CheckConsumers, "CheckConsumers"); TEST(GraphRewriteTest, nodes_use_count) { auto f = get_function(); diff --git a/src/core/tests/layout.cpp b/src/core/tests/layout.cpp index c6c2627fe9a3ce..1214b6503ed0ad 100644 --- a/src/core/tests/layout.cpp +++ b/src/core/tests/layout.cpp @@ -195,3 +195,11 @@ TEST(layout, attribute_adapter) { at.set("NHCW"); EXPECT_EQ(l, l2); } + +TEST(layout, compare_string) { + Layout l = "HWC"; + EXPECT_EQ("[H,W,C]", l.to_string()); + Layout l2 = l.to_string().c_str(); + EXPECT_EQ(l2, l); + EXPECT_EQ("[H,W,C]", l2.to_string()); +} diff --git a/src/core/tests/model.cpp b/src/core/tests/model.cpp index f5550c62c79f11..3ba65dd8c0a44d 100644 --- a/src/core/tests/model.cpp +++ b/src/core/tests/model.cpp @@ -1034,6 +1034,31 @@ TEST(model, add_output_port) { EXPECT_EQ(f->get_results()[1]->input_value(0).get_node(), relu1.get()); } +TEST(model, add_output_to_new_subgraph) { + auto arg0 = std::make_shared(ov::element::f32, ov::PartialShape{1}); + arg0->set_friendly_name("data"); + arg0->get_output_tensor(0).set_names({"input"}); + + auto relu1 = std::make_shared(arg0); + relu1->set_friendly_name("relu1"); + relu1->get_output_tensor(0).set_names({"relu_t1"}); + + auto relu2 = std::make_shared(relu1); + relu2->set_friendly_name("relu2"); + relu2->get_output_tensor(0).set_names({"relu_t2"}); + auto f = std::make_shared(relu2, ov::ParameterVector{arg0}); + f->validate_nodes_and_infer_types(); + + EXPECT_EQ(f->get_results().size(), 1); + + ov::Output out; + EXPECT_NO_THROW( + out = f->add_output(ov::opset8::Constant::create(ov::element::i32, {1}, std::vector{1})->output(0))); + EXPECT_NO_THROW(f->get_ordered_ops()); + EXPECT_EQ(out.get_node(), f->get_results()[1].get()); + EXPECT_EQ(f->get_results().size(), 2); +} + TEST(model, add_output_incorrect_tensor_name) { auto arg0 = std::make_shared(ov::element::f32, ov::PartialShape{1}); arg0->set_friendly_name("data"); diff --git a/src/core/tests/op_eval/memory.cpp b/src/core/tests/op_eval/memory.cpp deleted file mode 100644 index 81ca0f0ce7fa3a..00000000000000 --- a/src/core/tests/op_eval/memory.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include - -#include "engines_util/execute_tools.hpp" -#include "gtest/gtest.h" -#include "ngraph/op/util/variable.hpp" -#include "ngraph/op/util/variable_context.hpp" -#include "ngraph/opsets/opset7.hpp" -#include "ngraph/util.hpp" -#include "ngraph/validation_util.hpp" -#include "util/all_close_f.hpp" -#include "util/test_tools.hpp" - -using namespace std; -using namespace ngraph; -using namespace ngraph::opset7; - -OPENVINO_SUPPRESS_DEPRECATED_START - -shared_ptr AssignReadGraph() { - auto p = make_shared(element::f32, Shape{3}); - auto variable = make_shared(VariableInfo{PartialShape::dynamic(), element::dynamic, "var_1"}); - auto read_value = make_shared(p, variable); - auto assign = make_shared(read_value, variable); - return make_shared(OutputVector{assign}, ParameterVector{p}, VariableVector{variable}); -} - -shared_ptr AssignReadAddGraph() { - auto p = make_shared(element::f32, Shape{3}); - auto c = std::make_shared(element::f32, Shape{3}, std::vector({0, 0, 0})); - auto variable = make_shared(VariableInfo{PartialShape::dynamic(), element::dynamic, "var_1"}); - auto read_value = make_shared(c, variable); - auto add = make_shared(p, read_value); - auto assign = make_shared(add, variable); - return make_shared(OutputVector{assign}, ParameterVector{p}, VariableVector{variable}); -} - -shared_ptr AssignReadMultiVariableGraph() { - auto c = std::make_shared(element::f32, Shape{3}, std::vector({0, 0, 0})); - - auto variable = make_shared(VariableInfo{PartialShape::dynamic(), element::dynamic, "var_1"}); - auto variable_2 = make_shared(VariableInfo{PartialShape::dynamic(), element::dynamic, "var_2"}); - - auto read_value = make_shared(c, variable); - auto read_value_2 = make_shared(c, variable_2); - - auto add = make_shared(read_value_2, read_value); - - auto assign = make_shared(add, variable); - auto assign_2 = make_shared(read_value_2, variable_2); - - return make_shared(OutputVector{assign}, ParameterVector{}, VariableVector{variable, variable_2}); -} - -TEST(op_eval, assign_readvalue_without_evaluation_context) { - auto fun = AssignReadGraph(); - auto result = make_shared(); - - const int COUNT_RUNS = 10; - std::vector inputs{-5, 0, 5}; - std::vector expected_result{0, 0, 0}; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)})); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} - -TEST(op_eval, assign_readvalue_evaluation_context) { - auto fun = AssignReadGraph(); - auto result = make_shared(); - const auto& variables = fun->get_variables(); - EXPECT_EQ(variables.size(), 1); - - std::vector inputs{-5, 0, 5}; - std::vector expected_result{0, 0, 0}; - - EvaluationContext eval_context; - HostTensorPtr h_tensor = make_host_tensor(Shape{3}, inputs); - VariableContext variable_context; - variable_context.set_variable_value(variables[0], std::make_shared(h_tensor)); - eval_context.emplace("VariableContext", variable_context); - - const int COUNT_RUNS = 10; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} - -TEST(op_eval, assign_readvalue_add) { - auto fun = AssignReadAddGraph(); - const auto& variables = fun->get_variables(); - EXPECT_EQ(variables.size(), 1); - - std::vector inputs{-5, 0, 5}; - - // creating context - EvaluationContext eval_context; - auto variable_context = VariableContext(); - auto variable_value = make_shared(make_host_tensor(Shape{3}, inputs)); - variable_context.set_variable_value(variables[0], variable_value); - eval_context.emplace("VariableContext", variable_context); - - auto result = make_shared(); - const int COUNT_RUNS = 10; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - auto result_data = read_vector(result); - - auto cnt = static_cast(i + 1); - std::vector expected_result{inputs[0] * cnt, inputs[1] * cnt, inputs[2] * cnt}; - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} - -TEST(op_eval, assign_readvalue_reset_before_evaluate) { - auto fun = AssignReadAddGraph(); - const auto& variables = fun->get_variables(); - EXPECT_EQ(variables.size(), 1); - - std::vector inputs{-5, 0, 5}; - - // creating context - EvaluationContext eval_context; - auto variable_context = VariableContext(); - auto variable_value = make_shared(make_host_tensor(Shape{3}, inputs)); - variable_value->set_reset(false); - variable_context.set_variable_value(variables[0], variable_value); - eval_context.emplace("VariableContext", variable_context); - - auto result = make_shared(); - const int COUNT_RUNS = 10; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - auto result_data = read_vector(result); - - auto cnt = static_cast(i + 2); - std::vector expected_result{inputs[0] * cnt, inputs[1] * cnt, inputs[2] * cnt}; - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} - -TEST(op_eval, assign_readvalue_add_reset) { - auto fun = AssignReadAddGraph(); - std::vector inputs{-5, 0, 5}; - const auto& variables = fun->get_variables(); - EXPECT_EQ(variables.size(), 1); - - // creating a Context - EvaluationContext eval_context; - auto variable_context = VariableContext(); - auto variable_value = make_shared(make_host_tensor(Shape{3}, inputs)); - variable_context.set_variable_value(variables[0], variable_value); - eval_context.emplace("VariableContext", variable_context); - - auto result = make_shared(); - const int COUNT_RUNS = 10; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - auto result_data = read_vector(result); - - auto cnt = static_cast(i + 1); - std::vector expected_result{inputs[0] * cnt, inputs[1] * cnt, inputs[2] * cnt}; - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } - - const auto& found_context = eval_context.find("VariableContext"); - EXPECT_NE(found_context, eval_context.end()); - found_context->second.as().reset_variable_context(); - - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - auto result_data = read_vector(result); - - auto cnt = static_cast(i + 1); - std::vector expected_result{inputs[0] * cnt, inputs[1] * cnt, inputs[2] * cnt}; - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} - -TEST(op_eval, assign_readvalue_add_modify) { - auto fun = AssignReadAddGraph(); - std::vector inputs{-5, 0, 5}; - const auto& variables = fun->get_variables(); - EXPECT_EQ(variables.size(), 1); - - // creating context - EvaluationContext eval_context; - auto variable_context = VariableContext(); - auto variable_value = make_shared(make_host_tensor(Shape{3}, inputs)); - variable_context.set_variable_value(variables[0], variable_value); - eval_context.emplace("VariableContext", variable_context); - - auto result = make_shared(); - const int COUNT_RUNS = 10; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - - auto cnt = static_cast(i + 1); - std::vector expected_result{inputs[0] * cnt, inputs[1] * cnt, inputs[2] * cnt}; - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } - - const auto& found_context = eval_context.find("VariableContext"); - EXPECT_NE(found_context, eval_context.end()); - const auto& var_value = found_context->second.as().get_variable_value(variables[0]); - EXPECT_NE(var_value, nullptr); - var_value->set_value(make_host_tensor(Shape{3}, {1, 2, 3})); - - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {make_host_tensor(Shape{3}, inputs)}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - - auto cnt = static_cast(i + 1); - std::vector expected_result{1 + inputs[0] * cnt, 2 + inputs[1] * cnt, 3 + inputs[2] * cnt}; - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} - -TEST(op_eval, assign_readvalue_add_modify_multi_variables) { - auto fun = AssignReadMultiVariableGraph(); - std::vector inputs_1{2, 2, 2}; - std::vector inputs_2{1, 3, 5}; - auto var_1 = fun->get_variable_by_id("var_1"); - auto var_2 = fun->get_variable_by_id("var_2"); - EXPECT_NE(var_1, nullptr); - EXPECT_NE(var_2, nullptr); - - // creating context - EvaluationContext eval_context; - auto variable_context = VariableContext(); - auto variable_value_1 = make_shared(make_host_tensor(Shape{3}, inputs_1)); - auto variable_value_2 = make_shared(make_host_tensor(Shape{3}, inputs_2)); - variable_value_1->set_reset(false); - variable_value_2->set_reset(false); - variable_context.set_variable_value(var_1, variable_value_1); - variable_context.set_variable_value(var_2, variable_value_2); - eval_context.emplace("VariableContext", variable_context); - - auto result = make_shared(); - const int COUNT_RUNS = 10; - - std::vector expected_result = inputs_1; - for (size_t i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - - for (size_t j = 0; j < expected_result.size(); ++j) { - expected_result[j] += inputs_2[j]; - } - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } - - const auto& found_context = eval_context.find("VariableContext"); - EXPECT_NE(found_context, eval_context.end()); - auto var_context = found_context->second.as(); - - auto var_value = variable_context.get_variable_value(var_1); - EXPECT_NE(var_value, nullptr); - var_value->set_value(make_host_tensor(Shape{3}, {1, 2, 3})); - - auto var_value_2 = variable_context.get_variable_value(var_2); - EXPECT_NE(var_value_2, nullptr); - var_value_2->set_reset(true); - - expected_result = {1, 2, 3}; - for (int i = 0; i < COUNT_RUNS; ++i) { - ASSERT_TRUE(fun->evaluate({result}, {}, eval_context)); - EXPECT_EQ(result->get_element_type(), element::f32); - EXPECT_EQ(result->get_shape(), Shape{3}); - - ASSERT_TRUE(test::all_close_f(read_vector(result), expected_result)); - } -} diff --git a/src/core/tests/opset.cpp b/src/core/tests/opset.cpp index cfc5cb7250b41a..289ae1188cea53 100644 --- a/src/core/tests/opset.cpp +++ b/src/core/tests/opset.cpp @@ -72,7 +72,7 @@ INSTANTIATE_TEST_SUITE_P(opset, class MyOpOld : public ov::op::Op { public: - static constexpr ov::DiscreteTypeInfo type_info{"MyOpOld", static_cast(0)}; + static constexpr ov::DiscreteTypeInfo type_info{"MyOpOld"}; const ov::DiscreteTypeInfo& get_type_info() const override { return type_info; } @@ -121,7 +121,7 @@ TEST(opset, custom_opset) { opset.insert(); opset.insert(); EXPECT_EQ(opset.get_types_info().size(), 3); - EXPECT_TRUE(opset.contains_type("MyOpNewFromOld")); - EXPECT_TRUE(opset.contains_type("MyOpNew")); - EXPECT_TRUE(opset.contains_type("MyOpIncorrect")); + EXPECT_TRUE(opset.contains_type(std::string("MyOpNewFromOld"))); + EXPECT_TRUE(opset.contains_type(std::string("MyOpNew"))); + EXPECT_TRUE(opset.contains_type(std::string("MyOpIncorrect"))); } diff --git a/src/core/tests/partial_shape.cpp b/src/core/tests/partial_shape.cpp index 219069f6d44feb..3b6ae3f829a1ea 100644 --- a/src/core/tests/partial_shape.cpp +++ b/src/core/tests/partial_shape.cpp @@ -808,7 +808,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_ok) { Strides window_strides{1, 1, 1, 1}; Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; - + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -818,6 +818,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_ok) { window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result_shape.same_scheme( PartialShape{Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); @@ -833,6 +834,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_data Strides window_strides{1, 1, 1, 1}; Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -846,6 +848,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_data is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_window_dilation) { @@ -858,6 +861,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_wind Strides window_strides{1, 1, 1, 1}; Strides window_dilation{1, 0, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -871,6 +875,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_wind is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_window_strides) { @@ -883,6 +888,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_wind Strides window_strides{1, 1, 1, 0}; Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -896,6 +902,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_dynamic_zero_wind is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ok) { @@ -909,6 +916,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ok Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -918,6 +926,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ok window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result_shape.same_scheme(PartialShape::dynamic(4))); } @@ -932,6 +941,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ze Strides window_strides{1, 1, 1, 1}; Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -945,6 +955,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ze is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_neg_padding_ok) { @@ -957,6 +968,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ne Strides window_strides{1, 1, 1, 1}; Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -966,6 +978,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_dynamic_ne window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result_shape.same_scheme(PartialShape::dynamic(4))); } @@ -981,6 +994,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_ok Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -990,6 +1004,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_ok window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result_shape.same_scheme(PartialShape::dynamic(4))); } @@ -1005,6 +1020,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -1018,6 +1034,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_window_dilated_dim_zero) { @@ -1031,6 +1048,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi Strides window_dilation{1, 1, 3, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -1044,6 +1062,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_window_all_in_padding_ok) { @@ -1057,6 +1076,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -1066,6 +1086,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result_shape.same_scheme(PartialShape::dynamic(4))); } @@ -1081,6 +1102,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = false; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -1094,6 +1116,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_wi is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_dilated_window_not_all_in_padding) { @@ -1107,6 +1130,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_di Strides window_dilation{1, 1, 2, 1}; bool is_window_all_in_padding_allowed = false; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -1116,6 +1140,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_dynamic_rank_static_dynamic_di window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(result_shape.same_scheme(PartialShape::dynamic(4))); } @@ -1131,6 +1156,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -1140,6 +1166,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE( result_shape.same_scheme(PartialShape{Dimension::dynamic(), Dimension::dynamic(), 4, Dimension::dynamic()})); @@ -1156,6 +1183,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -1165,6 +1193,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE( result_shape.same_scheme(PartialShape{Dimension::dynamic(), Dimension::dynamic(), 5, Dimension::dynamic()})); @@ -1181,6 +1210,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -1190,6 +1220,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE( result_shape.same_scheme(PartialShape{Dimension::dynamic(), Dimension::dynamic(), 3, Dimension::dynamic()})); @@ -1206,6 +1237,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -1219,6 +1251,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dynamic_window_not_too_big_padding) { @@ -1232,6 +1265,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn Strides window_dilation{1, 1, 1, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), data_shape, data_dilation, @@ -1241,6 +1275,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn window_strides, window_dilation, is_window_all_in_padding_allowed); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE( result_shape.same_scheme(PartialShape{Dimension::dynamic(), Dimension::dynamic(), 2, Dimension::dynamic()})); @@ -1257,6 +1292,7 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn Strides window_dilation{1, 1, 2, 1}; bool is_window_all_in_padding_allowed = true; + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_THROW( { PartialShape result_shape = infer_windowed_reduction_output_shape(node.get(), @@ -1270,4 +1306,5 @@ TEST(partial_shape, infer_windowed_reduction_rank_static_dynamic_rank_static_dyn is_window_all_in_padding_allowed); }, NodeValidationFailure); + OPENVINO_SUPPRESS_DEPRECATED_END } diff --git a/src/core/tests/pass/serialization/deterministicity.cpp b/src/core/tests/pass/serialization/deterministicity.cpp index 2abfe1b6e53052..6193dddbd7a2a1 100644 --- a/src/core/tests/pass/serialization/deterministicity.cpp +++ b/src/core/tests/pass/serialization/deterministicity.cpp @@ -71,6 +71,7 @@ class SerializationDeterministicityTest : public ov::test::TestsCommon, public D #ifdef ENABLE_OV_ONNX_FRONTEND TEST_F(SerializationDeterministicityTest, BasicModel) { + GTEST_SKIP() << "Ticket 99981"; const std::string model = CommonTestUtils::getModelFromTestModelZoo(ov::util::path_join({SERIALIZED_ZOO, "ir/add_abc.onnx"})); @@ -88,6 +89,7 @@ TEST_F(SerializationDeterministicityTest, BasicModel) { } TEST_F(SerializationDeterministicityTest, ModelWithMultipleLayers) { + GTEST_SKIP() << "Ticket 99981"; const std::string model = CommonTestUtils::getModelFromTestModelZoo(ov::util::path_join({SERIALIZED_ZOO, "ir/addmul_abc.onnx"})); diff --git a/src/core/tests/pass/serialization/serialize.cpp b/src/core/tests/pass/serialization/serialize.cpp index a172e2eb92cd34..d2866901ba2223 100644 --- a/src/core/tests/pass/serialization/serialize.cpp +++ b/src/core/tests/pass/serialization/serialize.cpp @@ -59,12 +59,14 @@ class SerializationTest : public ov::test::TestsCommon, public testing::WithPara }; TEST_P(SerializationTest, CompareFunctions) { + GTEST_SKIP() << "Ticket 99981"; CompareSerialized([this](const std::shared_ptr& m) { ov::pass::Serialize(m_out_xml_path, m_out_bin_path).run_on_model(m); }); } TEST_P(SerializationTest, SerializeHelper) { + GTEST_SKIP() << "Ticket 99981"; CompareSerialized([this](const std::shared_ptr& m) { ov::serialize(m, m_out_xml_path, m_out_bin_path); }); diff --git a/src/core/tests/pass_config.cpp b/src/core/tests/pass_config.cpp index 3c9395aedc2bd2..e2c0d15e943eae 100644 --- a/src/core/tests/pass_config.cpp +++ b/src/core/tests/pass_config.cpp @@ -32,7 +32,7 @@ class RenameReLU : public ngraph::pass::MatcherPass { } }; -NGRAPH_RTTI_DEFINITION(RenameReLU, "RenameReLU", 0); +NGRAPH_RTTI_DEFINITION(RenameReLU, "RenameReLU"); class RenameSigmoid : public ngraph::pass::MatcherPass { public: @@ -50,7 +50,7 @@ class RenameSigmoid : public ngraph::pass::MatcherPass { } }; -NGRAPH_RTTI_DEFINITION(RenameSigmoid, "RenameSigmoid", 0); +NGRAPH_RTTI_DEFINITION(RenameSigmoid, "RenameSigmoid"); class TestFunctionPass : public ngraph::pass::FunctionPass { public: @@ -67,7 +67,7 @@ class TestFunctionPass : public ngraph::pass::FunctionPass { } }; -NGRAPH_RTTI_DEFINITION(TestFunctionPass, "TestFunctionPass", 0); +NGRAPH_RTTI_DEFINITION(TestFunctionPass, "TestFunctionPass"); class TestGraphRewritePass : public ngraph::pass::GraphRewrite { public: @@ -78,7 +78,7 @@ class TestGraphRewritePass : public ngraph::pass::GraphRewrite { } }; -NGRAPH_RTTI_DEFINITION(TestGraphRewritePass, "TestGraphRewritePass", 0); +NGRAPH_RTTI_DEFINITION(TestGraphRewritePass, "TestGraphRewritePass"); std::tuple, std::shared_ptr, std::shared_ptr> get_test_function() { auto data = std::make_shared(ngraph::element::f32, ngraph::Shape{3, 1, 2}); @@ -289,7 +289,7 @@ class TestNestedMatcher : public ngraph::pass::MatcherPass { } }; -NGRAPH_RTTI_DEFINITION(TestNestedMatcher, "TestNestedMatcher", 0); +NGRAPH_RTTI_DEFINITION(TestNestedMatcher, "TestNestedMatcher"); class TestNestedGraphRewrite : public pass::GraphRewrite { public: @@ -299,7 +299,7 @@ class TestNestedGraphRewrite : public pass::GraphRewrite { } }; -NGRAPH_RTTI_DEFINITION(TestNestedGraphRewrite, "TestNestedGraphRewrite", 0); +NGRAPH_RTTI_DEFINITION(TestNestedGraphRewrite, "TestNestedGraphRewrite"); TEST(PassConfig, EnableDisablePasses10) { std::shared_ptr f; diff --git a/src/core/tests/rtti.cpp b/src/core/tests/rtti.cpp index bb0362f52a366f..e5a1619e75d3ce 100644 --- a/src/core/tests/rtti.cpp +++ b/src/core/tests/rtti.cpp @@ -42,7 +42,7 @@ class OpTypeVersionParent : public OpType { class OpTypeVersionParentOld : public OpType { public: - OPENVINO_OP("OpTypeVersionParentOld", "my_version1", OpType, 1); + OPENVINO_OP("OpTypeVersionParentOld", "my_version1", OpType); OpTypeVersionParentOld() = default; std::shared_ptr clone_with_new_inputs(const OutputVector& inputs) const override { @@ -56,7 +56,6 @@ TEST(rtti, op_with_type) { auto type_info = op.get_type_info(); ASSERT_EQ(type_info, OpType::get_type_info_static()); ASSERT_EQ(strcmp(type_info.name, "OpType"), 0); - ASSERT_EQ(type_info.version, 0); ASSERT_EQ(strcmp(type_info.version_id, "extension"), 0); ASSERT_NE(type_info.parent, nullptr); ASSERT_EQ(*type_info.parent, ngraph::op::Op::get_type_info_static()); @@ -67,7 +66,6 @@ TEST(rtti, op_with_type_version) { auto type_info = op.get_type_info(); ASSERT_EQ(type_info, OpTypeVersion::get_type_info_static()); ASSERT_EQ(strcmp(type_info.name, "OpTypeVersion"), 0); - ASSERT_EQ(type_info.version, 0); ASSERT_EQ(strcmp(type_info.version_id, "my_version"), 0); ASSERT_NE(type_info.parent, nullptr); ASSERT_EQ(*type_info.parent, ngraph::op::Op::get_type_info_static()); @@ -78,7 +76,6 @@ TEST(rtti, op_with_type_version_parent) { auto type_info = op.get_type_info(); ASSERT_EQ(type_info, OpTypeVersionParent::get_type_info_static()); ASSERT_EQ(strcmp(type_info.name, "OpTypeVersionParent"), 0); - ASSERT_EQ(type_info.version, 0); ASSERT_EQ(strcmp(type_info.version_id, "my_version"), 0); ASSERT_NE(type_info.parent, nullptr); ASSERT_EQ(*type_info.parent, OpType::get_type_info_static()); @@ -90,7 +87,6 @@ TEST(rtti, op_with_type_version_parent_old) { ASSERT_EQ(type_info, OpTypeVersionParentOld::get_type_info_static()); ASSERT_EQ(strcmp(type_info.name, "OpTypeVersionParentOld"), 0); ASSERT_EQ(strcmp(type_info.version_id, "my_version1"), 0); - ASSERT_EQ(type_info.version, 1); ASSERT_NE(type_info.parent, nullptr); ASSERT_EQ(*type_info.parent, OpType::get_type_info_static()); } diff --git a/src/core/tests/type_info.cpp b/src/core/tests/type_info.cpp index 1beac34b3055c2..98c1f9c8e146ed 100644 --- a/src/core/tests/type_info.cpp +++ b/src/core/tests/type_info.cpp @@ -10,11 +10,11 @@ OPENVINO_SUPPRESS_DEPRECATED_START TEST(type_info, compare_old_type) { - ov::DiscreteTypeInfo type1("type1", static_cast(0)); - ov::DiscreteTypeInfo type2("type2", static_cast(0)); - ov::DiscreteTypeInfo type3("type1", 1ul); - ov::DiscreteTypeInfo type4("type3", static_cast(0), &type1); - ov::DiscreteTypeInfo type5("type3", static_cast(0), &type2); + ov::DiscreteTypeInfo type1("type1"); + ov::DiscreteTypeInfo type2("type2"); + ov::DiscreteTypeInfo type3("type1"); + ov::DiscreteTypeInfo type4("type3", &type1); + ov::DiscreteTypeInfo type5("type3", &type2); ASSERT_TRUE(type1 != type2); ASSERT_TRUE(type1 == type1); ASSERT_TRUE(type1 < type2); @@ -46,40 +46,37 @@ TEST(type_info, compare_new_type) { } TEST(type_info, compare_new_with_old_type) { - ov::DiscreteTypeInfo type1("type1", static_cast(0), "version1"); - ov::DiscreteTypeInfo type1_o("type1", static_cast(0)); + ov::DiscreteTypeInfo type1("type1", "version1"); + ov::DiscreteTypeInfo type1_o("type1", "version1"); ASSERT_TRUE(type1 == type1_o); } TEST(type_info, check_hash_value) { - const auto& hash_val = [](const char* name, const char* version_id, uint64_t version) -> size_t { + const auto& hash_val = [](const char* name, const char* version_id) -> size_t { size_t name_hash = name ? std::hash()(std::string(name)) : 0; - size_t version_hash = std::hash()(version); size_t version_id_hash = version_id ? std::hash()(std::string(version_id)) : 0; // don't use parent for hash calculation, it is not a part of type (yet) - return ov::util::hash_combine(std::vector{name_hash, version_hash, version_id_hash}); + return ov::util::hash_combine(std::vector{name_hash, version_id_hash}); }; - ov::DiscreteTypeInfo type("type1", 0, "version1"); - ov::DiscreteTypeInfo type_old("type1", 1); - ov::DiscreteTypeInfo type_with_version("type1", 1, "version1"); - ov::DiscreteTypeInfo type_empty_name("", static_cast(0)); - ov::DiscreteTypeInfo type_empty_ver("type", static_cast(0), ""); - EXPECT_EQ(hash_val(type.name, type.version_id, type.version), type.hash()); - EXPECT_EQ(hash_val(type_old.name, type_old.version_id, type_old.version), type_old.hash()); - EXPECT_EQ(hash_val(type_with_version.name, type_with_version.version_id, type_with_version.version), - type_with_version.hash()); - EXPECT_EQ(hash_val(type_empty_name.name, type_empty_name.version_id, type_empty_name.version), - type_empty_name.hash()); - EXPECT_EQ(hash_val(type_empty_ver.name, type_empty_ver.version_id, type_empty_ver.version), type_empty_ver.hash()); + ov::DiscreteTypeInfo type("type1", "version1"); + ov::DiscreteTypeInfo type_old("type1"); + ov::DiscreteTypeInfo type_with_version("type1", "version1"); + ov::DiscreteTypeInfo type_empty_name(""); + ov::DiscreteTypeInfo type_empty_ver("type", ""); + EXPECT_EQ(hash_val(type.name, type.version_id), type.hash()); + EXPECT_EQ(hash_val(type_old.name, type_old.version_id), type_old.hash()); + EXPECT_EQ(hash_val(type_with_version.name, type_with_version.version_id), type_with_version.hash()); + EXPECT_EQ(hash_val(type_empty_name.name, type_empty_name.version_id), type_empty_name.hash()); + EXPECT_EQ(hash_val(type_empty_ver.name, type_empty_ver.version_id), type_empty_ver.hash()); } TEST(type_info, find_in_map) { std::vector vector_names; - ov::DiscreteTypeInfo a("Mod", 1ul, "opset1"); - ov::DiscreteTypeInfo b("Prelu", static_cast(0), "opset1"); - ov::DiscreteTypeInfo c("Vector", static_cast(0)); - ov::DiscreteTypeInfo d("Mod", 1ul, "opset3"); - ov::DiscreteTypeInfo f("Mod", 2ul); + ov::DiscreteTypeInfo a("Mod", "opset1"); + ov::DiscreteTypeInfo b("Prelu", "opset1"); + ov::DiscreteTypeInfo c("Vector"); + ov::DiscreteTypeInfo d("Mod", "opset3"); + ov::DiscreteTypeInfo f("Mod", "opset2"); std::map test_map; test_map[a] = 1; @@ -94,20 +91,20 @@ TEST(type_info, find_in_map) { test_map[type] = 2; std::string name = type.name; vector_names.emplace_back(name); - ov::DiscreteTypeInfo t(vector_names.rbegin()->c_str(), 1000); - ov::DiscreteTypeInfo t2(vector_names.rbegin()->c_str(), static_cast(0)); + ov::DiscreteTypeInfo t(vector_names.rbegin()->c_str()); + ov::DiscreteTypeInfo t2(vector_names.rbegin()->c_str()); test_map[t] = 3; test_map[t2] = 4; std::string name1 = "a" + name; vector_names.emplace_back(name1); - ov::DiscreteTypeInfo t3(vector_names.rbegin()->c_str(), 1000); - ov::DiscreteTypeInfo t4(vector_names.rbegin()->c_str(), static_cast(0)); + ov::DiscreteTypeInfo t3(vector_names.rbegin()->c_str()); + ov::DiscreteTypeInfo t4(vector_names.rbegin()->c_str()); test_map[t3] = 5; test_map[t4] = 6; std::string name2 = name + "z"; vector_names.emplace_back(name2); - ov::DiscreteTypeInfo t5(vector_names.rbegin()->c_str(), 1000); - ov::DiscreteTypeInfo t6(vector_names.rbegin()->c_str(), static_cast(0)); + ov::DiscreteTypeInfo t5(vector_names.rbegin()->c_str()); + ov::DiscreteTypeInfo t6(vector_names.rbegin()->c_str()); test_map[t5] = 7; test_map[t6] = 8; } diff --git a/src/core/tests/type_prop/binary_convolution.cpp b/src/core/tests/type_prop/binary_convolution.cpp index 27794152f147a2..e09b09966ee169 100644 --- a/src/core/tests/type_prop/binary_convolution.cpp +++ b/src/core/tests/type_prop/binary_convolution.cpp @@ -2,16 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "common_test_utils/test_assertions.hpp" #include "gtest/gtest.h" #include "ngraph/ngraph.hpp" #include "util/type_prop.hpp" using namespace std; using namespace ngraph; +using namespace testing; TEST(type_prop, bin_convolution_auto_padding_same) { - const PartialShape data_batch_shape{1, 1, 5, 5}; - const PartialShape filters_shape{1, 1, 3, 3}; + PartialShape data_batch_shape{1, 1, 5, 5}; + PartialShape filters_shape{1, 1, 3, 3}; + set_shape_labels(data_batch_shape, 10); + set_shape_labels(filters_shape, 20); Strides strides{1, 1}; CoordinateDiff pads_begin{0, 0}; CoordinateDiff pads_end{0, 0}; @@ -33,14 +37,17 @@ TEST(type_prop, bin_convolution_auto_padding_same) { pad_value, auto_pad); - ASSERT_TRUE(conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 1, 5, 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_THAT(get_shape_labels(conv->get_output_partial_shape(0)), ElementsAre(10, 20, ov::no_label, ov::no_label)); + EXPECT_EQ(conv->get_output_partial_shape(0), (PartialShape{1, 1, 5, 5})); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, bin_convolution_auto_padding_same_lower_spatial_dims_static) { - const PartialShape data_batch_shape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; - const PartialShape filters_shape{Dimension::dynamic(), Dimension::dynamic(), 3, 3}; + PartialShape data_batch_shape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; + PartialShape filters_shape{Dimension::dynamic(), Dimension::dynamic(), 3, 3}; + set_shape_labels(data_batch_shape, 10); + set_shape_labels(filters_shape, 20); Strides strides{1, 1}; CoordinateDiff pads_begin{0, 0}; CoordinateDiff pads_end{0, 0}; @@ -62,9 +69,10 @@ TEST(type_prop, bin_convolution_auto_padding_same_lower_spatial_dims_static) { pad_value, auto_pad); - ASSERT_TRUE(conv->get_output_partial_shape(0).same_scheme({Dimension::dynamic(), Dimension::dynamic(), 5, 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_THAT(get_shape_labels(conv->get_output_partial_shape(0)), ElementsAre(10, 20, ov::no_label, ov::no_label)); + EXPECT_EQ(conv->get_output_partial_shape(0), (PartialShape{Dimension::dynamic(), Dimension::dynamic(), 5, 5})); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, bin_convolution_auto_padding_same_upper_spatial_dims_static) { @@ -91,14 +99,16 @@ TEST(type_prop, bin_convolution_auto_padding_same_upper_spatial_dims_static) { pad_value, auto_pad); - ASSERT_TRUE(conv->get_output_partial_shape(0).same_scheme({Dimension::dynamic(), Dimension::dynamic(), 5, 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_output_partial_shape(0), (PartialShape{Dimension::dynamic(), Dimension::dynamic(), 5, 5})); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, bin_convolution_auto_padding_same_data_batch_spatial_dims_dynamic) { - const PartialShape data_batch_shape{1, 1, Dimension::dynamic(), 5}; - const PartialShape filters_shape{Dimension::dynamic(), 1, 3, 3}; + PartialShape data_batch_shape{1, 1, Dimension::dynamic(), 5}; + PartialShape filters_shape{Dimension::dynamic(), 1, 3, 3}; + set_shape_labels(data_batch_shape, 10); + set_shape_labels(filters_shape, 20); Strides strides{1, 1}; CoordinateDiff pads_begin{0, 0}; CoordinateDiff pads_end{0, 0}; @@ -120,9 +130,10 @@ TEST(type_prop, bin_convolution_auto_padding_same_data_batch_spatial_dims_dynami pad_value, auto_pad); - ASSERT_TRUE(conv->get_output_partial_shape(0).same_scheme({1, Dimension::dynamic(), Dimension::dynamic(), 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 1})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{0, 1})); + EXPECT_THAT(get_shape_labels(conv->get_output_partial_shape(0)), ElementsAre(10, 20, 12, ov::no_label)); + EXPECT_EQ(conv->get_output_partial_shape(0), (PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), 5})); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 1})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{0, 1})); } TEST(type_prop, bin_convolution_dyn_data_batch) { @@ -131,7 +142,7 @@ TEST(type_prop, bin_convolution_dyn_data_batch) { const auto auto_pad = op::PadType::EXPLICIT; const auto data_batch = make_shared(element::f32, PartialShape::dynamic()); - const auto filters = make_shared(element::u1, PartialShape{1, 1, 3}); + const auto filters = make_shared(element::u1, PartialShape{1, 1, 3, 3}); const auto bin_conv = make_shared(data_batch, filters, Strides{}, @@ -141,9 +152,8 @@ TEST(type_prop, bin_convolution_dyn_data_batch) { mode, pad_value, auto_pad); - ASSERT_TRUE(bin_conv->get_output_partial_shape(0).rank().same_scheme(Rank{3})); - ASSERT_TRUE( - bin_conv->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 1, Dimension::dynamic()})); + + EXPECT_EQ(bin_conv->get_output_partial_shape(0), (PartialShape{-1, 1, {1, -1}, {1, -1}})); } TEST(type_prop, bin_convolution_dyn_filters) { @@ -162,9 +172,8 @@ TEST(type_prop, bin_convolution_dyn_filters) { mode, pad_value, auto_pad); - ASSERT_TRUE(bin_conv->get_output_partial_shape(0).rank().same_scheme(Rank{4})); - ASSERT_TRUE(bin_conv->get_output_partial_shape(0).same_scheme( - PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); + + EXPECT_EQ(bin_conv->get_output_partial_shape(0), (PartialShape{1, -1, {1, 5}, {1, 5}})); } TEST(type_prop, bin_convolution_dyn_data_batch_and_filters) { @@ -183,8 +192,8 @@ TEST(type_prop, bin_convolution_dyn_data_batch_and_filters) { mode, pad_value, auto_pad); - ASSERT_TRUE(bin_conv->get_output_partial_shape(0).rank().is_dynamic()); - ASSERT_TRUE(bin_conv->get_output_partial_shape(0).same_scheme(PartialShape::dynamic())); + + EXPECT_EQ(bin_conv->get_output_partial_shape(0), PartialShape::dynamic()); } TEST(type_prop, bin_convolution_invalid_inputs_et) { @@ -263,7 +272,7 @@ TEST(type_prop, bin_convolution_invalid_input_ranks) { // data batch and filters have incompatible ranks FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Data batch and filters inputs must have same rank"); + EXPECT_HAS_SUBSTRING(error.what(), "Data batch and filters rank do not match"); } catch (...) { FAIL() << "Rank validation check of inputs failed for unexpected reason"; } @@ -285,7 +294,7 @@ TEST(type_prop, bin_convolution_invalid_input_ranks) { // data batch and filters have incompatible ranks FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Data batch and filters inputs must have same rank"); + EXPECT_HAS_SUBSTRING(error.what(), "Data batch and filters rank do not match"); } catch (...) { FAIL() << "Rank validation check of inputs failed for unexpected reason"; } @@ -320,20 +329,21 @@ TEST(type_prop, bin_convolution_invalid_spatial_dims_parameters) { // Strides have incompatible number of spatial dimensions FAIL() << "Incompatible stride number of spatial dimensions not detected."; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("Strides should be defined for all and only spatial features.")); + EXPECT_HAS_SUBSTRING(error.what(), + std::string("Strides should be defined for all and only spatial dimensions.")); } catch (...) { FAIL() << "Strides validation check failed for unexpected reason."; } try { - const auto data_batch = make_shared(element::f32, PartialShape{1, 1, 5}); - const auto filters = make_shared(element::u1, PartialShape{1, 1, 3}); + const auto data_batch = make_shared(element::f32, PartialShape{1, 1, 5, 5}); + const auto filters = make_shared(element::u1, PartialShape{1, 1, 3, 3}); const auto bin_conv = make_shared(data_batch, filters, - strides_1d, + Strides{1, 1}, CoordinateDiff{}, CoordinateDiff{}, - dilations_2d, + dilations_3d, mode, pad_value, auto_pad); @@ -341,28 +351,87 @@ TEST(type_prop, bin_convolution_invalid_spatial_dims_parameters) { FAIL() << "Incompatible dilations number of spatial dimensions not detected."; } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING(error.what(), - std::string("Dilations should be defined for all and only spatial features.")); + std::string("Dilations should be defined for all and only spatial dimensions.")); } catch (...) { FAIL() << "Dilations validation check failed for unexpected reason."; } try { - const auto data_batch = make_shared(element::f32, PartialShape{1, 1, 5, 5, 5}); - const auto filters = make_shared(element::u1, PartialShape{1, 1, 3, 3, 3}); + const auto data_batch = make_shared(element::f32, PartialShape{1, 1, 5, 5}); + const auto filters = make_shared(element::u1, PartialShape{1, 1, 3, 3}); const auto bin_conv = make_shared(data_batch, filters, - strides_3d, + Strides{1, 1}, pads_begin_3d, pads_end_2d, - dilations_3d, + dilations_2d, mode, pad_value, auto_pad); // Pads have incompatible number of spatial dimensions FAIL() << "Incompatible pads number of spatial dimensions not detected."; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("Pads should be defined for all and only spatial features.")); + EXPECT_HAS_SUBSTRING(error.what(), + std::string("Pads begin and end should be defined for all and only spatial dimensions.")); } catch (...) { FAIL() << "Pads validation check failed for unexpected reason."; } } + +class TypePropBinaryConvolutionV1Test : public TypePropOpTest { +protected: + CoordinateDiff empty_pad{}; +}; + +TEST_F(TypePropBinaryConvolutionV1Test, default_ctor) { + const auto data = make_shared(element::f32, PartialShape{1, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + + const auto op = make_op(); + op->set_arguments(OutputVector{data, filters}); + op->set_strides({1, 3}); + op->set_dilations({1, 2}); + op->set_pads_begin({2, 2}); + op->set_pads_end({2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->set_mode(op::v1::BinaryConvolution::BinaryConvolutionMode::XNOR_POPCOUNT); + op->set_pad_value(1.0f); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 2); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 3})); + EXPECT_EQ(op->get_dilations(), Strides({1, 2})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({1, 2, 6, 1})); +} + +TEST_F(TypePropBinaryConvolutionV1Test, interval_shapes) { + PartialShape data_batch_pshape{{1, 3}, 1, {1, 5}, {3, 10}}; + PartialShape filters_pshape{2, {1, 3}, 3, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(filters_pshape, 20); + + constexpr auto et = element::f32; + constexpr auto auto_pad = op::PadType::EXPLICIT; + constexpr auto mode = op::v1::BinaryConvolution::BinaryConvolutionMode::XNOR_POPCOUNT; + constexpr auto pad_value = 1.0f; + + const auto data_batch = make_shared(et, data_batch_pshape); + const auto filters = make_shared(et, filters_pshape); + const auto op = make_op(data_batch, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + mode, + pad_value, + auto_pad); + + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(10, 20, ov::no_label, ov::no_label)); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({{1, 3}, 2, {1, 3}, {1, 8}})); + EXPECT_EQ(op->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(op->get_pads_end(), (CoordinateDiff{0, 0})); +} diff --git a/src/core/tests/type_prop/broadcast.cpp b/src/core/tests/type_prop/broadcast.cpp index e4a82aefb31285..e587512b4e7a70 100644 --- a/src/core/tests/type_prop/broadcast.cpp +++ b/src/core/tests/type_prop/broadcast.cpp @@ -613,7 +613,6 @@ TEST(type_prop, broadcast_v3_bidirectional_mode_string) { const auto broadcast_v3 = make_shared(arg, shape, "BIDIRECTIONAL"); ASSERT_EQ(broadcast_v3->get_broadcast_spec(), op::BroadcastType::BIDIRECTIONAL); - ASSERT_EQ(broadcast_v3->get_version(), 3); } TEST(type_prop, broadcast_v3_shape_unexpected_axes_mapping_input) { diff --git a/src/core/tests/type_prop/convolution.cpp b/src/core/tests/type_prop/convolution.cpp index 1aabaf769f7a74..f26899d58a7818 100644 --- a/src/core/tests/type_prop/convolution.cpp +++ b/src/core/tests/type_prop/convolution.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "common_test_utils/test_assertions.hpp" #include "convolution_shape_inference.hpp" #include "gtest/gtest.h" #include "ngraph/ngraph.hpp" @@ -9,8 +10,9 @@ using namespace std; using namespace ngraph; +using namespace testing; -TEST(type_prop, conv_v1_partial_rank) { +TEST(type_prop, convolution_v1_partial_rank) { PartialShape data_batch_shape{PartialShape::dynamic()}; PartialShape filters_shape{PartialShape::dynamic()}; Strides window_movement_strides{1, 1}; @@ -28,12 +30,14 @@ TEST(type_prop, conv_v1_partial_rank) { padding_above, window_dilation_strides); - ASSERT_TRUE(conv->get_output_partial_shape(0).is_dynamic()); + EXPECT_EQ(conv->get_output_partial_shape(0), PartialShape({-1, -1, {1, -1}, {1, -1}})); } -TEST(type_prop, conv_v1_partial_auto_padding_same) { - const PartialShape data_batch_shape{1, 1, 5, 5}; - const PartialShape filters_shape{1, 1, 3, 3}; +TEST(type_prop, convolution_v1_partial_auto_padding_same) { + PartialShape data_batch_shape{1, 1, 5, 5}; + PartialShape filters_shape{1, 1, 3, 3}; + set_shape_labels(data_batch_shape, 10); + set_shape_labels(filters_shape, 20); Strides strides{1, 1}; CoordinateDiff pads_begin{0, 0}; CoordinateDiff pads_end{0, 0}; @@ -46,14 +50,17 @@ TEST(type_prop, conv_v1_partial_auto_padding_same) { auto conv = make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations, auto_pad); - ASSERT_EQ(conv->get_output_partial_shape(0), (PartialShape{1, 1, 5, 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_output_partial_shape(0), (PartialShape{1, 1, 5, 5})); + EXPECT_THAT(get_shape_labels(conv->get_output_partial_shape(0)), ElementsAre(10, 20, ov::no_label, ov::no_label)); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); } -TEST(type_prop, conv_v1_partial_auto_padding_same_nc_dims_dynamic_same_lower) { - const PartialShape data_batch_shape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; - const PartialShape filters_shape{1, 1, 3, 3}; +TEST(type_prop, convolution_v1_partial_auto_padding_same_nc_dims_dynamic_same_lower) { + PartialShape data_batch_shape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; + PartialShape filters_shape{1, 1, 3, 3}; + set_shape_labels(data_batch_shape, 10); + set_shape_labels(filters_shape, 20); Strides strides{1, 1}; CoordinateDiff pads_begin{0, 0}; CoordinateDiff pads_end{0, 0}; @@ -66,12 +73,13 @@ TEST(type_prop, conv_v1_partial_auto_padding_same_nc_dims_dynamic_same_lower) { auto conv = make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations, auto_pad); - ASSERT_EQ(conv->get_output_partial_shape(0), PartialShape({Dimension::dynamic(), 1, 5, 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_output_partial_shape(0), PartialShape({Dimension::dynamic(), 1, 5, 5})); + EXPECT_THAT(get_shape_labels(conv->get_output_partial_shape(0)), ElementsAre(10, 20, ov::no_label, ov::no_label)); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); } -TEST(type_prop, conv_v1_partial_auto_padding_same_nc_dims_dynamic_same_upper) { +TEST(type_prop, convolution_v1_partial_auto_padding_same_nc_dims_dynamic_same_upper) { const PartialShape data_batch_shape{Dimension::dynamic(), Dimension::dynamic(), 5, 5}; const PartialShape filters_shape{1, 1, 2, 2}; Strides strides{1, 1}; @@ -91,10 +99,12 @@ TEST(type_prop, conv_v1_partial_auto_padding_same_nc_dims_dynamic_same_upper) { ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{1, 1})); } -TEST(type_prop, conv_v1_partial_auto_padding_same_spatial_dims_dynamic) { - const PartialShape data_batch_shape{1, 1, Dimension::dynamic(), 5}; - const PartialShape filters_shape{1, 1, 3, 3}; - Strides strides{1, 1}; +TEST(type_prop, convolution_v1_partial_auto_padding_same_spatial_dims_dynamic) { + PartialShape data_batch_shape{1, 1, Dimension::dynamic(), {3, 5}}; + PartialShape filters_shape{1, 1, 3, 3}; + set_shape_labels(data_batch_shape, 10); + set_shape_labels(filters_shape, 20); + Strides strides{2, 2}; CoordinateDiff pads_begin{0, 0}; CoordinateDiff pads_end{0, 0}; Strides dilations{1, 1}; @@ -106,12 +116,13 @@ TEST(type_prop, conv_v1_partial_auto_padding_same_spatial_dims_dynamic) { auto conv = make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations, auto_pad); - ASSERT_EQ(conv->get_output_partial_shape(0), PartialShape({1, 1, Dimension::dynamic(), 5})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 1})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{0, 1})); + EXPECT_EQ(conv->get_output_partial_shape(0), PartialShape({1, 1, Dimension::dynamic(), {2, 3}})); + EXPECT_THAT(get_shape_labels(conv->get_output_partial_shape(0)), ElementsAre(10, 20, 12, ov::no_label)); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{0, 0})); } -TEST(type_prop, conv_v1_partial_data_shape_dynamic) { +TEST(type_prop, convolution_v1_partial_data_shape_dynamic) { const PartialShape data_batch_shape{PartialShape::dynamic()}; const PartialShape filters_shape{1, 1, 3, 3}; Strides strides{1, 1}; @@ -126,22 +137,149 @@ TEST(type_prop, conv_v1_partial_data_shape_dynamic) { auto conv = make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations, auto_pad); - ASSERT_EQ(conv->get_output_partial_shape(0), + EXPECT_EQ(conv->get_output_partial_shape(0), PartialShape({Dimension::dynamic(), 1, Dimension::dynamic(), Dimension::dynamic()})); - ASSERT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(conv->get_pads_end(), (CoordinateDiff{0, 0})); +} + +class TypePropConvolutionV1Test : public TypePropOpTest { +protected: + CoordinateDiff empty_pad{}; +}; + +TEST_F(TypePropConvolutionV1Test, default_ctor) { + const auto data = make_shared(element::f32, PartialShape{1, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + + const auto op = make_op(); + op->set_arguments(OutputVector{data, filters}); + op->set_strides({1, 3}); + op->set_dilations({1, 2}); + op->set_pads_begin({2, 2}); + op->set_pads_end({2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 2); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 3})); + EXPECT_EQ(op->get_dilations(), Strides({1, 2})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({1, 2, 6, 1})); +} + +TEST_F(TypePropConvolutionV1Test, data_dynamic_rank_filters_2d) { + const auto data = make_shared(element::f32, PartialShape::dynamic()); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1, 1}; + + auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_UPPER); + + EXPECT_THAT(op->get_pads_begin(), ElementsAre(0, 0)); + EXPECT_THAT(op->get_pads_end(), ElementsAre(0, 0)); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 2, -1, -1})); +} + +TEST_F(TypePropConvolutionV1Test, data_rank_to_low) { + const auto data = make_shared(element::f32, PartialShape{2, 3}); + const auto filters = make_shared(element::f32, PartialShape{2, 3}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Expected a 3D, 4D or 5D tensor for the input")); } -TEST(type_prop, convolution_default_constructed) { - auto conv = make_shared(); - conv->set_auto_pad(op::PadType::SAME_LOWER); +TEST_F(TypePropConvolutionV1Test, data_rank_to_high) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4, 4, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Expected a 3D, 4D or 5D tensor for the input")); +} + +TEST_F(TypePropConvolutionV1Test, data_and_filters_rank_not_compatible) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Data batch and filters rank do not match")); +} + +TEST_F(TypePropConvolutionV1Test, data_and_filters_channel_number_not_compatible) { + const auto data = make_shared(element::f32, PartialShape{2, 2, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Data batch channel count (2) does not match filter input channel count (3)")); +} + +TEST_F(TypePropConvolutionV1Test, strides_not_defined_only_for_spatial_dims) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 1, 1}; + const auto dilations = Strides{1, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Strides should be defined for all and only spatial dimensions.")); +} + +TEST_F(TypePropConvolutionV1Test, dilations_not_defined_only_for_spatial_dims) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Dilations should be defined for all and only spatial dimensions.")); +} + +TEST_F(TypePropConvolutionV1Test, strides_has_zeros) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 0}; + const auto dilations = Strides{1, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Strides has zero dimension")); +} + +TEST_F(TypePropConvolutionV1Test, dilations_has_zeros) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{0, 1}; + + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, empty_pad, empty_pad, dilations, op::PadType::SAME_LOWER), + NodeValidationFailure, + HasSubstr("Filter dilations has zero dimension")); +} - const auto &input_shape = ov::PartialShape::dynamic(), filters_shape = ov::PartialShape{1, 1, 3, 3}; - const auto& input_shapes = std::vector{input_shape, filters_shape}; - std::vector output_shapes(1); - auto pad_begin = CoordinateDiff{}, pad_end = CoordinateDiff{}; +TEST_F(TypePropConvolutionV1Test, pads_not_defined_for_spatial_only) { + const auto data = make_shared(element::f32, PartialShape{2, 3, 5, 5}); + const auto filters = make_shared(element::f32, PartialShape{2, 3, 4, 4}); + const auto strides = Strides{1, 1}; + const auto dilations = Strides{1, 1}; + const auto pads_begin = CoordinateDiff{2, 2}; + const auto pads_end = CoordinateDiff{2, 2, 2}; - int64_t num_spatial = calculate_num_spatial(conv.get(), input_shape, filters_shape, 2, 2); - update_and_validate_attributes(conv.get(), num_spatial); - EXPECT_NO_THROW(shape_infer(conv.get(), pad_begin, pad_end, input_shapes, output_shapes)); + OV_EXPECT_THROW(auto op = make_op(data, filters, strides, pads_begin, pads_end, dilations), + NodeValidationFailure, + HasSubstr("Pads begin and end should be defined for all and only spatial dimensions.")); } diff --git a/src/core/tests/type_prop/convolution_backprop_data.cpp b/src/core/tests/type_prop/convolution_backprop_data.cpp index 74d95b827005f2..8d6babfed0135d 100644 --- a/src/core/tests/type_prop/convolution_backprop_data.cpp +++ b/src/core/tests/type_prop/convolution_backprop_data.cpp @@ -21,6 +21,7 @@ using namespace std; using namespace ngraph; +using namespace testing; // ---------------------------- v1 ---------------------------- TEST(type_prop, convolution_backprop_data_partial_auto_padding_upper) { @@ -72,9 +73,10 @@ TEST(type_prop, convolution_backprop_data_partial_auto_padding_lower) { } TEST(type_prop, convolution_backprop_data_auto_pad_explicit_with_output_padding) { - const PartialShape data_pshape{1, 16, 2, 2}; - const PartialShape filters_pshape{16, 6, 3, 3}; - + PartialShape data_pshape{1, 16, 2, 2}; + PartialShape filters_pshape{16, 6, 3, 3}; + set_shape_labels(data_pshape, 10); + set_shape_labels(filters_pshape, 20); const Strides strides{2, 2}; const Strides dilations{1, 1}; const CoordinateDiff padding_begin{1, 1}; @@ -94,6 +96,8 @@ TEST(type_prop, convolution_backprop_data_auto_pad_explicit_with_output_padding) auto_pad, output_padding); + EXPECT_THAT(get_shape_labels(conv_backprop->get_output_partial_shape(0)), + ElementsAre(10, 21, ov::no_label, ov::no_label)); ASSERT_EQ(conv_backprop->get_output_partial_shape(0), PartialShape(PartialShape{1, 6, 4, 4})); ASSERT_EQ(conv_backprop->get_pads_begin(), (CoordinateDiff{1, 1})); ASSERT_EQ(conv_backprop->get_pads_end(), (CoordinateDiff{1, 1})); @@ -125,10 +129,10 @@ TEST(type_prop, convolution_backprop_data_auto_pad_same_with_output_padding_and_ auto_pad, output_padding); - ASSERT_EQ(conv_backprop->get_output_partial_shape(0), PartialShape(PartialShape{1, 6, 3, 3})); - ASSERT_EQ(conv_backprop->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(conv_backprop->get_pads_end(), (CoordinateDiff{2, 2})); - ASSERT_EQ(conv_backprop->get_output_padding(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv_backprop->get_output_partial_shape(0), PartialShape(PartialShape{1, 6, 3, 3})); + EXPECT_EQ(conv_backprop->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(conv_backprop->get_pads_end(), (CoordinateDiff{2, 2})); + EXPECT_EQ(conv_backprop->get_output_padding(), (CoordinateDiff{1, 1})); } TEST(type_prop, convolution_backprop_data_output_shape_as_const) { @@ -222,8 +226,10 @@ TEST(type_prop, convolution_backprop_data_with_output_shape_dyn_static_ranks_fil } TEST(type_prop, convolution_backprop_data_with_output_shape_dyn_static_ranks_filters_cin_cout_dyn) { - const PartialShape data_pshape{Dimension::dynamic(), 16, 5, 5}; - const PartialShape filters_pshape{Dimension::dynamic(), Dimension::dynamic(), 3, 3}; + PartialShape data_pshape{Dimension::dynamic(), 16, 5, 5}; + PartialShape filters_pshape{Dimension::dynamic(), Dimension::dynamic(), 3, 3}; + set_shape_labels(data_pshape, 10); + set_shape_labels(filters_pshape, 20); const element::Type_t et = element::f32; auto data = make_shared(et, data_pshape); @@ -238,6 +244,8 @@ TEST(type_prop, convolution_backprop_data_with_output_shape_dyn_static_ranks_fil Strides{}, op::PadType::SAME_UPPER); + EXPECT_THAT(get_shape_labels(conv_backprop->get_output_partial_shape(0)), + ElementsAre(10, 21, ov::no_label, ov::no_label)); ASSERT_EQ(conv_backprop->get_output_partial_shape(0), PartialShape(PartialShape{Dimension::dynamic(), Dimension::dynamic(), 3, 3})); } @@ -548,7 +556,7 @@ TEST(type_prop, convolution_backprop_data_invalid_input_ranks) { Strides{}); FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Data and filters rank do not match"); + EXPECT_HAS_SUBSTRING(error.what(), "Data batch and filters rank do not match"); } catch (...) { FAIL() << "Rank validation check of inputs failed for unexpected reason"; } @@ -568,7 +576,7 @@ TEST(type_prop, convolution_backprop_data_invalid_input_ranks) { Strides{}); FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Data and filters inputs must have rank 3, 4 or 5"); + EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D, 4D or 5D tensor for the input. Got:"); } catch (...) { FAIL() << "Rank validation check of inputs failed for unexpected reason"; } @@ -588,7 +596,7 @@ TEST(type_prop, convolution_backprop_data_invalid_input_ranks) { Strides{}); FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Data and filters inputs must have rank 3, 4 or 5"); + EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D, 4D or 5D tensor for the input. Got:"); } catch (...) { FAIL() << "Rank validation check of inputs failed for unexpected reason"; } @@ -633,8 +641,9 @@ TEST(type_prop, convolution_backprop_data_invalid_input_channel_dims) { // data input shape does not have correct dimension C_IN FAIL() << "Incompatibile input shapes not detected."; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - std::string("Input channels dimension of data and filters inputs must be equal")); + EXPECT_HAS_SUBSTRING( + error.what(), + std::string("Data batch channel count (32) does not match filter input channel count (16)")); } catch (...) { FAIL() << "Input shapes validation check failed for unexpected reason."; } @@ -656,11 +665,11 @@ TEST(type_prop, convolution_backprop_data_invalid_output_shape_spatial_dims) { CoordinateDiff{}, CoordinateDiff{}, Strides{}); - // output_shape has invalid spatials dimensions (should be 2) + // output_shape has invalid spatial dimensions (should be 2) FAIL() << "Incompatible output shape optional input not detected"; } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING(error.what(), - std::string("Output shape should be specified only and for all spatial dimensions.")); + std::string("Output shape should be defined for all and only spatial dimensions.")); } catch (...) { FAIL() << "Output shape validation check failed for unexpected reason."; } @@ -752,7 +761,7 @@ TEST(type_prop, convolution_backprop_data_invalid_conv_param_spatial_dims) { make_shared(data, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads begin should be defined for all and only spatial dimensions."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -768,7 +777,7 @@ TEST(type_prop, convolution_backprop_data_invalid_conv_param_spatial_dims) { make_shared(data, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads end should be defined for all and only spatial dimensions."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -825,16 +834,59 @@ TEST(type_prop, convolution_backprop_data_invalid_conv_param_spatial_dims) { } TEST(type_prop, convolution_back_prop_data_default_constructed) { - auto conv = make_shared(); - - const auto &input_shape = ov::PartialShape::dynamic(), filters_shape = ov::PartialShape{1, 1, 3, 3}, - output_spatial_shape_shape = ov::PartialShape({2}); - const auto& input_shapes = std::vector{input_shape, filters_shape, output_spatial_shape_shape}; - std::vector output_shapes = {ov::PartialShape::dynamic()}; - auto pad_begin = CoordinateDiff{}, pad_end = CoordinateDiff{}; - const auto& output_spatial_shape = ov::PartialShape{3, 3}; - int64_t num_spatial = - calculate_num_spatial(conv.get(), input_shape, filters_shape, output_spatial_shape_shape, 2, 2); - update_and_validate_attributes_back_prop(conv.get(), num_spatial); - EXPECT_NO_THROW(shape_infer(conv.get(), pad_begin, pad_end, output_spatial_shape, input_shapes, output_shapes)); + const auto data = make_shared(element::f32, PartialShape::dynamic()); + const auto filters = make_shared(element::f32, PartialShape{1, 1, 1, 3, 3}); + const auto out_spatial = op::Constant::create(element::i32, Shape{3}, {5, 4, 10}); + + const auto op = make_shared(); + op->set_arguments(OutputVector{data, filters, out_spatial}); + op->set_strides({1, 1, 1}); + op->set_dilations({1, 1, 1}); + op->set_pads_begin({2, 2, 2}); + op->set_pads_end({2, 2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 3); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 1, 1})); + EXPECT_EQ(op->get_dilations(), Strides({1, 1, 1})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2, 2})); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 1, 5, 4, 10})); +} + +TEST(type_prop, convolution_back_prop_data_interval_shapes_output_shape_as_shape_of) { + PartialShape data_pshape{{1, 3}, {2, 6}, {1, 5}, {3, 10}, {20, 100}}; + PartialShape filters_pshape{{2, 3}, {1, 3}, 3, 3, 3}; + PartialShape out_spatial_pshape{3, {2, 4}, 3}; + + set_shape_labels(data_pshape, 10); + set_shape_labels(filters_pshape, 20); + set_shape_labels(out_spatial_pshape, 30); + + const element::Type_t et = element::f32; + Strides strides{1, 2, 1}; + Strides dilations{1, 1, 1}; + CoordinateDiff pads_begin{0, 2, 1}; + CoordinateDiff pads_end{0, 0, 0}; + const auto auto_pad = op::PadType::SAME_LOWER; + + auto data_batch = make_shared(et, data_pshape); + auto filters = make_shared(et, filters_pshape); + auto out_spatial = make_shared(element::i32, out_spatial_pshape); + auto spatial_shape_of = std::make_shared(out_spatial); + + const auto op = make_shared(data_batch, + filters, + spatial_shape_of, + strides, + pads_begin, + pads_end, + dilations, + auto_pad); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(10, 21, 30, 31, 32)); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({{1, 3}, {1, 3}, 3, {2, 4}, 3})); + EXPECT_EQ(op->get_pads_begin(), (CoordinateDiff{0, 0, 0})); + EXPECT_EQ(op->get_pads_end(), (CoordinateDiff{0, 0, 0})); } diff --git a/src/core/tests/type_prop/deformable_convolution.cpp b/src/core/tests/type_prop/deformable_convolution.cpp index 4d48704a0564ea..7ebe9c6d3d41ea 100644 --- a/src/core/tests/type_prop/deformable_convolution.cpp +++ b/src/core/tests/type_prop/deformable_convolution.cpp @@ -8,11 +8,15 @@ using namespace std; using namespace ngraph; +using namespace testing; TEST(type_prop, deformable_convolution_partial_auto_padding_same) { - const PartialShape data_batch_pshape{1, 4, 5, 5}; - const PartialShape offsets_pshape{1, 36, 5, 5}; - const PartialShape filters_pshape{4, 1, 3, 3}; + PartialShape data_batch_pshape{1, 4, 5, 5}; + PartialShape offsets_pshape{1, 36, 5, 5}; + PartialShape filters_pshape{4, 1, 3, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(offsets_pshape, 20); + set_shape_labels(filters_pshape, 30); const element::Type_t et = element::f32; Strides strides{1, 1}; @@ -37,9 +41,11 @@ TEST(type_prop, deformable_convolution_partial_auto_padding_same) { group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_THAT(get_shape_labels(deformable_conv->get_output_partial_shape(0)), + ElementsAre(10, 30, ov::no_label, ov::no_label)); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, deformable_convolution_partial_auto_padding_same_lower_data_batch_nc_dims_dynamic) { @@ -70,9 +76,9 @@ TEST(type_prop, deformable_convolution_partial_auto_padding_same_lower_data_batc group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 4, 5, 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{Dimension::dynamic(), 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, deformable_convolution_partial_auto_padding_same_upper_data_batch_nc_dims_dynamic) { @@ -103,9 +109,9 @@ TEST(type_prop, deformable_convolution_partial_auto_padding_same_upper_data_batc group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, deformable_convolution_partial_auto_padding_same_spatial_dims_dynamic) { @@ -136,15 +142,17 @@ TEST(type_prop, deformable_convolution_partial_auto_padding_same_spatial_dims_dy group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme({1, 4, Dimension::dynamic(), 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, Dimension::dynamic(), 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 1})); } TEST(type_prop, deformable_convolution_data_batch_dynamic) { - const PartialShape data_batch_pshape{PartialShape::dynamic()}; - const PartialShape offsets_pshape{2, 36, 5, 5}; - const PartialShape filters_pshape{4, 4, 3, 3}; + PartialShape data_batch_pshape{PartialShape::dynamic()}; + PartialShape offsets_pshape{2, 36, 5, 5}; + PartialShape filters_pshape{4, 4, 3, 3}; + set_shape_labels(offsets_pshape, 20); + set_shape_labels(filters_pshape, 30); const element::Type_t et = element::f32; const auto auto_pad = op::PadType::EXPLICIT; @@ -165,13 +173,14 @@ TEST(type_prop, deformable_convolution_data_batch_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( - PartialShape{2, 4, Dimension::dynamic(), Dimension::dynamic()})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{2, 4, {1, -1}, {1, -1}})); + EXPECT_THAT(get_shape_labels(deformable_conv->get_output_partial_shape(0)), + ElementsAre(20, 30, ov::no_label, ov::no_label)); } TEST(type_prop, deformable_convolution_offsets_dynamic) { @@ -198,17 +207,17 @@ TEST(type_prop, deformable_convolution_offsets_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); } TEST(type_prop, deformable_convolution_auto_pad_same_filters_dynamic) { const PartialShape data_batch_pshape{1, 4, 5, 5}; - const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape offsets_pshape{1, 36, 5, 5}; const PartialShape filters_pshape{PartialShape::dynamic()}; const element::Type_t et = element::f32; @@ -230,13 +239,12 @@ TEST(type_prop, deformable_convolution_auto_pad_same_filters_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_UPPER); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( - PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_UPPER); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, Dimension::dynamic(), 5, 5})); } TEST(type_prop, deformable_convolution_deformable_data_batch_and_filters_dynamic) { @@ -263,13 +271,12 @@ TEST(type_prop, deformable_convolution_deformable_data_batch_and_filters_dynamic group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( - PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, -1, {1, -1}, {1, -1}})); } TEST(type_prop, deformable_convolution_deformable_all_inputs_dynamic) { @@ -294,12 +301,12 @@ TEST(type_prop, deformable_convolution_deformable_all_inputs_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape::dynamic())); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape::dynamic())); } TEST(type_prop, deformable_convolution_invalid_et_inputs) { @@ -324,9 +331,9 @@ TEST(type_prop, deformable_convolution_invalid_et_inputs) { // data batch input must be of same element type as filters and deformable values FAIL() << "Invalid element type of inputs not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Element types of inputs do not match. Got: data batch (f16), " - "offsets (f32) and filters (f32)"); + EXPECT_HAS_SUBSTRING( + error.what(), + "Element types of inputs do not match. Got: data batch (f16), offsets (f32) and filters (f32)"); } catch (...) { FAIL() << "Element types of inputs validation check failed for unexpected reason."; } @@ -416,10 +423,7 @@ TEST(type_prop, deformable_convolution_invalid_input_ranks) { // data batch has invalid rank 5, should be 4 FAIL() << "Incompatible data batch input rank not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Ranks of inputs do not match. Got: data batch " - "shape [1,4,5,5,5], offsets shape [1,4,4,4], filters " - "shape [4,4,3,3]"); + EXPECT_HAS_SUBSTRING(error.what(), "Input must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check of data batch input failed for unexpected reason"; } @@ -443,10 +447,7 @@ TEST(type_prop, deformable_convolution_invalid_input_ranks) { // deformable values has invalid rank 5, should be 4 FAIL() << "Incompatible offsets input rank not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Ranks of inputs do not match. Got: data batch shape " - "[1,4,5,5], offsets shape [1,4,4,4,4], filters shape " - "[4,4,3,3]"); + EXPECT_HAS_SUBSTRING(error.what(), "Offsets must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check of offsets input failed for unexpected reason"; } @@ -470,10 +471,7 @@ TEST(type_prop, deformable_convolution_invalid_input_ranks) { // filters has invalid rank 5, should be 4 FAIL() << "Incompatible filter input rank not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Ranks of inputs do not match. Got: data batch shape " - "[1,4,5,5], offsets shape [1,4,4,4], filters shape " - "[4,4,3,3,3]"); + EXPECT_HAS_SUBSTRING(error.what(), "Filters must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check of filter input failed for unexpected reason"; } @@ -497,7 +495,7 @@ TEST(type_prop, deformable_convolution_invalid_input_ranks) { // inputs have rank 5, should be 4 FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Inputs must be of rank 4"); + EXPECT_HAS_SUBSTRING(error.what(), "Input must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check for 2 spatial dimension inputs failed for unexpected reason"; } @@ -521,7 +519,7 @@ TEST(type_prop, deformable_convolution_invalid_input_ranks) { // inputs have rank 3, should be 4 FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Inputs must be of rank 4"); + EXPECT_HAS_SUBSTRING(error.what(), "Input must be of rank 4. Got: 3"); } catch (...) { FAIL() << "Rank validation check for 2 spatial dimension inputs failed for unexpected reason"; } @@ -679,9 +677,7 @@ TEST(type_prop, deformable_convolution_invalid_offsets_channels_dim) { FAIL() << "Invalid channels dimension of offsets input not detected"; } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING(error.what(), - "The channels dimension of offsets input must be " - "evenly divisible by the 'deformable group' value along the " - "channels axis."); + "Offsets channels dimension (35) must be evenly divisible by the 'deformable group': 2"); } catch (...) { FAIL() << "Channels dimension of offsets input validation check failed for " "unexpected reason."; @@ -759,9 +755,7 @@ TEST(type_prop, deformable_convolution_invalid_data_batch_channels_dim_with_grou // data batch channels is not evenly divisible by the attribute group value FAIL() << "Invalid channels dimension of data batch input not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "The input data shape must be evenly divisible by the 'group' value " - "along the channels axis."); + EXPECT_HAS_SUBSTRING(error.what(), "Input channels dimension (5) must be evenly divisible by the 'group': 4"); } catch (...) { FAIL() << "Data batch channel dimension validation check failed for unexpected " "reason."; @@ -800,9 +794,7 @@ TEST(type_prop, deformable_convolution_invalid_filters_channels_dim_with_group) // filters channels output is not evenly divisible by the attribute group value FAIL() << "Invalid channels output dimension of filters input not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "The filters shape must be evenly divisible by the 'group' value along " - "the channels axis"); + EXPECT_HAS_SUBSTRING(error.what(), "Filters channels dimension (5) must be evenly divisible by the 'group'"); } catch (...) { FAIL() << "Filters channels output dimension validation check failed for unexpected " "reason."; @@ -882,7 +874,7 @@ TEST(type_prop, deformable_convolution_invalid_offsets_spatial_dims) { // deformable values has incorrect spatial dimensions FAIL() << "Invalid spatial dimensions of offsets not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of offsets and output must be equal"); + EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of offsets and output must be compatible"); } catch (...) { FAIL() << "Spatial dimension of offsets validation check failed for unexpected reason"; } @@ -913,7 +905,7 @@ TEST(type_prop, deformable_convolution_invalid_conv_param_spatial_dims) { dilations); FAIL() << "Invalid strides spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial dimension"); } catch (...) { FAIL() << "Strides spatial dimensions validation check failed for unexpected reason"; } @@ -935,7 +927,7 @@ TEST(type_prop, deformable_convolution_invalid_conv_param_spatial_dims) { dilations); FAIL() << "Invalid strides spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial dimension"); } catch (...) { FAIL() << "Strides spatial dimensions validation check failed for unexpected reason"; } @@ -959,7 +951,7 @@ TEST(type_prop, deformable_convolution_invalid_conv_param_spatial_dims) { dilations); FAIL() << "Invalid dilations spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial dimensions"); } catch (...) { FAIL() << "Dilations spatial dimensions validation check failed for unexpected reason"; } @@ -981,7 +973,7 @@ TEST(type_prop, deformable_convolution_invalid_conv_param_spatial_dims) { dilations); FAIL() << "Invalid dilations spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial dimensions"); } catch (...) { FAIL() << "Dilations spatial dimensions validation check failed for unexpected reason"; } @@ -1005,7 +997,7 @@ TEST(type_prop, deformable_convolution_invalid_conv_param_spatial_dims) { dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions"); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -1027,8 +1019,61 @@ TEST(type_prop, deformable_convolution_invalid_conv_param_spatial_dims) { dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions"); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } } + +class TypePropDeformableConvolutionV1Test : public TypePropOpTest { +protected: + CoordinateDiff empty_pad{}; +}; + +TEST_F(TypePropDeformableConvolutionV1Test, default_ctor) { + const auto data = make_shared(element::f32, PartialShape{1, 4, 5, 5}); + const auto offsets = make_shared(element::f32, PartialShape{1, 36, 7, 2}); + const auto filters = make_shared(element::f32, PartialShape{4, 1, 3, 3}); + + const auto op = make_op(); + op->set_arguments(OutputVector{data, offsets, filters}); + op->set_strides({1, 3}); + op->set_dilations({1, 2}); + op->set_pads_begin({2, 2}); + op->set_pads_end({2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->set_group(4); + op->set_deformable_group(2); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 3); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 3})); + EXPECT_EQ(op->get_dilations(), Strides({1, 2})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2})); + + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({1, 4, 7, 2})); +} + +TEST_F(TypePropDeformableConvolutionV1Test, interval_shapes) { + PartialShape data_batch_pshape{{1, 3}, {2, 6}, {1, 5}, {3, 10}}; + PartialShape offsets_shape{1, 36, 4, 5}; + PartialShape filters_pshape{{2, 5}, {1, 3}, {2, 3}, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(offsets_shape, 20); + set_shape_labels(filters_pshape, 30); + + const element::Type_t et = element::f32; + const auto auto_pad = op::PadType::EXPLICIT; + + const auto data_batch = make_shared(et, data_batch_pshape); + const auto offsets = make_shared(et, offsets_shape); + const auto filters = make_shared(et, filters_pshape); + const auto op = make_op(data_batch, offsets, filters, Strides{}, empty_pad, empty_pad, Strides{}, auto_pad, 4, 2); + + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(10, 30, ov::no_label, ov::no_label)); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({1, {2, 5}, {1, 4}, {1, 8}})); + EXPECT_EQ(op->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(op->get_pads_end(), (CoordinateDiff{0, 0})); +} diff --git a/src/core/tests/type_prop/deformable_convolution_opset8.cpp b/src/core/tests/type_prop/deformable_convolution_opset8.cpp index 1a7f977cda105e..f48a7b462124aa 100644 --- a/src/core/tests/type_prop/deformable_convolution_opset8.cpp +++ b/src/core/tests/type_prop/deformable_convolution_opset8.cpp @@ -4,17 +4,21 @@ #include "gtest/gtest.h" #include "ngraph/ngraph.hpp" -#include "ngraph/opsets/opset8.hpp" +#include "openvino/opsets/opset8.hpp" #include "util/type_prop.hpp" using namespace std; using namespace ngraph; -using namespace ngraph::opset8; +using namespace ov::opset8; +using namespace testing; TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same) { - const PartialShape data_batch_pshape{1, 4, 5, 5}; - const PartialShape offsets_pshape{1, 36, 5, 5}; - const PartialShape filters_pshape{4, 1, 3, 3}; + PartialShape data_batch_pshape{1, 4, 5, 5}; + PartialShape offsets_pshape{1, 36, 5, 5}; + PartialShape filters_pshape{4, 1, 3, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(offsets_pshape, 20); + set_shape_labels(filters_pshape, 30); const element::Type_t et = element::f32; Strides strides{1, 1}; @@ -39,9 +43,11 @@ TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same) { group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_THAT(get_shape_labels(deformable_conv->get_output_partial_shape(0)), + ElementsAre(10, 30, ov::no_label, ov::no_label)); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_lower_data_batch_nc_dims_dynamic) { @@ -72,9 +78,9 @@ TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_lower_da group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 4, 5, 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{Dimension::dynamic(), 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_upper_data_batch_nc_dims_dynamic) { @@ -105,9 +111,9 @@ TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_upper_da group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_spatial_dims_dynamic) { @@ -138,9 +144,9 @@ TEST(type_prop, deformable_convolution_opset8_partial_auto_padding_same_spatial_ group, deformable_group); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme({1, 4, Dimension::dynamic(), 5})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, Dimension::dynamic(), 5})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 1})); } TEST(type_prop, deformable_convolution_opset8_data_batch_dynamic) { @@ -167,13 +173,12 @@ TEST(type_prop, deformable_convolution_opset8_data_batch_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( - PartialShape{2, 4, Dimension::dynamic(), Dimension::dynamic()})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{2, 4, {1, -1}, {1, -1}})); } TEST(type_prop, deformable_convolution_opset8_offsets_dynamic) { @@ -200,17 +205,17 @@ TEST(type_prop, deformable_convolution_opset8_offsets_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); } TEST(type_prop, deformable_convolution_opset8_auto_pad_same_filters_dynamic) { const PartialShape data_batch_pshape{1, 4, 5, 5}; - const PartialShape offsets_pshape{1, 36, 3, 3}; + const PartialShape offsets_pshape{1, 36, 5, 5}; const PartialShape filters_pshape{PartialShape::dynamic()}; const element::Type_t et = element::f32; @@ -232,13 +237,12 @@ TEST(type_prop, deformable_convolution_opset8_auto_pad_same_filters_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_UPPER); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( - PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_UPPER); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, Dimension::dynamic(), 5, 5})); } TEST(type_prop, deformable_convolution_opset8_deformable_data_batch_and_filters_dynamic) { @@ -265,11 +269,11 @@ TEST(type_prop, deformable_convolution_opset8_deformable_data_batch_and_filters_ group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{0, 0})); ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme( PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); } @@ -296,11 +300,11 @@ TEST(type_prop, deformable_convolution_opset8_deformable_all_inputs_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::EXPLICIT); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{})); ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape::dynamic())); } @@ -418,10 +422,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_input_ranks) { // data batch has invalid rank 5, should be 4 FAIL() << "Incompatible data batch input rank not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Ranks of inputs do not match. Got: data batch " - "shape [1,4,5,5,5], offsets shape [1,4,4,4], filters " - "shape [4,4,3,3]"); + EXPECT_HAS_SUBSTRING(error.what(), "Input must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check of data batch input failed for unexpected reason"; } @@ -445,10 +446,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_input_ranks) { // deformable values has invalid rank 5, should be 4 FAIL() << "Incompatible offsets input rank not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Ranks of inputs do not match. Got: data batch shape " - "[1,4,5,5], offsets shape [1,4,4,4,4], filters shape " - "[4,4,3,3]"); + EXPECT_HAS_SUBSTRING(error.what(), "Offsets must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check of offsets input failed for unexpected reason"; } @@ -472,10 +470,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_input_ranks) { // filters has invalid rank 5, should be 4 FAIL() << "Incompatible filter input rank not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Ranks of inputs do not match. Got: data batch shape " - "[1,4,5,5], offsets shape [1,4,4,4], filters shape " - "[4,4,3,3,3]"); + EXPECT_HAS_SUBSTRING(error.what(), "Filters must be of rank 4. Got: 5"); } catch (...) { FAIL() << "Rank validation check of filter input failed for unexpected reason"; } @@ -499,7 +494,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_input_ranks) { // inputs have rank 5, should be 4 FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Inputs must be of rank 4"); + EXPECT_HAS_SUBSTRING(error.what(), "Input must be of rank 4"); } catch (...) { FAIL() << "Rank validation check for 2 spatial dimension inputs failed for unexpected reason"; } @@ -523,7 +518,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_input_ranks) { // inputs have rank 3, should be 4 FAIL() << "Incompatible input ranks not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Inputs must be of rank 4"); + EXPECT_HAS_SUBSTRING(error.what(), "Input must be of rank 4"); } catch (...) { FAIL() << "Rank validation check for 2 spatial dimension inputs failed for unexpected reason"; } @@ -681,9 +676,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_offsets_channels_dim) { FAIL() << "Invalid channels dimension of offsets input not detected"; } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING(error.what(), - "The channels dimension of offsets input must be " - "evenly divisible by the 'deformable group' value along the " - "channels axis."); + "Offsets channels dimension (35) must be evenly divisible by the 'deformable group'"); } catch (...) { FAIL() << "Channels dimension of offsets input validation check failed for " "unexpected reason."; @@ -761,9 +754,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_data_batch_channels_dim_wi // data batch channels is not evenly divisible by the attribute group value FAIL() << "Invalid channels dimension of data batch input not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "The input data shape must be evenly divisible by the 'group' value " - "along the channels axis."); + EXPECT_HAS_SUBSTRING(error.what(), "Input channels dimension (5) must be evenly divisible by the 'group'"); } catch (...) { FAIL() << "Data batch channel dimension validation check failed for unexpected " "reason."; @@ -802,9 +793,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_filters_channels_dim_with_ // filters channels output is not evenly divisible by the attribute group value FAIL() << "Invalid channels output dimension of filters input not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "The filters shape must be evenly divisible by the 'group' value along " - "the channels axis"); + EXPECT_HAS_SUBSTRING(error.what(), "Filters channels dimension (5) must be evenly divisible by the 'group'"); } catch (...) { FAIL() << "Filters channels output dimension validation check failed for unexpected " "reason."; @@ -884,7 +873,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_offsets_spatial_dims) { // deformable values has incorrect spatial dimensions FAIL() << "Invalid spatial dimensions of offsets not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of offsets and output must be equal"); + EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of offsets and output must be compatible"); } catch (...) { FAIL() << "Spatial dimension of offsets validation check failed for unexpected reason"; } @@ -910,7 +899,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) { make_shared(data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid strides spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Strides spatial dimensions validation check failed for unexpected reason"; } @@ -927,7 +916,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) { make_shared(data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid strides spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Strides should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Strides spatial dimensions validation check failed for unexpected reason"; } @@ -946,7 +935,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) { make_shared(data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid dilations spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Dilations spatial dimensions validation check failed for unexpected reason"; } @@ -963,7 +952,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) { make_shared(data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid dilations spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Dilations should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Dilations spatial dimensions validation check failed for unexpected reason"; } @@ -982,7 +971,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) { make_shared(data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -999,7 +988,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_conv_param_spatial_dims) { make_shared(data_batch, offsets, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads should be defined for all and only spatial features."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -1040,7 +1029,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_mask_spatial_dims) { // deformable values has incorrect spatial dimensions FAIL() << "Invalid spatial dimensions of mask not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of mask and output must be equal"); + EXPECT_HAS_SUBSTRING(error.what(), "Spatial dimensions of mask and output must be compatible"); } catch (...) { FAIL() << "Spatial dimension of mask validation check failed for unexpected reason"; } @@ -1073,12 +1062,12 @@ TEST(type_prop, deformable_convolution_opset8_mask_dynamic) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); - ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_output_partial_shape(0), (PartialShape{1, 4, 5, 5})); } TEST(type_prop, deformable_convolution_opset8_invalid_mask_channels_dim) { @@ -1161,9 +1150,7 @@ TEST(type_prop, deformable_convolution_opset8_invalid_mask_channels_dim) { FAIL() << "Invalid channels dimension of mask input not detected"; } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING(error.what(), - "The channels dimension of mask input must be " - "evenly divisible by the 'deformable group' value along the " - "channels axis."); + "Mask channels dimension (9) must be evenly divisible by the 'deformable group'"); } catch (...) { FAIL() << "Channels dimension of mask input validation check failed for " "unexpected reason."; @@ -1239,10 +1226,66 @@ TEST(type_prop, deformable_convolution_opset8_mask) { group, deformable_group); - ASSERT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); - ASSERT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); - ASSERT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_auto_pad(), op::PadType::SAME_LOWER); + EXPECT_EQ(deformable_conv->get_strides(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_dilations(), (Strides{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_begin(), (CoordinateDiff{1, 1})); + EXPECT_EQ(deformable_conv->get_pads_end(), (CoordinateDiff{1, 1})); ASSERT_TRUE(deformable_conv->get_output_partial_shape(0).same_scheme(PartialShape{1, 4, 5, 5})); } + +class TypePropDeformableConvolutionV8Test : public TypePropOpTest { +protected: + CoordinateDiff empty_pad{}; +}; + +TEST_F(TypePropDeformableConvolutionV8Test, default_ctor) { + const auto data = make_shared(element::f32, PartialShape{1, 4, 5, 5}); + const auto offsets = make_shared(element::f32, PartialShape{1, 36, 7, 2}); + const auto filters = make_shared(element::f32, PartialShape{4, 1, 3, 3}); + const auto masks = make_shared(element::f32, PartialShape{1, 18, 7, -1}); + + const auto op = make_op(); + op->set_arguments(OutputVector{data, offsets, filters, masks}); + op->set_strides({1, 3}); + op->set_dilations({1, 2}); + op->set_pads_begin({2, 2}); + op->set_pads_end({2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->set_group(4); + op->set_deformable_group(2); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 4); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 3})); + EXPECT_EQ(op->get_dilations(), Strides({1, 2})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2})); + + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({1, 4, 7, 2})); +} + +TEST_F(TypePropDeformableConvolutionV8Test, interval_shapes) { + PartialShape data_batch_pshape{{1, 3}, {2, 6}, {1, 5}, {3, 10}}; + PartialShape offsets_shape{1, 36, 4, 5}; + PartialShape filters_pshape{{2, 5}, {1, 3}, {2, 3}, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(offsets_shape, 20); + set_shape_labels(filters_pshape, 30); + + const element::Type_t et = element::f32; + const auto auto_pad = op::PadType::EXPLICIT; + + const auto data_batch = make_shared(et, data_batch_pshape); + const auto offsets = make_shared(et, offsets_shape); + const auto filters = make_shared(et, filters_pshape); + const auto masks = make_shared(element::f32, PartialShape{-1, 18, {1, 10}, 3}); + const auto op = + make_op(data_batch, offsets, filters, masks, Strides{}, empty_pad, empty_pad, Strides{}, auto_pad, 4, 2); + + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(10, 30, ov::no_label, ov::no_label)); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({1, {2, 5}, {1, 4}, {1, 8}})); + EXPECT_EQ(op->get_pads_begin(), (CoordinateDiff{0, 0})); + EXPECT_EQ(op->get_pads_end(), (CoordinateDiff{0, 0})); +} diff --git a/src/core/tests/type_prop/group_convolution.cpp b/src/core/tests/type_prop/group_convolution.cpp index 62c3c028cd453e..76d206df4c8454 100644 --- a/src/core/tests/type_prop/group_convolution.cpp +++ b/src/core/tests/type_prop/group_convolution.cpp @@ -2,17 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "convolution_shape_inference.hpp" +#include "common_test_utils/test_assertions.hpp" #include "gtest/gtest.h" #include "ngraph/ngraph.hpp" #include "util/type_prop.hpp" using namespace std; using namespace ngraph; +using namespace testing; TEST(type_prop, group_convolution_auto_padding_same_lower) { - const PartialShape data_batch_pshape{1, 4, 5, 5}; - const PartialShape filters_pshape{2, 1, 2, 3, 3}; + PartialShape data_batch_pshape{1, 4, 5, 5}; + PartialShape filters_pshape{2, 1, 2, 3, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(filters_pshape, 20); element::Type_t et = element::f32; Strides strides{1, 1}; CoordinateDiff pads_begin{0, 0}; @@ -26,6 +29,8 @@ TEST(type_prop, group_convolution_auto_padding_same_lower) { auto groupConv = make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations, auto_pad); + EXPECT_THAT(get_shape_labels(groupConv->get_output_partial_shape(0)), + ElementsAre(10, 20, ov::no_label, ov::no_label)); ASSERT_EQ(groupConv->get_output_partial_shape(0), PartialShape({1, 2, 5, 5})); ASSERT_EQ(groupConv->get_pads_begin(), (CoordinateDiff{1, 1})); ASSERT_EQ(groupConv->get_pads_end(), (CoordinateDiff{1, 1})); @@ -74,8 +79,10 @@ TEST(type_prop, group_convolution_auto_padding_same_lower_spatial_dims_static) { } TEST(type_prop, group_convolution_auto_padding_same_upper_spatial_dims_static) { - const PartialShape data_batch_pshape{1, Dimension::dynamic(), 5, 5}; - const PartialShape filters_pshape{Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic(), 2, 2}; + PartialShape data_batch_pshape{1, Dimension::dynamic(), 5, 5}; + PartialShape filters_pshape{Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic(), 2, 2}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(filters_pshape, 20); const element::Type_t et = element::f32; const auto auto_pad = op::PadType::SAME_UPPER; @@ -89,14 +96,19 @@ TEST(type_prop, group_convolution_auto_padding_same_upper_spatial_dims_static) { Strides{}, auto_pad); + EXPECT_THAT(get_shape_labels(groupConv->get_output_partial_shape(0)), + ElementsAre(10, ov::no_label, ov::no_label, ov::no_label)); ASSERT_EQ(groupConv->get_output_partial_shape(0), PartialShape({1, Dimension::dynamic(), 5, 5})); ASSERT_EQ(groupConv->get_pads_begin(), (CoordinateDiff{0, 0})); ASSERT_EQ(groupConv->get_pads_end(), (CoordinateDiff{1, 1})); } TEST(type_prop, group_convolution_static_ranks_filters_groups_dyn) { - const PartialShape data_batch_pshape{Dimension::dynamic(), 4, 5, 5}; - const PartialShape filters_pshape{Dimension::dynamic(), 1, 2, 3, 3}; + PartialShape data_batch_pshape{Dimension::dynamic(), 4, 5, 5}; + PartialShape filters_pshape{Dimension::dynamic(), 1, 2, 3, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(filters_pshape, 20); + const element::Type_t et = element::f32; const auto auto_pad = op::PadType::SAME_LOWER; @@ -109,7 +121,8 @@ TEST(type_prop, group_convolution_static_ranks_filters_groups_dyn) { CoordinateDiff{}, Strides{}, auto_pad); - + EXPECT_THAT(get_shape_labels(groupConv->get_output_partial_shape(0)), + ElementsAre(10, 20, ov::no_label, ov::no_label)); ASSERT_EQ(groupConv->get_output_partial_shape(0), PartialShape({Dimension::dynamic(), 2, 5, 5})); ASSERT_EQ(groupConv->get_pads_begin(), (CoordinateDiff{1, 1})); ASSERT_EQ(groupConv->get_pads_end(), (CoordinateDiff{1, 1})); @@ -348,50 +361,43 @@ TEST(type_prop, group_convolution_invalid_input_ranks) { } TEST(type_prop, group_convolution_invalid_input_channel_dims) { - try { + constexpr auto et = element::f32; + // data batch shape does not have correct dimension C_IN * GROUPS + { const PartialShape data_batch_pshape{1, 6, 5, 5}; - const PartialShape filters_pshape{2, 1, 2, 3, 3}; - element::Type_t et = element::f32; + const PartialShape filters_pshape{1, 1, 3, 3, 3}; auto data_batch = make_shared(et, data_batch_pshape); auto filters = make_shared(et, filters_pshape); - auto groupConv = make_shared(data_batch, - filters, - Strides{}, - CoordinateDiff{}, - CoordinateDiff{}, - Strides{}); - // data batch shape does not have correct dimension C_IN * GROUPS - FAIL() << "Invalid input channels dimension of data batch not detected."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "Input channels dimension of data batch has incompatible value " - "with filter shape."); - } catch (...) { - FAIL() << "Input channels dimension of data batch validation check failed for unexpected " - "reason."; + + OV_EXPECT_THROW( + const auto op = make_shared(data_batch, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}), + NodeValidationFailure, + HasSubstr("Input channels dimension of data batch is incompatible with filter groups or input channels.")); } - try { + // data batch shape does not have correct dimension C_IN * GROUPS + { const PartialShape data_batch_pshape{1, 3, 5, 5}; - const PartialShape filters_pshape{2, 1, Dimension::dynamic(), 3, 3}; - element::Type_t et = element::f32; + const PartialShape filters_pshape{-1, 1, 2, 3, 3}; auto data_batch = make_shared(et, data_batch_pshape); auto filters = make_shared(et, filters_pshape); - auto groupConv = make_shared(data_batch, - filters, - Strides{}, - CoordinateDiff{}, - CoordinateDiff{}, - Strides{}); - // data batch shape does not have correct dimension C_IN * GROUPS - FAIL() << "Invalid input channels dimension of data batch not detected."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Input channels dimension of data batch not a multiple of group size"); - } catch (...) { - FAIL() << "Input channels dimension of data batch validation check failed for unexpected " - "reason."; + + OV_EXPECT_THROW( + const auto op = make_shared(data_batch, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}), + NodeValidationFailure, + HasSubstr("Input channels dimension of data batch is incompatible with filter groups or input channels.")); } } @@ -469,23 +475,23 @@ TEST(type_prop, group_convolution_invalid_conv_param_spatial_dims) { } // invalid padding spatial dimensions - try { + { Strides strides{1, 1}; Strides dilations{1, 1}; CoordinateDiff pads_begin{0, 0, 0}; CoordinateDiff pads_end{0, 0}; - auto data_batch = make_shared(et, data_batch_pshape); - auto filters = make_shared(et, PartialShape::dynamic()); - auto groupConv = - make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations); - FAIL() << "Invalid padding spatial dimensions not detected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads begin should be defined for all and only spatial dimensions."); - } catch (...) { - FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; + auto data_batch = make_shared(et, PartialShape::dynamic()); + auto filters = make_shared(et, filters_pshape); + + OV_EXPECT_THROW( + auto op = + make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations), + NodeValidationFailure, + HasSubstr("Pads begin and end should be defined for all and only spatial dimensions.")); } - try { + + { Strides strides{1, 1}; Strides dilations{1, 1}; CoordinateDiff pads_begin{0, 0}; @@ -493,26 +499,58 @@ TEST(type_prop, group_convolution_invalid_conv_param_spatial_dims) { auto data_batch = make_shared(et, PartialShape::dynamic()); auto filters = make_shared(et, filters_pshape); - auto groupConv = - make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations); - FAIL() << "Invalid padding spatial dimensions not detected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads end should be defined for all and only spatial dimensions."); - } catch (...) { - FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; + + OV_EXPECT_THROW( + auto op = + make_shared(data_batch, filters, strides, pads_begin, pads_end, dilations), + NodeValidationFailure, + HasSubstr("Pads begin and end should be defined for all and only spatial dimensions.")); } } -TEST(type_prop, group_convolution_default_constructed) { - auto conv = make_shared(); - conv->set_auto_pad(op::PadType::SAME_LOWER); +TEST(type_prop, group_convolution_interval_shapes) { + PartialShape data_batch_pshape{{1, 3}, {2, 6}, {1, 5}, {3, 10}, {20, 100}}; + PartialShape filters_pshape{{2, 3}, {1, 3}, {2, 3}, 3, 3, 3}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(filters_pshape, 20); + + const element::Type_t et = element::f32; + const auto auto_pad = op::PadType::EXPLICIT; - const auto &input_shape = ov::PartialShape::dynamic(), filters_shape = ov::PartialShape{1, 1, 1, 3, 3}; - const auto& input_shapes = std::vector{input_shape, filters_shape}; - std::vector output_shapes(1); - auto pad_begin = CoordinateDiff{}, pad_end = CoordinateDiff{}; + auto data_batch = make_shared(et, data_batch_pshape); + auto filters = make_shared(et, filters_pshape); + auto groupConv = make_shared(data_batch, + filters, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad); + EXPECT_THAT(get_shape_labels(groupConv->get_output_partial_shape(0)), + ElementsAre(10, ov::no_label, ov::no_label, ov::no_label, ov::no_label)); + EXPECT_EQ(groupConv->get_output_partial_shape(0), PartialShape({{1, 3}, {2, 9}, {1, 3}, {1, 8}, {18, 98}})); + EXPECT_EQ(groupConv->get_pads_begin(), (CoordinateDiff{0, 0, 0})); + EXPECT_EQ(groupConv->get_pads_end(), (CoordinateDiff{0, 0, 0})); +} - int64_t num_spatial = calculate_num_spatial(conv.get(), input_shape, filters_shape, 2, 3); - update_and_validate_attributes(conv.get(), num_spatial); - EXPECT_NO_THROW(shape_infer(conv.get(), pad_begin, pad_end, input_shapes, output_shapes)); +TEST(type_prop, group_convolution_default_constructed) { + const auto data = make_shared(element::f32, PartialShape::dynamic()); + const auto filters = make_shared(element::f32, PartialShape{1, 1, 1, 3, 3}); + + const auto op = make_shared(); + op->set_arguments(OutputVector{data, filters}); + op->set_strides({1, 1}); + op->set_dilations({1, 1}); + op->set_pads_begin({2, 2}); + op->set_pads_end({2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 2); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 1})); + EXPECT_EQ(op->get_dilations(), Strides({1, 1})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2})); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 1, {2, -1}, {2, -1}})); } diff --git a/src/core/tests/type_prop/group_convolution_backprop_data.cpp b/src/core/tests/type_prop/group_convolution_backprop_data.cpp index df363fab80457b..da13820d999bf2 100644 --- a/src/core/tests/type_prop/group_convolution_backprop_data.cpp +++ b/src/core/tests/type_prop/group_convolution_backprop_data.cpp @@ -9,6 +9,7 @@ using namespace std; using namespace ngraph; +using namespace testing; TEST(type_prop, group_convolution_backprop_data_shape_infer) { const PartialShape data_pshape{1, 16, 6, 6}; // [N, C_IN * GROUPS, H, W] @@ -156,12 +157,10 @@ TEST(type_prop, group_convolution_backprop_data_shape_infer_with_output_shape_st } TEST(type_prop, group_convolution_backprop_data_shape_infer_with_output_shape_static_ranks_filters_group_cout_dyn) { - const PartialShape data_pshape{Dimension::dynamic(), 16, 5, 5}; // [N, C_IN * GROUPS, H, W] - const PartialShape filters_pshape{Dimension::dynamic(), - 16, - Dimension::dynamic(), - 3, - 3}; // [GROUPS, C_IN, C_OUT, kH, kW] + PartialShape data_pshape{Dimension::dynamic(), 16, 5, 5}; // [N, C_IN * GROUPS, H, W] + PartialShape filters_pshape{Dimension::dynamic(), 16, Dimension::dynamic(), 3, 3}; // [GROUPS, C_IN, C_OUT, kH, kW] + set_shape_labels(data_pshape, 10); + set_shape_labels(filters_pshape, 20); const element::Type_t et = element::f32; auto data = make_shared(et, data_pshape); @@ -174,6 +173,7 @@ TEST(type_prop, group_convolution_backprop_data_shape_infer_with_output_shape_st Strides{}, op::PadType::SAME_UPPER); + EXPECT_THAT(get_shape_labels(gcbd->get_output_partial_shape(0)), ElementsAre(10, 22, ov::no_label, ov::no_label)); ASSERT_EQ(gcbd->get_output_partial_shape(0), (PartialShape{Dimension::dynamic(), Dimension::dynamic(), 3, 3})); } @@ -270,12 +270,10 @@ TEST(type_prop, group_convolution_backprop_data_shape_infer_static_ranks_data_ci } TEST(type_prop, group_convolution_backprop_data_shape_infer_static_ranks_filters_group_cout_dyn) { - const PartialShape data_pshape{1, 20, 224, 224}; // [N, C_IN * GROUPS, H, W] - const PartialShape filters_pshape{Dimension::dynamic(), - Dimension::dynamic(), - 2, - 3, - 3}; // [GROUPS, C_IN, C_OUT, kH, kW] + PartialShape data_pshape{1, 20, 224, 224}; // [N, C_IN * GROUPS, H, W] + PartialShape filters_pshape{Dimension::dynamic(), Dimension::dynamic(), 2, 3, 3}; // [GROUPS, C_IN, C_OUT, kH, kW] + set_shape_labels(data_pshape, 10); + set_shape_labels(filters_pshape, 20); const element::Type_t et = element::f32; const Strides strides{2, 2}; @@ -291,7 +289,8 @@ TEST(type_prop, group_convolution_backprop_data_shape_infer_static_ranks_filters padding_begin, padding_end, dilations); - + EXPECT_THAT(get_shape_labels(gcbd->get_output_partial_shape(0)), + ElementsAre(10, ov::no_label, ov::no_label, ov::no_label)); ASSERT_EQ(gcbd->get_output_partial_shape(0), (PartialShape{1, Dimension::dynamic(), 447, 447})); } @@ -318,8 +317,10 @@ TEST(type_prop, group_convolution_backprop_data_shape_infer_static_ranks_data_sp } TEST(type_prop, group_convolution_backprop_data_shape_infer_static_ranks_filters_spatial_dim_dyn) { - const PartialShape data_pshape{Dimension::dynamic(), 20, 224, Dimension::dynamic()}; // [N, C_IN * GROUPS, H, W] - const PartialShape filters_pshape{4, 5, 2, 3, 3}; // [GROUPS, C_IN, C_OUT, kH, kW] + PartialShape data_pshape{Dimension::dynamic(), 20, 224, Dimension::dynamic()}; // [N, C_IN * GROUPS, H, W] + PartialShape filters_pshape{4, 5, 2, 3, 3}; // [GROUPS, C_IN, C_OUT, kH, kW] + set_shape_labels(data_pshape, 10); + set_shape_labels(filters_pshape, 20); const element::Type_t et = element::f32; const Strides strides{2, 2}; @@ -336,6 +337,8 @@ TEST(type_prop, group_convolution_backprop_data_shape_infer_static_ranks_filters padding_end, dilations); + EXPECT_THAT(get_shape_labels(gcbd->get_output_partial_shape(0)), + ElementsAre(10, ov::no_label, ov::no_label, ov::no_label)); ASSERT_EQ(gcbd->get_output_partial_shape(0), (PartialShape{Dimension::dynamic(), 8, 447, Dimension(1, -1)})); } @@ -627,7 +630,8 @@ TEST(type_prop, group_convolution_backprop_data_invalid_input_channel_dims) { } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING( error.what(), - std::string("Input channels dimension of data batch has incompatible value with filter shape.")); + std::string( + "Input channels dimension of data batch is incompatible with filter groups or input channels.")); } catch (...) { FAIL() << "Input shapes validation check failed for unexpected reason."; } @@ -646,7 +650,8 @@ TEST(type_prop, group_convolution_backprop_data_invalid_input_channel_dims) { } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING( error.what(), - std::string("Input channels dimension of data batch has incompatible value with filter shape.")); + std::string( + "Input channels dimension of data batch is incompatible with filter groups or input channels.")); } catch (...) { FAIL() << "Input shapes validation check failed for unexpected reason."; } @@ -671,7 +676,7 @@ TEST(type_prop, group_convolution_backprop_data_invalid_output_shape_spatial_dim FAIL() << "Incompatible output shape optional input not detected"; } catch (const NodeValidationFailure& error) { EXPECT_HAS_SUBSTRING(error.what(), - std::string("Output shape should be specified only and for all spatial dimensions.")); + std::string("Output shape should be defined for all and only spatial dimensions.")); } catch (...) { FAIL() << "Output shape validation check failed for unexpected reason."; } @@ -763,7 +768,7 @@ TEST(type_prop, group_convolution_backprop_data_invalid_conv_param_spatial_dims) make_shared(data, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads begin should be defined for all and only spatial dimensions."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -779,7 +784,7 @@ TEST(type_prop, group_convolution_backprop_data_invalid_conv_param_spatial_dims) make_shared(data, filters, strides, pads_begin, pads_end, dilations); FAIL() << "Invalid padding spatial dimensions not detected"; } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Pads end should be defined for all and only spatial dimensions."); + EXPECT_HAS_SUBSTRING(error.what(), "Pads begin and end should be defined for all and only spatial dimensions."); } catch (...) { FAIL() << "Padding spatial dimensions validation check failed for unexpected reason"; } @@ -835,17 +840,54 @@ TEST(type_prop, group_convolution_backprop_data_invalid_conv_param_spatial_dims) } } -TEST(type_prop, group_convolution_back_prop_data_default_constructed) { - auto conv = make_shared(); - - const auto &input_shape = ov::PartialShape::dynamic(), filters_shape = ov::PartialShape{1, 1, 1, 3, 3}, - output_spatial_shape_shape = ov::PartialShape({2}); - const auto& input_shapes = std::vector{input_shape, filters_shape, output_spatial_shape_shape}; - std::vector output_shapes = {ov::PartialShape::dynamic()}; - auto pad_begin = CoordinateDiff{}, pad_end = CoordinateDiff{}; - const auto& output_spatial_shape = ov::PartialShape{3, 3}; - int64_t num_spatial = - calculate_num_spatial(conv.get(), input_shape, filters_shape, output_spatial_shape_shape, 2, 3); - update_and_validate_attributes_back_prop(conv.get(), num_spatial); - EXPECT_NO_THROW(shape_infer(conv.get(), pad_begin, pad_end, output_spatial_shape, input_shapes, output_shapes)); +TEST(type_prop, group_convolution_backprop_data_default_constructed) { + const auto data = make_shared(element::f32, PartialShape::dynamic()); + const auto filters = make_shared(element::f32, PartialShape{1, 1, 1, 3, 3, 3}); + const auto out_spatial = op::Constant::create(element::i32, Shape{3}, {5, 4, 10}); + + const auto op = make_shared(); + op->set_arguments(OutputVector{data, filters, out_spatial}); + op->set_strides({1, 1, 1}); + op->set_dilations({1, 1, 1}); + op->set_pads_begin({2, 2, 2}); + op->set_pads_end({2, 2, 2}); + op->set_auto_pad(op::PadType::EXPLICIT); + op->validate_and_infer_types(); + + EXPECT_EQ(op->get_input_size(), 3); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_strides(), Strides({1, 1, 1})); + EXPECT_EQ(op->get_dilations(), Strides({1, 1, 1})); + EXPECT_EQ(op->get_pads_begin(), CoordinateDiff({2, 2, 2})); + EXPECT_EQ(op->get_pads_end(), CoordinateDiff({2, 2, 2})); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 1, 5, 4, 10})); +} + +TEST(type_prop, group_convolution_backprop_data_interval_shapes) { + PartialShape data_batch_pshape{{1, 3}, {2, 6}, {1, 5}, {3, 10}, {20, 100}}; + PartialShape filters_pshape{{2, 3}, {1, 3}, 1, 3, 3, 3}; + PartialShape out_spatial_pshape{{2, 3}, -1, 10}; + set_shape_labels(data_batch_pshape, 10); + set_shape_labels(filters_pshape, 20); + set_shape_labels(out_spatial_pshape, 30); + + const element::Type_t et = element::f32; + const auto auto_pad = op::PadType::SAME_LOWER; + + const auto data_batch = make_shared(et, data_batch_pshape); + const auto filters = make_shared(et, filters_pshape); + const auto out_spatial_shape_of = make_shared(make_shared(et, out_spatial_pshape)); + const auto op = make_shared(data_batch, + filters, + out_spatial_shape_of, + Strides{}, + CoordinateDiff{}, + CoordinateDiff{}, + Strides{}, + auto_pad); + + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(10, 20, 30, 31, 32)); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({{1, 3}, {2, 3}, {2, 3}, -1, 10})); + EXPECT_EQ(op->get_pads_begin(), (CoordinateDiff{0, 0, 0})); + EXPECT_EQ(op->get_pads_end(), (CoordinateDiff{0, 0, 0})); } diff --git a/src/core/tests/type_prop/interpolate.cpp b/src/core/tests/type_prop/interpolate.cpp index b220ecd8a8f754..7f0f5ff3a5bb68 100644 --- a/src/core/tests/type_prop/interpolate.cpp +++ b/src/core/tests/type_prop/interpolate.cpp @@ -214,6 +214,28 @@ TEST(type_prop, interpolate_v4_interval_logic) { ASSERT_TRUE(interp->get_output_partial_shape(0).same_scheme(out_shape)); } +TEST(type_prop, interpolate_v4_incorrect_mode) { + const auto image = std::make_shared(element::f32, Shape{1, 3, 30, 60}); + const auto target_shape = std::make_shared(element::i32, Shape{2}); + const auto scales = op::Constant::create(element::f32, Shape{2}, {6.f, 12.f}); + const auto axes = op::Constant::create(element::i64, Shape{2}, {2, 3}); + + ov::op::util::InterpolateBase::InterpolateAttrs attrs; + attrs.shape_calculation_mode = ov::op::util::InterpolateBase::ShapeCalcMode::SCALES; + attrs.mode = ov::op::util::InterpolateBase::InterpolateMode::BICUBIC_PILLOW; + attrs.pads_begin = {0, 0, 0, 0}; + attrs.pads_end = {0, 0, 0, 0}; + + OV_EXPECT_THROW(auto interp = std::make_shared(image, target_shape, scales, axes, attrs), + ov::NodeValidationFailure, + HasSubstr("Unsupported interpolation mode used with version 4 of the Interpolate op")); + + attrs.mode = ov::op::util::InterpolateBase::InterpolateMode::BILINEAR_PILLOW; + OV_EXPECT_THROW(auto interp = std::make_shared(image, target_shape, scales, axes, attrs), + ov::NodeValidationFailure, + HasSubstr("Unsupported interpolation mode used with version 4 of the Interpolate op")); +} + TEST(type_prop, interpolate_v11_scales) { const auto image = std::make_shared(element::f32, Shape{1, 3, 30, 60}); const auto scales = op::Constant::create(element::f32, Shape{2}, {0.2f, 0.2f}); diff --git a/src/core/tests/type_prop/matmul.cpp b/src/core/tests/type_prop/matmul.cpp index f7b6116a259adf..327e8574c53301 100644 --- a/src/core/tests/type_prop/matmul.cpp +++ b/src/core/tests/type_prop/matmul.cpp @@ -528,7 +528,7 @@ TEST(type_prop, matmul_propagate_labels_on_interval_dims) { } TEST(type_prop, matmul_propagate_label_on_b_input_after_reshape) { - constexpr size_t my_label = 2; + constexpr ov::label_t my_label = 2; auto marked_dim = Dimension(2, 3); ov::DimensionTracker::set_label(marked_dim, my_label); diff --git a/src/core/tests/type_prop/psroi_pooling.cpp b/src/core/tests/type_prop/psroi_pooling.cpp index 77df086f1f0e7b..7d60b9babb4160 100644 --- a/src/core/tests/type_prop/psroi_pooling.cpp +++ b/src/core/tests/type_prop/psroi_pooling.cpp @@ -2,224 +2,265 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/psroi_pooling.hpp" - +#include "common_test_utils/test_assertions.hpp" #include "gtest/gtest.h" -#include "ngraph/ngraph.hpp" -#include "util/type_prop.hpp" - -using namespace ngraph; - -TEST(type_prop, psroi_pooling_average) { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - ASSERT_EQ(op->get_shape(), (Shape{150, 2, 6, 6})); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); -} - -TEST(type_prop, psroi_pooling_bilinear) { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 18, 6, 1.0f, 2, 2, "bilinear"); - ASSERT_EQ(op->get_shape(), (Shape{150, 18, 6, 6})); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); -} - -TEST(type_prop, psroi_pooling_invalid_type) { - try { - auto inputs = std::make_shared(element::Type_t::i32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("Feature maps' data type must be floating point")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } - - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::i32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("Coords' data type must be floating point")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } -} - -TEST(type_prop, psroi_pooling_invalid_mode) { - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "invalid_mode"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("Expected 'average' or 'bilinear' mode")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } -} - -TEST(type_prop, psroi_pooling_invalid_shapes) { - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("PSROIPooling expects 4 dimensions for input")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } - - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 1, 72, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("PSROIPooling expects 2 dimensions for box coordinates")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } -} - -TEST(type_prop, psroi_pooling_invalid_group_size) { - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 0, 1.0f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("group_size has to be greater than 0")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } - - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 5, 1.0f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - std::string("Number of input's channels must be a multiply of group_size * group_size")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } -} - -TEST(type_prop, psroi_pooling_invalid_output_dim) { - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 17, 2, 1.0f, 0, 0, "average"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING( - error.what(), - std::string("output_dim must be equal to input channels divided by group_size * group_size")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } -} - -TEST(type_prop, psroi_pooling_invalid_spatial_bins) { - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 17, 2, 1.0f, 0, 0, "bilinear"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("spatial_bins_x has to be greater than 0")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } - - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 17, 2, 1.0f, 1, 0, "bilinear"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), std::string("spatial_bins_y has to be greater than 0")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } - - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 17, 2, 1.0f, 2, 5, "bilinear"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - std::string("Number of input's channels must be a multiply of " - "spatial_bins_x * spatial_bins_y")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } - - try { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 5, 5}); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 10, 2, 1.0f, 2, 4, "bilinear"); - FAIL() << "Exception expected"; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - std::string("output_dim must be equal to input channels divided by " - "spatial_bins_x * spatial_bins_y")); - } catch (...) { - FAIL() << "Unknown exception was thrown"; - } -} - -TEST(type_prop, psroi_pooling_dynamic_ranks) { - { - auto inputs = std::make_shared(element::Type_t::f32, PartialShape::dynamic()); - auto coords = std::make_shared(element::Type_t::f32, Shape{150, 5}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - ASSERT_EQ(op->get_output_partial_shape(0), PartialShape::dynamic()); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); - } - { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::f32, PartialShape::dynamic()); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - ASSERT_EQ(op->get_output_partial_shape(0), PartialShape::dynamic()); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); - } -} - -TEST(type_prop, psroi_pooling_dynamic_num_boxes) { - auto inputs = std::make_shared(element::Type_t::f32, Shape{1, 72, 4, 5}); - auto coords = std::make_shared(element::Type_t::f32, PartialShape{{Dimension::dynamic(), 5}}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - ASSERT_EQ(op->get_output_partial_shape(0), (PartialShape{{Dimension::dynamic(), 2, 6, 6}})); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); -} - -TEST(type_prop, psroi_pooling_static_rank_dynamic_shape) { - { - auto inputs = std::make_shared( - element::Type_t::f32, - PartialShape{{Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()}}); - auto coords = std::make_shared(element::Type_t::f32, - PartialShape{{Dimension::dynamic(), Dimension::dynamic()}}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - ASSERT_EQ(op->get_output_partial_shape(0), (PartialShape{{Dimension::dynamic(), 2, 6, 6}})); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); - } - { - auto inputs = std::make_shared( - element::Type_t::f32, - PartialShape{{Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()}}); - auto coords = std::make_shared(element::Type_t::f32, PartialShape{{200, Dimension::dynamic()}}); - auto op = std::make_shared(inputs, coords, 2, 6, 0.0625f, 0, 0, "average"); - ASSERT_EQ(op->get_shape(), (Shape{200, 2, 6, 6})); - ASSERT_EQ(op->get_element_type(), element::Type_t::f32); - } +#include "openvino/opsets/opset11.hpp" +#include "type_prop.hpp" + +using namespace ov; +using namespace ov::opset11; +using namespace testing; + +class TypePropPSROIPoolingV0 : public TypePropOpTest { +protected: + float spatial_scale = 0.625f; + int bin_not_used = 0; + Shape pooling_roi_2x2{2, 2}; +}; + +TEST_F(TypePropPSROIPoolingV0, basic_average) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, bin_not_used, bin_not_used, "average"); + + EXPECT_EQ(op->get_shape(), (Shape{150, 2, 6, 6})); + EXPECT_EQ(op->get_element_type(), element::f32); +} + +TEST_F(TypePropPSROIPoolingV0, basic_bilinear) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + auto op = make_op(inputs, coords, 18, 6, 1.0f, 2, 2, "bilinear"); + + EXPECT_EQ(op->get_shape(), (Shape{150, 18, 6, 6})); + EXPECT_EQ(op->get_element_type(), element::f32); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_features_element_type) { + const auto inputs = std::make_shared(element::i32, Shape{1, 72, 4, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 6, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("Feature maps' data type must be floating point")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_rois_element_type) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 5}); + const auto coords = std::make_shared(element::u16, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 6, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("Coords' data type must be floating point")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_pooling_mode) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 6, spatial_scale, bin_not_used, bin_not_used, "invalid"), + NodeValidationFailure, + HasSubstr("Expected 'average' or 'bilinear' mode")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_features_rank) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 6, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("Expected a 4D tensor for the feature maps input")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_rois_rank) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 2}); + const auto coords = std::make_shared(element::f32, Shape{150}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 6, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("Expected a 2D tensor for the ROIs input with box coordinates")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_group_size) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 2}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 0, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("group_size has to be greater than 0")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_number_of_channels_and_group_size_in_avg_mode) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 2}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 2, 5, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("Number of input's channels must be a multiply of output_dim * group_size * group_size")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_output_dim_in_avg_mode) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 4, 2}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 17, 2, spatial_scale, bin_not_used, bin_not_used, "average"), + NodeValidationFailure, + HasSubstr("Number of input's channels must be a multiply of output_dim * group_size * group_size")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_spatial_bins_x) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 5, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 17, 2, spatial_scale, 0, 1, "bilinear"), + NodeValidationFailure, + HasSubstr("spatial_bins_x has to be greater than 0")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_spatial_bins_y) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 5, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW(auto op = make_op(inputs, coords, 17, 2, spatial_scale, 1, 0, "bilinear"), + NodeValidationFailure, + HasSubstr("spatial_bins_y has to be greater than 0")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_number_of_channels_and_spatial_bins_in_bilinear_mode) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 5, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW( + auto op = make_op(inputs, coords, 17, 2, spatial_scale, 2, 5, "bilinear"), + NodeValidationFailure, + HasSubstr("Number of input's channels must be a multiply of output_dim * spatial_bins_x * spatial_bins_y")); +} + +TEST_F(TypePropPSROIPoolingV0, invalid_output_dim_in_bilinear_mode) { + const auto inputs = std::make_shared(element::f32, Shape{1, 72, 5, 5}); + const auto coords = std::make_shared(element::f32, Shape{150, 5}); + + OV_EXPECT_THROW( + auto op = make_op(inputs, coords, 10, 2, spatial_scale, 2, 4, "bilinear"), + NodeValidationFailure, + HasSubstr("Number of input's channels must be a multiply of output_dim * spatial_bins_x * spatial_bins_y")); +} + +TEST_F(TypePropPSROIPoolingV0, features_dynamic_rank) { + auto coords_shape = PartialShape{150, 5}; + set_shape_labels(coords_shape, 20); + + const auto inputs = std::make_shared(element::f16, PartialShape::dynamic()); + const auto coords = std::make_shared(element::f16, coords_shape); + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, 0, 0, "average"); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({150, 2, 6, 6})); // 4d + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), + ElementsAre(20, ov::no_label, ov::no_label, ov::no_label)); +} + +TEST_F(TypePropPSROIPoolingV0, rois_dynamic_rank) { + auto feat_shape = PartialShape{1, 72, 4, 5}; + set_shape_labels(feat_shape, 10); + + const auto inputs = std::make_shared(element::f16, feat_shape); + const auto coords = std::make_shared(element::f16, PartialShape::dynamic()); + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, 0, 0, "average"); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 2, 6, 6})); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), Each(ov::no_label)); +} + +TEST_F(TypePropPSROIPoolingV0, dynamic_num_boxes) { + auto coords_shape = PartialShape{{Dimension::dynamic(), 5}}; + set_shape_labels(coords_shape, 20); + + const auto inputs = std::make_shared(element::f16, PartialShape::dynamic()); + const auto coords = std::make_shared(element::f16, coords_shape); + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, 0, 0, "average"); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 2, 6, 6})); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), + ElementsAre(20, ov::no_label, ov::no_label, ov::no_label)); +} + +TEST_F(TypePropPSROIPoolingV0, feat_static_rank_dynamic_shape) { + auto feat_shape = PartialShape::dynamic(4); + auto coords_shape = PartialShape{{Dimension::dynamic(), 5}}; + set_shape_labels(feat_shape, 10); + set_shape_labels(coords_shape, 20); + + const auto inputs = std::make_shared(element::f16, feat_shape); + const auto coords = std::make_shared(element::f16, coords_shape); + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, 0, 0, "average"); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 2, 6, 6})); // 4d + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), + ElementsAre(20, ov::no_label, ov::no_label, ov::no_label)); +} + +TEST_F(TypePropPSROIPoolingV0, feat_and_rois_static_rank_dynamic_shape) { + auto feat_shape = PartialShape::dynamic(4); + auto coords_shape = PartialShape::dynamic(2); + set_shape_labels(feat_shape, 10); + set_shape_labels(coords_shape, 20); + + const auto inputs = std::make_shared(element::f16, feat_shape); + const auto coords = std::make_shared(element::f16, coords_shape); + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, 0, 0, "average"); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({-1, 2, 6, 6})); // 4d + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), + ElementsAre(20, ov::no_label, ov::no_label, ov::no_label)); +} + +TEST_F(TypePropPSROIPoolingV0, feat_and_rois_interval_shapes) { + auto feat_shape = PartialShape{{1, 2}, {10, 100}, {10, 20}, {30, 90}}; + auto coords_shape = PartialShape{{3, 10}, {1, 5}}; + set_shape_labels(feat_shape, 10); + set_shape_labels(coords_shape, 20); + + const auto inputs = std::make_shared(element::f16, feat_shape); + const auto coords = std::make_shared(element::f16, coords_shape); + const auto op = make_op(inputs, coords, 2, 6, spatial_scale, 0, 0, "average"); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({{3, 10}, 2, 6, 6})); // 4d + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), + ElementsAre(20, ov::no_label, ov::no_label, ov::no_label)); +} + +TEST_F(TypePropPSROIPoolingV0, default_ctor) { + auto feat_shape = PartialShape{2, {10, 100}, 10, 10}; + auto coords_shape = PartialShape{{3, 10}, {1, 5}}; + set_shape_labels(feat_shape, 10); + set_shape_labels(coords_shape, 20); + + const auto inputs = std::make_shared(element::f16, feat_shape); + const auto coords = std::make_shared(element::f16, coords_shape); + + const auto op = make_op(); + op->set_arguments(OutputVector{inputs, coords}); + op->set_output_dim(2); + op->set_group_size(6); + op->set_spatial_scale(spatial_scale); + op->set_mode("average"); + op->validate_and_infer_types(); + + EXPECT_FLOAT_EQ(op->get_spatial_scale(), spatial_scale); + EXPECT_EQ(op->get_mode(), "average"); + EXPECT_EQ(op->get_group_size(), 6); + EXPECT_EQ(op->get_input_size(), 2); + EXPECT_EQ(op->get_output_size(), 1); + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), PartialShape({{3, 10}, 2, 6, 6})); // 4d + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), + ElementsAre(20, ov::no_label, ov::no_label, ov::no_label)); } diff --git a/src/core/tests/type_prop/roi_pooling.cpp b/src/core/tests/type_prop/roi_pooling.cpp index 0fa337a37ea5be..e86b52eef52641 100644 --- a/src/core/tests/type_prop/roi_pooling.cpp +++ b/src/core/tests/type_prop/roi_pooling.cpp @@ -2,109 +2,171 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "common_test_utils/test_assertions.hpp" #include "gtest/gtest.h" -#include "ngraph/ngraph.hpp" +#include "openvino/opsets/opset11.hpp" +#include "type_prop.hpp" using namespace std; -using namespace ngraph; - -TEST(type_prop, roi_pooling_basic_shape_inference) { - const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); - const auto rois = make_shared(element::f32, Shape{4, 5}); - const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); - ASSERT_EQ(op->get_method(), "max"); - ASSERT_EQ(op->get_shape(), (Shape{4, 3, 2, 2})); +using namespace ov; +using namespace ov::opset11; +using namespace testing; + +class TypePropROIPoolingV0 : public TypePropOpTest { +protected: + float spatial_scale = 0.625f; + Shape pooling_roi_2x2{2, 2}; +}; + +TEST_F(TypePropROIPoolingV0, default_ctor) { + const auto feat_maps = make_shared(element::f32, PartialShape{{0, 3}, {1, 3}, {1, 6}, {1, 6}}); + const auto rois = make_shared(element::f32, PartialShape{{2, 4}, {1, 5}}); + + const auto op = make_op(); + op->set_arguments(OutputVector{feat_maps, rois}); + op->set_spatial_scale(spatial_scale); + op->set_method("max"); + op->set_output_roi({3, 4}); + op->validate_and_infer_types(); + + EXPECT_FLOAT_EQ(op->get_spatial_scale(), spatial_scale); + EXPECT_EQ(op->get_output_roi(), Shape({3, 4})); + EXPECT_EQ(op->get_method(), "max"); + EXPECT_EQ(op->get_input_size(), 2); + EXPECT_EQ(op->get_element_type(), element::f32); + EXPECT_EQ(static_cast(op.get())->get_output_size(), 1); + EXPECT_EQ(op->get_output_partial_shape(0), (PartialShape{{2, 4}, {1, 3}, 3, 4})); } -TEST(type_prop, roi_pooling_dynamic_channels_dim) { - const auto feat_maps = make_shared(element::f32, PartialShape{1, Dimension(), 6, 6}); - const auto rois = make_shared(element::f32, Shape{4, 5}); - const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"); - ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{4, Dimension(), 2, 2})); +TEST_F(TypePropROIPoolingV0, basic_shape_inference) { + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); + const auto rois = make_shared(element::f32, Shape{4, 5}); + const auto op = make_op(feat_maps, rois, pooling_roi_2x2, 0.625f); + + EXPECT_EQ(op->get_element_type(), element::f32); + EXPECT_EQ(op->get_method(), "max"); + EXPECT_EQ(op->get_shape(), (Shape{4, 3, 2, 2})); } -TEST(type_prop, roi_pooling_dynamic_num_rois_dim) { - const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); - const auto rois = make_shared(element::f32, PartialShape{Dimension(), 5}); - const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); - ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{Dimension(), 3, 2, 2})); +TEST_F(TypePropROIPoolingV0, dynamic_channels_dim) { + auto feat_shape = PartialShape{1, -1, 6, 6}; + auto rois_shape = PartialShape{4, 5}; + set_shape_labels(feat_shape, 10); + set_shape_labels(rois_shape, 20); + + const auto feat_maps = make_shared(element::f32, feat_shape); + const auto rois = make_shared(element::f32, rois_shape); + const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "max"); + + EXPECT_EQ(op->get_element_type(), element::f32); + EXPECT_EQ(op->get_output_partial_shape(0), (PartialShape{4, -1, 2, 2})); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(20, 11, ov::no_label, ov::no_label)); } -TEST(type_prop, roi_pooling_dynamic_rank_feat_maps) { - const auto feat_maps = make_shared(element::f32, PartialShape::dynamic()); - const auto rois = make_shared(element::f32, Shape{4, 5}); - const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); - ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{4, Dimension(), 2, 2})); +TEST_F(TypePropROIPoolingV0, dynamic_num_rois_dim) { + auto feat_shape = PartialShape{1, 3, 6, 6}; + auto rois_shape = PartialShape{-1, 5}; + set_shape_labels(feat_shape, 10); + set_shape_labels(rois_shape, 20); + + const auto feat_maps = make_shared(element::f64, feat_shape); + const auto rois = make_shared(element::f64, rois_shape); + const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "bilinear"); + + EXPECT_EQ(op->get_element_type(), element::f64); + EXPECT_EQ(op->get_output_partial_shape(0), (PartialShape{-1, 3, 2, 2})); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), ElementsAre(20, 11, ov::no_label, ov::no_label)); } -TEST(type_prop, roi_pooling_dynamic_rank_rois) { - const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); - const auto rois = make_shared(element::f32, PartialShape::dynamic()); - const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); - ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{Dimension(), 3, 2, 2})); +TEST_F(TypePropROIPoolingV0, dynamic_rank_feat_maps) { + const auto feat_maps = make_shared(element::f16, PartialShape::dynamic()); + const auto rois = make_shared(element::f16, Shape{4, 5}); + const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale); + + EXPECT_EQ(op->get_element_type(), element::f16); + EXPECT_EQ(op->get_output_partial_shape(0), (PartialShape{4, -1, 2, 2})); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), Each(ov::no_label)); } -TEST(type_prop, roi_pooling_incompatible_input_rank) { - const auto feat_maps = make_shared(element::f32, Shape{1, 3, 2, 6, 6}); - const auto rois = make_shared(element::f32, Shape{3, 5}); - // feat_maps must be of rank 4 - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, dynamic_rank_feat_rois) { + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape::dynamic()); + const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale); + + EXPECT_EQ(op->get_element_type(), element::f32); + EXPECT_EQ(op->get_output_partial_shape(0), (PartialShape{-1, 3, 2, 2})); + EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), Each(ov::no_label)); } -TEST(type_prop, roi_pooling_incompatible_pooling_shape) { - Shape pool_shape{2, 2, 2}; - const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f32, Shape{3, 5}); - // pool_shape must be of rank 2 {pooled_h, pooled_w} - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, pool_shape, 0.625f, "max"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, incompatible_input_rank) { + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "max"), + NodeValidationFailure, + HasSubstr("Expected a 4D tensor for the feature maps input")); } -TEST(type_prop, roi_pooling_incompatible_rois_second_dim) { - const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f32, Shape{3, 4}); - // the second dim of rois must be 5. [batch_id, x_1, y_1, x_2, y_2] - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, incompatible_pooling_shape) { + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, Shape{2, 2, 2}, spatial_scale, "max"), + NodeValidationFailure, + HasSubstr("The dimension of pooled size is expected to be equal to 2")); +} + +TEST_F(TypePropROIPoolingV0, incompatible_rois_second_dim) { + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 4}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "max"), + NodeValidationFailure, + HasSubstr("The second dimension of ROIs input should contain batch id and box coordinates. This " + "dimension is expected to be equal to 5")); } -TEST(type_prop, roi_pooling_incompatible_feature_maps_element_type) { - const auto feat_maps = make_shared(element::i32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f32, Shape{3, 5}); - // feat_maps element type must be floating point type - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, incompatible_feature_maps_element_type) { + const auto feat_maps = make_shared(element::i32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "max"), + NodeValidationFailure, + HasSubstr("The data type for input and ROIs is expected to be a floating point type")); } -TEST(type_prop, roi_pooling_incompatible_rois_element_type) { - const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f16, Shape{3, 5}); - // rois element type must be equal to feat_maps element type (floating point type) - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "bilinear"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, incompatible_rois_element_type) { + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::i16, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "bilinear"), + NodeValidationFailure, + HasSubstr("The data type for input and ROIs is expected to be a floating point type")); } -TEST(type_prop, roi_pooling_invalid_pooling_method) { - const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f16, Shape{3, 5}); - // ROIPooling method is invalid: not max nor bilinear - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "invalid"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, invalid_pooling_method) { + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, pooling_roi_2x2, spatial_scale, "invalid"), + NodeValidationFailure, + HasSubstr("Pooling method attribute should be either \'max\' or \'bilinear\'")); } -TEST(type_prop, roi_pooling_invalid_spatial_scale) { - const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f16, Shape{3, 5}); - // ROIPooling spatial scale attribute must be a positive floating point number - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{2, 2}, -0.625f, "max"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, invalid_spatial_scale) { + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, pooling_roi_2x2, -1.0f), + NodeValidationFailure, + HasSubstr("The spatial scale attribute should be a positive floating point number")); } -TEST(type_prop, roi_pooling_invalid_pooled_size) { - const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); - const auto rois = make_shared(element::f16, Shape{3, 5}); - // ROIPooling pooled_h and pooled_w must be non-negative integers - ASSERT_THROW(const auto unused = make_shared(feat_maps, rois, Shape{1, 0}, 0.625f, "max"), - ngraph::NodeValidationFailure); +TEST_F(TypePropROIPoolingV0, invalid_pooled_size) { + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{3, 5}); + + OV_EXPECT_THROW(const auto op = make_op(feat_maps, rois, Shape{1, 0}, spatial_scale), + NodeValidationFailure, + HasSubstr("Pooled size attributes pooled_h and pooled_w should should be positive integers")); } diff --git a/src/core/tests/type_prop/variadic_split.cpp b/src/core/tests/type_prop/variadic_split.cpp index 41aa784b90334f..175c04f217c8e5 100644 --- a/src/core/tests/type_prop/variadic_split.cpp +++ b/src/core/tests/type_prop/variadic_split.cpp @@ -39,7 +39,9 @@ class VariadicSplitTest : public TestWithParam { std::generate_n(std::back_inserter(in_labels), p_shape.size(), ov::SeqGen(10)); auto exp_labels = in_labels; + OPENVINO_SUPPRESS_DEPRECATED_START const auto n_axis = normalize_axis("", axis, p_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END exp_labels[n_axis] = ov::no_label; return {in_labels, exp_labels}; diff --git a/src/core/tests/validation_utils.cpp b/src/core/tests/validation_utils.cpp index ca5c0d66f3bd6f..694633ae03b152 100644 --- a/src/core/tests/validation_utils.cpp +++ b/src/core/tests/validation_utils.cpp @@ -4,10 +4,11 @@ #include -#include -#include -#include -#include +#include "openvino/core/type.hpp" +#include "openvino/core/validation_util.hpp" +#include "openvino/opsets/opset8.hpp" +#include "openvino/util/common_util.hpp" +#include "validation_util.hpp" TEST(get_constant_from_source, invalidation_check) { auto a = ov::opset8::Constant::create(ov::element::i64, {100}, {123}); @@ -17,7 +18,9 @@ TEST(get_constant_from_source, invalidation_check) { auto r = std::make_shared(div, s, true); auto tmp_consumer = std::make_shared(s); + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_TRUE(ov::get_constant_from_source(r)); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(r->get_output_tensor(0).get_lower_value()); ASSERT_TRUE(r->get_output_tensor(0).get_upper_value()); @@ -42,7 +45,9 @@ TEST(get_constant_from_source, extract_static_dim_from_dynamic_shape_check) { auto zero = ov::opset8::Constant::create(ov::element::i64, {1}, {0}); const auto extract_static_dimension = std::make_shared(shape, one, zero); + OPENVINO_SUPPRESS_DEPRECATED_START ASSERT_TRUE(ov::get_constant_from_source(extract_static_dimension)); + OPENVINO_SUPPRESS_DEPRECATED_END ASSERT_TRUE(extract_static_dimension->get_output_tensor(0).get_lower_value()); ASSERT_TRUE(extract_static_dimension->get_output_tensor(0).get_upper_value()); @@ -61,7 +66,7 @@ TEST(constantfold_subgraph, split) { auto axis = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {0}); auto split = std::make_shared(mul, axis, lenghts); std::vector expected(std::next(input.begin(), input.size() / 2), input.end()); - auto ret = ov::constantfold_subgraph(split->output(1)); + auto ret = ov::util::constantfold_subgraph(split->output(1)); ASSERT_NE(ret, nullptr); auto actual = ret->cast_vector(); ASSERT_EQ(expected, actual); diff --git a/src/core/tests/visitors/op/roi_pooling.cpp b/src/core/tests/visitors/op/roi_pooling.cpp index 8438a797728eb1..a5b49fe9cca3d2 100644 --- a/src/core/tests/visitors/op/roi_pooling.cpp +++ b/src/core/tests/visitors/op/roi_pooling.cpp @@ -25,7 +25,7 @@ TEST(attributes, roi_pooling_op) { NodeBuilder builder(op, {data, coords}); const auto g_op = ov::as_type_ptr(builder.create()); - EXPECT_EQ(g_op->get_output_size(), op->get_output_size()); + EXPECT_EQ(g_op->get_output_roi(), op->get_output_roi()); EXPECT_EQ(g_op->get_spatial_scale(), op->get_spatial_scale()); EXPECT_EQ(g_op->get_method(), op->get_method()); } diff --git a/src/frontends/CMakeLists.txt b/src/frontends/CMakeLists.txt index 41902d02d24b4e..3172c51d072e15 100644 --- a/src/frontends/CMakeLists.txt +++ b/src/frontends/CMakeLists.txt @@ -8,22 +8,18 @@ if(ENABLE_TESTS) add_subdirectory(tests) endif() -if(ENABLE_OV_ONNX_FRONTEND) - add_subdirectory(onnx) +if(ENABLE_OV_IR_FRONTEND) + add_subdirectory(ir) endif() -if(ENABLE_OV_PADDLE_FRONTEND) - add_subdirectory(paddle) +if(ENABLE_OV_ONNX_FRONTEND) + add_subdirectory(onnx) endif() if(ENABLE_OV_PYTORCH_FRONTEND) add_subdirectory(pytorch) endif() -if(ENABLE_OV_IR_FRONTEND) - add_subdirectory(ir) -endif() - if(ENABLE_OV_TF_FRONTEND OR ENABLE_OV_TF_LITE_FRONTEND) add_subdirectory(tensorflow_common) endif() @@ -35,3 +31,7 @@ endif() if (ENABLE_OV_TF_LITE_FRONTEND) add_subdirectory(tensorflow_lite) endif() + +if(ENABLE_OV_PADDLE_FRONTEND) + add_subdirectory(paddle) +endif() diff --git a/src/frontends/common/include/openvino/frontend/decoder.hpp b/src/frontends/common/include/openvino/frontend/decoder.hpp index 02d82116595e31..d51f41844f6a1f 100644 --- a/src/frontends/common/include/openvino/frontend/decoder.hpp +++ b/src/frontends/common/include/openvino/frontend/decoder.hpp @@ -64,21 +64,12 @@ class FRONTEND_API DecoderBase { /// /// \param input_port_idx Input port index by which data is consumed /// \param producer_name A producer name - /// \return producer_output_port_index Output port index from which data is generated - virtual void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index) const = 0; - - /// \brief Get a producer name and its output port index - /// - /// \param input_port_idx Input port index by which data is consumed - /// \param producer_name A producer name + /// \param producer_output_port_name Output port name if exists /// \param producer_output_port_index Output port index from which data is generated - /// \param op_type_by_name Map of operation name to their types virtual void get_input_node(size_t input_port_idx, std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const = 0; + std::string& producer_output_port_name, + size_t& producer_output_port_index) const = 0; /// \brief Get operation type virtual const std::string& get_op_type() const = 0; diff --git a/src/frontends/common/include/openvino/frontend/extension/conversion.hpp b/src/frontends/common/include/openvino/frontend/extension/conversion.hpp index c4aea65af2e565..92d66e485c6c36 100644 --- a/src/frontends/common/include/openvino/frontend/extension/conversion.hpp +++ b/src/frontends/common/include/openvino/frontend/extension/conversion.hpp @@ -37,6 +37,10 @@ class FRONTEND_API ConversionExtension : public ConversionExtensionBase { : ConversionExtensionBase(op_type), m_converter_named(converter) {} + ConversionExtension(const std::string& op_type, const CreatorFunctionNamedAndIndexed& converter) + : ConversionExtensionBase(op_type), + m_converter_named_and_indexed(converter) {} + const CreatorFunction& get_converter() const { return m_converter; }; @@ -45,11 +49,16 @@ class FRONTEND_API ConversionExtension : public ConversionExtensionBase { return m_converter_named; }; + const CreatorFunctionNamedAndIndexed& get_converter_named_and_indexed() const { + return m_converter_named_and_indexed; + }; + ~ConversionExtension() override; private: CreatorFunction m_converter; CreatorFunctionNamed m_converter_named; + CreatorFunctionNamedAndIndexed m_converter_named_and_indexed; }; } // namespace frontend diff --git a/src/frontends/common/include/openvino/frontend/extension/op.hpp b/src/frontends/common/include/openvino/frontend/extension/op.hpp index 0c9912609897a9..27ed4e0c7ecfba 100644 --- a/src/frontends/common/include/openvino/frontend/extension/op.hpp +++ b/src/frontends/common/include/openvino/frontend/extension/op.hpp @@ -112,6 +112,30 @@ class OpExtensionBase : public BaseConversionType { OpExtensionBase(const std::string& fw_type_name, const std::map& attr_names_map = {}, const std::map& attr_values_map = {}); + + OpExtensionBase(const std::map& attr_names_map, + const std::map& attr_values_map = {}) + : OpExtensionBase(OVOpType::get_type_info_static().name, attr_names_map, attr_values_map) {} + + OpExtensionBase(const std::string& fw_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map = {}); + + OpExtensionBase(const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}) + : OpExtensionBase(OVOpType::get_type_info_static().name, + in_names_vec, + out_names_vec, + attr_names_map, + attr_values_map) {} + + OpExtensionBase(const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}); }; template @@ -131,6 +155,35 @@ class OpExtensionBase : public BaseConversionType { const std::string& fw_type_name, const std::map& attr_names_map = {}, const std::map& attr_values_map = {}); + + explicit OpExtensionBase(const std::string& fw_ov_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map = {}) + : OpExtensionBase(fw_ov_type_name, fw_ov_type_name, attr_names_map, attr_values_map) {} + + OpExtensionBase(const std::string& ov_type_name, + const std::string& fw_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map = {}); + + OpExtensionBase(const std::string& fw_ov_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}) + : OpExtensionBase(fw_ov_type_name, + fw_ov_type_name, + in_names_vec, + out_names_vec, + attr_names_map, + attr_values_map) {} + + explicit OpExtensionBase(const std::string& ov_type_name, + const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}); }; class FWVisitor : public ov::AttributeVisitor { @@ -198,6 +251,130 @@ class OpConversionFunction { std::map m_attr_values_map; }; +class OpConversionFunctionNamed { +public: + explicit OpConversionFunctionNamed(const std::function()>& op_creator, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}) + : m_op_creator(op_creator), + m_in_names_vec(in_names_vec), + m_out_names_vec(out_names_vec), + m_attr_names_map(attr_names_map), + m_attr_values_map(attr_values_map) {} + + std::map operator()(const NodeContext& context) { + auto node = m_op_creator(); + + std::vector> inputs; + for (const auto& name : m_in_names_vec) { + for (size_t i = 0; i < context.get_input_size(name); ++i) { + inputs.push_back(context.get_input(name, static_cast(i))); + } + } + + node->set_arguments(inputs); + FWVisitor fw_visitor(context, m_attr_names_map, m_attr_values_map); + node->visit_attributes(fw_visitor); + node->validate_and_infer_types(); + std::map out; + OPENVINO_ASSERT(m_out_names_vec.size() == node->get_output_size(), + "each output should has a name, names number: ", + m_out_names_vec.size(), + ", output size: ", + node->get_output_size()); + int i = 0; + for (const auto& name : m_out_names_vec) { + out[name].emplace_back(node->output(i++)); + } + return out; + } + +private: + std::function()> m_op_creator; + std::vector m_in_names_vec; + std::vector m_out_names_vec; + std::map m_attr_names_map; + std::map m_attr_values_map; +}; + +class FWVisitorInputAttributes : public ov::AttributeVisitor { +public: + explicit FWVisitorInputAttributes(const NodeContext& context, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}) + : m_context(context), + m_attr_names_map(attr_names_map), + m_attr_values_map(attr_values_map) {} + + void on_adapter(const std::string& name, ValueAccessor& adapter) override { + auto p_value = m_attr_values_map.find(name); + auto p_name = m_attr_names_map.find(name); + bool is_value_found = p_value != m_attr_values_map.end(); + bool is_name_mapping_found = p_name != m_attr_names_map.end(); + OPENVINO_ASSERT(!(is_value_found && is_name_mapping_found), + "For attribute " + name + + " both name mapping and value mapping are provided." + " The behavior for this case is undefined. Please leave only one mapping."); + + if (is_value_found) { + adapter.set_as_any(p_value->second); + } else if (is_name_mapping_found) { + auto a = m_context.get_values_from_const_input(static_cast(p_name->second)); + adapter.set_as_any(a); + } else { + OPENVINO_ASSERT(false, + "\nValue for attribute \"", + name, + "\" is not set or mapping between " + "framework and openvino node attributes is incorrect."); + } + } + +private: + const NodeContext& m_context; + const std::map& m_attr_names_map; + const std::map& m_attr_values_map; +}; + +class OpConversionFunctionInputAttributes { +public: + explicit OpConversionFunctionInputAttributes(const std::function()>& op_creator, + const std::map& attr_names_map = {}, + const std::map& attr_values_map = {}) + : m_op_creator(op_creator), + m_attr_names_map(attr_names_map), + m_attr_values_map(attr_values_map) { + first_attr_index = std::numeric_limits::max(); + for (const auto& it : m_attr_names_map) { + first_attr_index = std::min(first_attr_index, it.second); + } + } + + ov::OutputVector operator()(const NodeContext& context) { + auto node = m_op_creator(); + + std::vector> inputs; + for (size_t i = 0; i < context.get_input_size(); ++i) { + if (i < first_attr_index) { + inputs.push_back(context.get_input(static_cast(i))); + } + } + node->set_arguments(inputs); + FWVisitorInputAttributes fw_visitor(context, m_attr_names_map, m_attr_values_map); + node->visit_attributes(fw_visitor); + node->validate_and_infer_types(); + return node->outputs(); + } + +private: + std::function()> m_op_creator; + std::map m_attr_names_map; + std::map m_attr_values_map; + size_t first_attr_index; +}; + template OpExtensionBase::OpExtensionBase(const std::string& ov_type_name, const std::string& fw_type_name, @@ -211,6 +388,36 @@ OpExtensionBase::OpExtensionBase(const std::string& ov attr_names_map, attr_values_map)) {} +template +OpExtensionBase::OpExtensionBase(const std::string& ov_type_name, + const std::string& fw_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map) + : BaseConversionType(fw_type_name, + OpConversionFunctionInputAttributes( + [ov_type_name]() { + return create_ov_node_by_name(ov_type_name); + }, + attr_names_map, + attr_values_map)) {} + +template +OpExtensionBase::OpExtensionBase(const std::string& ov_type_name, + const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map, + const std::map& attr_values_map) + : BaseConversionType(fw_type_name, + OpConversionFunctionNamed( + [ov_type_name]() -> std::shared_ptr { + return create_ov_node_by_name(ov_type_name); + }, + in_names_vec, + out_names_vec, + attr_names_map, + attr_values_map)) {} + template OpExtensionBase::OpExtensionBase(const std::string& fw_type_name, const std::map& attr_names_map, @@ -223,29 +430,84 @@ OpExtensionBase::OpExtensionBase(const std::string attr_names_map, attr_values_map)) {} +template +OpExtensionBase::OpExtensionBase(const std::string& fw_type_name, + const std::map& attr_names_map, + const std::map& attr_values_map) + : BaseConversionType(fw_type_name, + OpConversionFunctionInputAttributes( + []() { + return std::make_shared(); + }, + attr_names_map, + attr_values_map)) {} + +template +OpExtensionBase::OpExtensionBase(const std::string& fw_type_name, + const std::vector& in_names_vec, + const std::vector& out_names_vec, + const std::map& attr_names_map, + const std::map& attr_values_map) + : BaseConversionType(fw_type_name, + OpConversionFunctionNamed( + []() { + return std::make_shared(); + }, + in_names_vec, + out_names_vec, + attr_names_map, + attr_values_map)) {} + template using OpExtension = ov::frontend::OpExtensionBase; +#define FRONTEND_EXPAND(X) X +#define FRONTEND_CAT_(x, y) x##y +#define FRONTEND_CAT(x, y) FRONTEND_CAT_(x, y) +// extract common attribute and values +#define GEN_VAR_COMMON(...) \ + auto params = make_spec_tuple(__VA_ARGS__); \ + const auto& name = std::get<0>(params); \ + const auto& attr_mp = std::get<1>(params); \ + const auto& val_mp = std::get<2>(params); +// extract paddle specific param + common values +#define GEN_VAR_PADDLE(in_names, out_names, ...) \ + const std::vector in_names_vec(in_names); \ + const std::vector out_names_vec(out_names); \ + GEN_VAR_COMMON(__VA_ARGS__) +// make common except paddle OpExtension +#define MAKE_MAP_COMMON(FRAMEWORK, ...) \ + GEN_VAR_COMMON(__VA_ARGS__) \ + if (!name.empty()) \ + return std::make_shared>(name, attr_mp, val_mp); \ + return std::make_shared>(attr_mp, val_mp); +#define MAKE_MAP_onnx(...) MAKE_MAP_COMMON(onnx, __VA_ARGS__) +#define MAKE_MAP_tensorflow(...) MAKE_MAP_COMMON(tensorflow, __VA_ARGS__) +#define MAKE_MAP_tensorflow_lite(...) MAKE_MAP_COMMON(tensorflow_lite, __VA_ARGS__) +// make paddle OpExtension +#define MAKE_MAP_paddle(...) \ + FRONTEND_EXPAND(GEN_VAR_PADDLE(__VA_ARGS__)) \ + if (!name.empty()) \ + return std::make_shared>(name, \ + in_names_vec, \ + out_names_vec, \ + attr_mp, \ + val_mp); \ + return std::make_shared>(in_names_vec, out_names_vec, attr_mp, val_mp); + // Per each FRAMEWORK this macro can be used once in one operation class definition // It defines a member inline function that creates required extension. -#define OPENVINO_FRAMEWORK_MAP(FRAMEWORK, ...) \ - template \ - struct __openvino_framework_map_helper_##FRAMEWORK { \ - static auto get() -> std::shared_ptr> { \ - auto make_spec_tuple = [](const std::string& s = "", \ - const std::map& attr_mp = {}, \ - const std::map& val_mp = {}) { \ - return std::make_tuple(s, attr_mp, val_mp); \ - }; \ - auto params = make_spec_tuple(__VA_ARGS__); \ - const auto& name = std::get<0>(params); \ - const auto& attr_mp = std::get<1>(params); \ - const auto& val_mp = std::get<2>(params); \ - if (!name.empty()) \ - return std::make_shared>(name, attr_mp, val_mp); \ - return std::make_shared>(attr_mp, val_mp); \ - } \ +#define OPENVINO_FRAMEWORK_MAP(FRAMEWORK, ...) \ + template \ + struct __openvino_framework_map_helper_##FRAMEWORK { \ + static auto get() -> std::shared_ptr> { \ + auto make_spec_tuple = [](const std::string& s = "", \ + const std::map& attr_mp = {}, \ + const std::map& val_mp = {}) { \ + return std::make_tuple(s, attr_mp, val_mp); \ + }; \ + FRONTEND_CAT(MAKE_MAP_, FRAMEWORK)(__VA_ARGS__) \ + } \ }; - } // namespace frontend } // namespace ov diff --git a/src/frontends/common/include/openvino/frontend/node_context.hpp b/src/frontends/common/include/openvino/frontend/node_context.hpp index f52bfe2dfe688c..0f57db65bb1f47 100644 --- a/src/frontends/common/include/openvino/frontend/node_context.hpp +++ b/src/frontends/common/include/openvino/frontend/node_context.hpp @@ -48,6 +48,12 @@ class FRONTEND_API NodeContext { FRONT_END_NOT_IMPLEMENTED(get_input); } + /// \brief Returns values from Constant input with the given index as ov::Any. + /// Throws an exception if the input cannot be represented as Constant. + virtual Any get_values_from_const_input(int idx) const { + FRONT_END_NOT_IMPLEMENTED(get_values_from_const_input); + } + virtual const std::string& get_op_type() const { return m_op_type; } @@ -111,8 +117,32 @@ class FRONTEND_API NodeContext { std::string m_op_type; }; +struct NamedOutput { + NamedOutput(const Output& _port) : port(_port) {} + NamedOutput(const std::string& _name, const Output& _port) : name(_name), port(_port) {} + + std::string name; + Output port; +}; + +using NamedOutputVector = std::vector; + +inline OutputVector indexed_from_named(const NamedOutputVector& outputs) { + OutputVector result; + result.reserve(outputs.size()); + std::transform(outputs.begin(), outputs.end(), std::back_inserter(result), [](const NamedOutput& x) { + return x.port; + }); + return result; +} + +inline NamedOutputVector named_from_indexed(const OutputVector& outputs) { + return NamedOutputVector(outputs.begin(), outputs.end()); +} + using CreatorFunction = std::function; using CreatorFunctionNamed = std::function(const NodeContext&)>; +using CreatorFunctionNamedAndIndexed = std::function; } // namespace frontend } // namespace ov diff --git a/src/frontends/ir/src/frontend.cpp b/src/frontends/ir/src/frontend.cpp index 27f5af781d0f39..0215eee96b0a33 100644 --- a/src/frontends/ir/src/frontend.cpp +++ b/src/frontends/ir/src/frontend.cpp @@ -61,10 +61,12 @@ size_t get_ir_version(std::istream& model) { } // namespace bool FrontEnd::supported_impl(const std::vector& variants) const { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; std::ifstream local_model_stream; std::istream* provided_model_stream = nullptr; - if (variants.empty() || variants.size() > 3) { + if (variants.empty() || variants.size() > 3 + extra_variants_num) { return false; } @@ -181,6 +183,7 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const weights = variant.as>(); } } + bool enable_mmap = variants[variants.size() - 1].is() ? variants[variants.size() - 1].as() : false; // Find weights if only path to xml was provided if (weights_path.empty()) { @@ -198,27 +201,31 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const } } if (!weights_path.empty()) { - std::ifstream bin_stream; - bin_stream.open(weights_path, std::ios::binary); - if (!bin_stream.is_open()) + if (enable_mmap) + weights = ov::load_mmap_object(weights_path); + else { + std::ifstream bin_stream; + bin_stream.open(weights_path, std::ios::binary); + if (!bin_stream.is_open()) #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) - IE_THROW() << "Weights file " + ov::util::wstring_to_string(weights_path) + " cannot be opened!"; + IE_THROW() << "Weights file " + ov::util::wstring_to_string(weights_path) + " cannot be opened!"; #else - IE_THROW() << "Weights file " + weights_path + " cannot be opened!"; + IE_THROW() << "Weights file " + weights_path + " cannot be opened!"; #endif - bin_stream.seekg(0, std::ios::end); - size_t file_size = bin_stream.tellg(); - bin_stream.seekg(0, std::ios::beg); + bin_stream.seekg(0, std::ios::end); + size_t file_size = bin_stream.tellg(); + bin_stream.seekg(0, std::ios::beg); - auto aligned_weights_buffer = std::make_shared(file_size); - bin_stream.read(aligned_weights_buffer->get_ptr(), aligned_weights_buffer->size()); - bin_stream.close(); + auto aligned_weights_buffer = std::make_shared(file_size); + bin_stream.read(aligned_weights_buffer->get_ptr(), aligned_weights_buffer->size()); + bin_stream.close(); - weights = std::make_shared>>( - aligned_weights_buffer->get_ptr(), - aligned_weights_buffer->size(), - aligned_weights_buffer); + weights = std::make_shared>>( + aligned_weights_buffer->get_ptr(), + aligned_weights_buffer->size(), + aligned_weights_buffer); + } } return create_input_model(); diff --git a/src/frontends/ir/src/input_model.cpp b/src/frontends/ir/src/input_model.cpp index 299dcdfb5086b0..cf919a5a5e315c 100644 --- a/src/frontends/ir/src/input_model.cpp +++ b/src/frontends/ir/src/input_model.cpp @@ -174,7 +174,9 @@ void parse_pre_process(pugi::xml_node& root, const char* data = weights->get_ptr() + offset; per_channel_values[item.first] = ngraph::opset1::Constant::create(input_type, mean_shape, data); } + OPENVINO_SUPPRESS_DEPRECATED_START auto const_node = get_constant_from_source(std::make_shared(per_channel_values, 0)); + OPENVINO_SUPPRESS_DEPRECATED_END IE_ASSERT(const_node); const auto& consumers = input_node->output(0).get_target_inputs(); auto add = std::make_shared(input_node, const_node); diff --git a/src/frontends/ir/src/ir_deserializer.cpp b/src/frontends/ir/src/ir_deserializer.cpp index 72eaeb9b07acf4..0a4ece0aae5e63 100644 --- a/src/frontends/ir/src/ir_deserializer.cpp +++ b/src/frontends/ir/src/ir_deserializer.cpp @@ -771,7 +771,7 @@ std::shared_ptr XmlDeserializer::create_node( const std::string& type_name = translate_type_name(params.type); std::shared_ptr ngraphNode; - ov::DiscreteTypeInfo type(type_name.c_str(), 0, params.version.c_str()); + ov::DiscreteTypeInfo type(type_name.c_str(), params.version.c_str()); auto extensionIt = m_extensions.find(type); if (extensionIt != m_extensions.end()) { @@ -885,7 +885,7 @@ std::shared_ptr XmlDeserializer::create_node( item.print(ss); IE_THROW() << "rt_info attribute: " << attribute_name << " has no \"version\" field: " << ss.str(); } - const auto& type_info = ov::DiscreteTypeInfo(attribute_name.c_str(), 0, attribute_version.c_str()); + const auto& type_info = ov::DiscreteTypeInfo(attribute_name.c_str(), attribute_version.c_str()); auto attr = attrs_factory.create_by_type_info(type_info); if (!attr.empty()) { if (attr.is()) { diff --git a/src/frontends/ir/src/os/win/win_mmap_object.cpp b/src/frontends/ir/src/os/win/win_mmap_object.cpp index 2a1bbd85c4693b..a3c85bd310144c 100644 --- a/src/frontends/ir/src/os/win/win_mmap_object.cpp +++ b/src/frontends/ir/src/os/win/win_mmap_object.cpp @@ -58,12 +58,16 @@ class MapHolder { } void set(const std::string& path) { + // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow + // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS) auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); map(path, h); } #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT void set(const std::wstring& path) { + // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow + // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS) auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); map(ov::util::wstring_to_string(path), h); } diff --git a/src/frontends/ir/tests/frontend_test_basic.cpp b/src/frontends/ir/tests/frontend_test_basic.cpp index 78d0099bed1461..29f412e9b21d6f 100644 --- a/src/frontends/ir/tests/frontend_test_basic.cpp +++ b/src/frontends/ir/tests/frontend_test_basic.cpp @@ -16,6 +16,15 @@ class IRFrontendTests : public ::testing::Test, public IRFrontendTestsImpl { } }; +class IRFrontendMMapTests : public ::testing::TestWithParam, public IRFrontendTestsImpl { +protected: + void SetUp() override {} + + void TearDown() override { + RemoveTemporalFiles(); + } +}; + TEST_F(IRFrontendTests, elementary_model_reading_v11) { std::string testModelV11 = R"V0G0N( @@ -241,7 +250,7 @@ TEST_F(IRFrontendTests, model_with_missing_weights) { ASSERT_THROW(core.read_model(testModelV11, ov::Tensor()), ov::Exception); } -TEST_F(IRFrontendTests, model_with_weights_reading_from_disk) { +TEST_P(IRFrontendMMapTests, model_with_weights_reading_from_disk) { std::string xmlModel = R"V0G0N( @@ -316,7 +325,9 @@ TEST_F(IRFrontendTests, model_with_weights_reading_from_disk) { std::shared_ptr model; - ASSERT_NO_THROW(model = core.read_model(xmlFileName, binFileName)); + ov::Core new_core; + new_core.set_property(ov::enable_mmap(GetParam())); + ASSERT_NO_THROW(model = new_core.read_model(xmlFileName, binFileName)); ASSERT_TRUE(!!model); std::shared_ptr modelRef; @@ -343,6 +354,8 @@ TEST_F(IRFrontendTests, model_with_weights_reading_from_disk) { EXPECT_TRUE(res.valid) << res.message; } +INSTANTIATE_TEST_SUITE_P(EnableMMapPropery, IRFrontendMMapTests, ::testing::Bool()); + TEST_F(IRFrontendTests, model_without_weights_reading_from_disk) { std::string xmlModel = R"V0G0N( diff --git a/src/frontends/onnx/frontend/CMakeLists.txt b/src/frontends/onnx/frontend/CMakeLists.txt index 2edeaae86ec594..2269c16c2e5c74 100644 --- a/src/frontends/onnx/frontend/CMakeLists.txt +++ b/src/frontends/onnx/frontend/CMakeLists.txt @@ -2,12 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - ie_add_compiler_flags(/wd4267) - ie_add_compiler_flags(/wd4018) - ie_add_compiler_flags(/wd4244) -endif() - ov_add_frontend(NAME onnx LINKABLE_FRONTEND PROTOBUF_LITE @@ -15,7 +9,7 @@ ov_add_frontend(NAME onnx FILEDESCRIPTION "FrontEnd to load and convert ONNX file format" LINK_LIBRARIES ngraph::builder onnx_common openvino::core::dev) -set(ONNX_OPSET_VERSION 17 CACHE INTERNAL "Supported version of ONNX operator set") +set(ONNX_OPSET_VERSION 18 CACHE INTERNAL "Supported version of ONNX operator set") target_compile_definitions(${TARGET_NAME} PRIVATE ONNX_OPSET_VERSION=${ONNX_OPSET_VERSION}) ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME} diff --git a/src/frontends/onnx/frontend/src/core/graph.cpp b/src/frontends/onnx/frontend/src/core/graph.cpp index e940c172e87eec..5d46da8805efe1 100644 --- a/src/frontends/onnx/frontend/src/core/graph.cpp +++ b/src/frontends/onnx/frontend/src/core/graph.cpp @@ -328,7 +328,7 @@ std::shared_ptr Graph::create_function() { const auto& onnx_outputs = m_model->get_graph().output(); for (std::size_t i{0}; i < function->get_output_size(); ++i) { const auto& result_node = function->get_output_op(i); - const std::string onnx_output_name = onnx_outputs.Get(i).name(); + const std::string onnx_output_name = onnx_outputs.Get(static_cast(i)).name(); result_node->set_friendly_name(onnx_output_name + "/sink_port_0"); const auto& previous_operation = result_node->get_input_node_shared_ptr(0); previous_operation->set_friendly_name(onnx_output_name); @@ -386,7 +386,7 @@ OutputVector Graph::make_ng_nodes(const Node& onnx_node) { const size_t outputs_size = std::accumulate(std::begin(ng_subgraph_outputs), std::end(ng_subgraph_outputs), - 0, + static_cast(0), [](const size_t lhs, const Output& rhs) { return lhs + rhs.get_node()->get_output_size(); }); @@ -420,10 +420,11 @@ void Graph::set_friendly_names(const Node& onnx_node, const OutputVector& ng_sub const auto common_node = detail::common_node_for_all_outputs(ng_subgraph_outputs); - for (size_t i = 0; i < ng_subgraph_outputs.size(); ++i) { + const auto ng_subgraph_output_size = static_cast(ng_subgraph_outputs.size()); + for (int i = 0; i < ng_subgraph_output_size; ++i) { // Trailing optional outputs may not be specified in the ONNX model. // Other optional outputs should have name set to an empty string. - if (i >= onnx_node.get_outputs_size()) { + if (i >= static_cast(onnx_node.get_outputs_size())) { break; } diff --git a/src/frontends/onnx/frontend/src/default_opset.hpp b/src/frontends/onnx/frontend/src/default_opset.hpp index 2044f25dac179a..f286ef62fd7fc5 100644 --- a/src/frontends/onnx/frontend/src/default_opset.hpp +++ b/src/frontends/onnx/frontend/src/default_opset.hpp @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/opsets/opset10.hpp" +#include "ngraph/opsets/opset11.hpp" namespace ngraph { namespace onnx_import { -namespace default_opset = ngraph::opset10; +namespace default_opset = ngraph::opset11; } } // namespace ngraph diff --git a/src/frontends/onnx/frontend/src/editor.cpp b/src/frontends/onnx/frontend/src/editor.cpp index 56ba992a5c882b..af968ac16572d5 100644 --- a/src/frontends/onnx/frontend/src/editor.cpp +++ b/src/frontends/onnx/frontend/src/editor.cpp @@ -213,9 +213,10 @@ void graph_topological_sort(GraphProto* graph) { std::multimap output_name_to_node; GraphProto result; - for (int i = 0; i < graph->node().size(); ++i) { + const auto nodes_number = static_cast(graph->node().size()); + for (int i = 0; i < nodes_number; ++i) { for (const auto& output_name : graph->node(i).output()) { - output_name_to_node.emplace(output_name, graph->mutable_node(static_cast(i))); + output_name_to_node.emplace(output_name, graph->mutable_node(i)); } } auto get_node_by_out_name = [&output_name_to_node](const std::string& out_name) -> const NodeProto* { diff --git a/src/frontends/onnx/frontend/src/op/clip.cpp b/src/frontends/onnx/frontend/src/op/clip.cpp index 77c6eceef3c431..29a207904a5798 100644 --- a/src/frontends/onnx/frontend/src/op/clip.cpp +++ b/src/frontends/onnx/frontend/src/op/clip.cpp @@ -41,7 +41,9 @@ OutputVector clip(const Node& node) { if (inputs.size() > 1 && !ngraph::op::is_null(inputs.at(1))) { min = inputs.at(1); } else { + OPENVINO_SUPPRESS_DEPRECATED_START min = ngraph::get_constant_lowest_of_type(data_type); + OPENVINO_SUPPRESS_DEPRECATED_END } // If third input is provided, assign to max input, otherwise set maximum @@ -49,7 +51,9 @@ OutputVector clip(const Node& node) { if (inputs.size() == 3 && !ngraph::op::is_null(inputs.at(2))) { max = inputs.at(2); } else { + OPENVINO_SUPPRESS_DEPRECATED_START max = ngraph::get_constant_max_of_type(data_type); + OPENVINO_SUPPRESS_DEPRECATED_END } const auto max_of_min_and_data = std::make_shared(min, data); diff --git a/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp b/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp index e31809218b6aaf..66337618e6193c 100644 --- a/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp +++ b/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp @@ -143,7 +143,9 @@ OutputVector dequantize_linear(const Output& x, NGRAPH_CHECK(x_shape.rank().is_static(), "Rank of the input data tensor has to be known (static)."); + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(node.get_description(), axis, x_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END validate_scale(scale, x, axis); const auto scale_reshaped = reshape_input(scale, axis, x_shape); diff --git a/src/frontends/onnx/frontend/src/op/dft.cpp b/src/frontends/onnx/frontend/src/op/dft.cpp index 79c3fbe0e589ef..28fbad5203e85f 100644 --- a/src/frontends/onnx/frontend/src/op/dft.cpp +++ b/src/frontends/onnx/frontend/src/op/dft.cpp @@ -4,75 +4,28 @@ #include "op/dft.hpp" -#include "default_opset.hpp" #include "onnx_import/core/null_node.hpp" #include "utils/common.hpp" +#include "utils/dft.hpp" namespace ngraph { namespace onnx_import { namespace op { namespace set_1 { - -namespace { -// For DFT, IDFT, IRDFT cases, if real data are provided (with shape [D_0, D_1, ..., D_{N-1}, 1]) -// it's needed to fill tensors with zero imaginary part to be aligned with Core ops requirements. -bool try_convert_real_to_complex(ov::Output& data) { - if (data.get_partial_shape().rank().is_static()) { - const auto length = data.get_partial_shape().rank().get_length(); - const auto last_axis_pos = length - 1; - const auto last_dim = data.get_partial_shape()[last_axis_pos]; - if (last_dim.is_static() && last_dim.get_length() == 1) { - ov::Output imag_part = default_opset::Constant::create(data.get_element_type(), {}, {0}); - imag_part = - std::make_shared(imag_part, std::make_shared(data)); - data = std::make_shared(OutputVector{data, imag_part}, last_axis_pos); - return true; - } - } - // [D_0, D_1, ..., D_{N-1}, 2] case, so additional transformations not needed or we are not able to check it during - // importing. - return false; -} -} // namespace - OutputVector dft(const Node& node) { const OutputVector ng_inputs{node.get_ng_inputs()}; - ov::Output data = ng_inputs.at(0); + const ov::Output data = ng_inputs.at(0); const auto dft_length_provided = ng_inputs.size() > 1 && !ngraph::op::is_null(ng_inputs[1]); const auto axis = node.get_attribute_value("axis", 1); - const auto axis_const = default_opset::Constant::create(element::i64, {1}, {axis}); const auto inverse = node.get_attribute_value("inverse", 0); const auto onesided = node.get_attribute_value("onesided", 0); - bool conversion_to_complex_applied = false; - if (inverse || !onesided) { // skip for RDFT case - conversion_to_complex_applied = try_convert_real_to_complex(data); - } - - ov::Output result; - if (inverse) { - if (onesided) { - result = dft_length_provided ? std::make_shared(data, axis_const, ng_inputs.at(1)) - : std::make_shared(data, axis_const); - if (conversion_to_complex_applied) { // align the output shape with a real numbers representation - const auto unsqueeze_axis = default_opset::Constant::create(element::i64, {}, {-1}); - result = std::make_shared(result, unsqueeze_axis); - } - } else { - result = dft_length_provided ? std::make_shared(data, axis_const, ng_inputs.at(1)) - : std::make_shared(data, axis_const); - } - } else { - if (onesided) { - result = dft_length_provided ? std::make_shared(data, axis_const, ng_inputs.at(1)) - : std::make_shared(data, axis_const); - } else { - result = dft_length_provided ? std::make_shared(data, axis_const, ng_inputs.at(1)) - : std::make_shared(data, axis_const); - } - } - return {result}; + return {dft::make_dft(data, + dft_length_provided ? ng_inputs.at(1) : std::make_shared(), + axis, + inverse == 1, + onesided == 1)}; } } // namespace set_1 diff --git a/src/frontends/onnx/frontend/src/op/flatten.cpp b/src/frontends/onnx/frontend/src/op/flatten.cpp index 52ded54ae3a1ba..4222fa03eed716 100644 --- a/src/frontends/onnx/frontend/src/op/flatten.cpp +++ b/src/frontends/onnx/frontend/src/op/flatten.cpp @@ -23,7 +23,9 @@ OutputVector flatten(const Node& node) { if (data_rank.is_static()) { const std::int64_t data_rank_value = data_rank.get_length(); // Accepted range is [-r, r] where r = rank(input). + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(node.get_description(), axis, data_rank_value, -data_rank_value, data_rank_value); + OPENVINO_SUPPRESS_DEPRECATED_END } return {ngraph::builder::opset1::flatten(data, static_cast(axis))}; } diff --git a/src/frontends/onnx/frontend/src/op/hardmax.cpp b/src/frontends/onnx/frontend/src/op/hardmax.cpp index eb9b421cd71ed6..0021cdc53f67d7 100644 --- a/src/frontends/onnx/frontend/src/op/hardmax.cpp +++ b/src/frontends/onnx/frontend/src/op/hardmax.cpp @@ -22,7 +22,9 @@ OutputVector hardmax(const Node& node) { auto axis = node.get_attribute_value("axis", 1); if (input_shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(node.get_description(), axis, input_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END } // reshape to 2D - "batch size" x "input feature dimensions" (NxD) @@ -61,7 +63,9 @@ OutputVector hardmax(const Node& node) { const auto& input_shape = input.get_partial_shape(); auto axis = node.get_attribute_value("axis", -1); + OPENVINO_SUPPRESS_DEPRECATED_START axis = ngraph::normalize_axis(node.get_description(), axis, input_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END const auto input_runtime_shape = std::make_shared(input); Output row_size = diff --git a/src/frontends/onnx/frontend/src/op/log_softmax.cpp b/src/frontends/onnx/frontend/src/op/log_softmax.cpp index c3b4ce405fb953..df442c8358bb02 100644 --- a/src/frontends/onnx/frontend/src/op/log_softmax.cpp +++ b/src/frontends/onnx/frontend/src/op/log_softmax.cpp @@ -37,12 +37,16 @@ OutputVector log_softmax(const Node& node, const int64_t DEFAULT_AXIS) { } case 1: { // checks if the axis belongs to the allowed values set (-1 and 0 for 1D) + OPENVINO_SUPPRESS_DEPRECATED_START ngraph::normalize_axis(node.get_description(), axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END result = std::make_shared(data, 0); break; } default: { + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_axis = ngraph::normalize_axis(node.get_description(), axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END result = onnx_logsoftmax(data, normalized_axis); break; diff --git a/src/frontends/onnx/frontend/src/op/lp_norm.cpp b/src/frontends/onnx/frontend/src/op/lp_norm.cpp index 9a1214f06f687b..8a92186548ec96 100644 --- a/src/frontends/onnx/frontend/src/op/lp_norm.cpp +++ b/src/frontends/onnx/frontend/src/op/lp_norm.cpp @@ -29,7 +29,9 @@ OutputVector lp_norm(const Node& node) { const std::int64_t p_norm{node.get_attribute_value("p", 2)}; const std::int64_t axis{node.get_attribute_value("axis", -1)}; + OPENVINO_SUPPRESS_DEPRECATED_START const size_t normalize_axis = ngraph::normalize_axis(node.get_description(), axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END CHECK_VALID_NODE(node, p_norm == 1 || p_norm == 2, diff --git a/src/frontends/onnx/frontend/src/op/mean_variance_normalization.cpp b/src/frontends/onnx/frontend/src/op/mean_variance_normalization.cpp index 304de07faa2a13..b8c3d04c5d3e83 100644 --- a/src/frontends/onnx/frontend/src/op/mean_variance_normalization.cpp +++ b/src/frontends/onnx/frontend/src/op/mean_variance_normalization.cpp @@ -31,8 +31,10 @@ namespace set_9 { OutputVector mean_variance_normalization(const Node& node) { auto data = node.get_ng_inputs().at(0); auto axes = node.get_attribute_value>("axes", {0, 2, 3}); + OPENVINO_SUPPRESS_DEPRECATED_START const std::vector normalized_axes = ngraph::normalize_axes(node.get_description(), axes, data.get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END auto const_axes = default_opset::Constant::create(element::i64, Shape{normalized_axes.size()}, normalized_axes); return { std::make_shared(data, const_axes, true, 1e-09f, ngraph::op::MVNEpsMode::OUTSIDE_SQRT)}; diff --git a/src/frontends/onnx/frontend/src/op/quantize_linear.cpp b/src/frontends/onnx/frontend/src/op/quantize_linear.cpp index 4f69691236a2ab..7bac76b0359359 100644 --- a/src/frontends/onnx/frontend/src/op/quantize_linear.cpp +++ b/src/frontends/onnx/frontend/src/op/quantize_linear.cpp @@ -88,13 +88,19 @@ std::tuple, std::shared_ptr> get_inp input_low = std::make_shared(y_scale, std::make_shared(output_low, zero_point)); - if (auto constant = ov::get_constant_from_source(input_low)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = ov::get_constant_from_source(input_low)) { + OPENVINO_SUPPRESS_DEPRECATED_END input_low = constant; + } input_high = std::make_shared(y_scale, std::make_shared(output_high, zero_point)); - if (auto constant = ov::get_constant_from_source(input_high)) + OPENVINO_SUPPRESS_DEPRECATED_START + if (auto constant = ov::get_constant_from_source(input_high)) { + OPENVINO_SUPPRESS_DEPRECATED_END input_high = constant; + } return std::make_tuple(input_low, input_high); } @@ -152,7 +158,9 @@ OutputVector quantize_linear(Output x, const auto& x_shape = x.get_partial_shape(); + OPENVINO_SUPPRESS_DEPRECATED_START axis = normalize_axis(node.get_description(), axis, x_shape.rank()); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& y_scale_shape = y_scale.get_partial_shape(); const auto& y_zero_point_shape = y_zero_point.get_partial_shape(); diff --git a/src/frontends/onnx/frontend/src/op/resize.cpp b/src/frontends/onnx/frontend/src/op/resize.cpp index f81f8a8eece512..cff54093cecd5c 100644 --- a/src/frontends/onnx/frontend/src/op/resize.cpp +++ b/src/frontends/onnx/frontend/src/op/resize.cpp @@ -24,12 +24,12 @@ static const std::unordered_set supported_transforms = {"half_pixel "asymmetric", "tf_half_pixel_for_nn"}; -using InterpolateMode = ngraph::op::v4::Interpolate::InterpolateMode; +using InterpolateMode = default_opset::Interpolate::InterpolateMode; static const std::map interp_mode_map = {{"nearest", static_cast(InterpolateMode::NEAREST)}, {"linear", static_cast(InterpolateMode::LINEAR_ONNX)}, {"cubic", static_cast(InterpolateMode::CUBIC)}}; -using Transform_mode = ngraph::op::v4::Interpolate::CoordinateTransformMode; +using Transform_mode = default_opset::Interpolate::CoordinateTransformMode; static const std::map transform_mode_map = { {"half_pixel", static_cast(Transform_mode::HALF_PIXEL)}, {"pytorch_half_pixel", static_cast(Transform_mode::PYTORCH_HALF_PIXEL)}, @@ -37,7 +37,7 @@ static const std::map transform_mode_map = { {"asymmetric", static_cast(Transform_mode::ASYMMETRIC)}, {"tf_half_pixel_for_nn", static_cast(Transform_mode::TF_HALF_PIXEL_FOR_NN)}}; -using Nearest_mode = ngraph::op::v4::Interpolate::NearestMode; +using Nearest_mode = default_opset::Interpolate::NearestMode; static const std::map nearest_mode_map = { {"round_prefer_floor", static_cast(Nearest_mode::ROUND_PREFER_FLOOR)}, {"round_prefer_ceil", static_cast(Nearest_mode::ROUND_PREFER_CEIL)}, @@ -57,9 +57,9 @@ static int mode_as_int(const std::map& converting_map, const s return result; } -using InterpolateV4Attrs = ngraph::op::v4::Interpolate::InterpolateAttrs; +using InterpolateAttrs = default_opset::Interpolate::InterpolateAttrs; -InterpolateV4Attrs get_resize_attrs(const onnx_import::Node& node) { +InterpolateAttrs get_resize_attrs(const onnx_import::Node& node) { auto get_str_attr = [&node](const std::string& name, const std::string& default_value) { return node.get_attribute_value(name, default_value); }; @@ -98,43 +98,17 @@ InterpolateV4Attrs get_resize_attrs(const onnx_import::Node& node) { supported_modes_str); } - InterpolateV4Attrs attrs; + InterpolateAttrs attrs; attrs.mode = static_cast(mode_as_int(interp_mode_map, mode)); attrs.coordinate_transformation_mode = static_cast(mode_as_int(transform_mode_map, transform_mode)); attrs.nearest_mode = static_cast(mode_as_int(nearest_mode_map, nearest_mode)); attrs.antialias = false; attrs.cube_coeff = node.get_attribute_value("cubic_coeff_a", -0.75); - - auto zero_pad = std::vector(1, 0); - - attrs.pads_begin = zero_pad; - attrs.pads_end = zero_pad; + attrs.pads_begin = {0}; + attrs.pads_end = {0}; return attrs; } - -std::shared_ptr calculate_output_shape_based_on_scales(const Output& data, - const Output& scales) { - const auto shape_of_data = std::make_shared(std::make_shared(data), - scales.get_element_type()); - const auto multiply = std::make_shared(shape_of_data, scales); - const auto output_shape = std::make_shared(multiply, ngraph::element::i64); - - return output_shape; -} - -std::shared_ptr calculate_scales_based_on_sizes(const Output& data, - const Output& sizes) { - const float epsilon = 1.0e-5f; - const auto shape_of_data = - std::make_shared(std::make_shared(data), ngraph::element::f32); - const auto converted_sizes = std::make_shared(sizes, ngraph::element::f32); - const auto divide = std::make_shared(converted_sizes, shape_of_data); - const auto eps_node = std::make_shared(ngraph::element::f32, Shape{}, epsilon); - const auto scales = std::make_shared(divide, eps_node); - - return scales; -} } // namespace namespace set_11 { @@ -149,16 +123,12 @@ OutputVector resize(const onnx_import::Node& node) { if (inputs.size() == 4 && !ngraph::op::is_null(inputs[3])) { attrs.shape_calculation_mode = default_opset::Interpolate::ShapeCalcMode::SIZES; const auto& sizes = inputs.at(3); - const auto scales = calculate_scales_based_on_sizes(data, sizes); - - return {std::make_shared(data, sizes, scales, attrs)}; + return {std::make_shared(data, sizes, attrs)}; + } else { + attrs.shape_calculation_mode = default_opset::Interpolate::ShapeCalcMode::SCALES; + const auto& scales = inputs.at(2); + return {std::make_shared(data, scales, attrs)}; } - - attrs.shape_calculation_mode = default_opset::Interpolate::ShapeCalcMode::SCALES; - - const auto& scales = inputs.at(2); - const auto output_shape = calculate_output_shape_based_on_scales(data, scales); - return {std::make_shared(data, output_shape, scales, attrs)}; } } // namespace set_11 @@ -169,6 +139,7 @@ OutputVector resize(const onnx_import::Node& node) { const auto& scales = inputs.at(1); auto attrs = get_resize_attrs(node); + attrs.shape_calculation_mode = default_opset::Interpolate::ShapeCalcMode::SIZES; if (attrs.mode == InterpolateMode::NEAREST) { attrs.nearest_mode = Nearest_mode::FLOOR; @@ -177,8 +148,12 @@ OutputVector resize(const onnx_import::Node& node) { attrs.coordinate_transformation_mode = Transform_mode::ASYMMETRIC; } - const auto output_shape = calculate_output_shape_based_on_scales(data, scales); - return {std::make_shared(data, output_shape, scales, attrs)}; + const auto shape_of_data = std::make_shared(std::make_shared(data), + scales.get_element_type()); + const auto multiply = std::make_shared(shape_of_data, scales); + const auto output_shape = std::make_shared(multiply, ngraph::element::i64); + + return {std::make_shared(data, output_shape, attrs)}; } } // namespace set_1 diff --git a/src/frontends/onnx/frontend/src/op/reverse_sequence.cpp b/src/frontends/onnx/frontend/src/op/reverse_sequence.cpp index ae9fcf09004546..d5b0de1a14b301 100644 --- a/src/frontends/onnx/frontend/src/op/reverse_sequence.cpp +++ b/src/frontends/onnx/frontend/src/op/reverse_sequence.cpp @@ -26,9 +26,13 @@ OutputVector reverse_sequence(const Node& node) { const auto data_rank = data.get_partial_shape().rank(); const auto batch_axis = node.get_attribute_value("batch_axis", 1); + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_batch_axis = ngraph::normalize_axis(node.get_description(), batch_axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END const auto time_axis = node.get_attribute_value("time_axis", 0); + OPENVINO_SUPPRESS_DEPRECATED_START const auto normalized_time_axis = ngraph::normalize_axis(node.get_description(), time_axis, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END NGRAPH_CHECK(normalized_batch_axis == 0 || normalized_batch_axis == 1, "Allowed values of the 'batch_axis' attribute for ReverseSequence " diff --git a/src/frontends/onnx/frontend/src/op/roi_align.cpp b/src/frontends/onnx/frontend/src/op/roi_align.cpp index 9b6959e126c38a..6773806fe993a7 100644 --- a/src/frontends/onnx/frontend/src/op/roi_align.cpp +++ b/src/frontends/onnx/frontend/src/op/roi_align.cpp @@ -68,9 +68,9 @@ OutputVector roi_align(const Node& node) { return {std::make_shared(data, rois, num_rois, - pooled_h, - pooled_w, - sampling_ratio, + static_cast(pooled_h), + static_cast(pooled_w), + static_cast(sampling_ratio), spatial_scale, pooling_mode, aligned_mode)}; diff --git a/src/frontends/onnx/frontend/src/op/scan.cpp b/src/frontends/onnx/frontend/src/op/scan.cpp index d986cbb3c4475a..7df3661c1e5064 100644 --- a/src/frontends/onnx/frontend/src/op/scan.cpp +++ b/src/frontends/onnx/frontend/src/op/scan.cpp @@ -51,9 +51,11 @@ OutputVector scan_to_tensor_iterator(const OutputVector& node_inputs, const auto axis_node = default_opset::Constant::create(element::i64, Shape{1}, {axis}); auto shape = node_inputs[in_idx + in_offset].get_partial_shape(); if (shape.rank().is_static()) { + OPENVINO_SUPPRESS_DEPRECATED_START axis = ov::normalize_axis(node_description, scan_input_axes[i], node_inputs[in_idx + in_offset].get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END shape[axis] = 1; } body_inputs[in_idx]->set_partial_shape(shape); @@ -81,9 +83,11 @@ OutputVector scan_to_tensor_iterator(const OutputVector& node_inputs, // Set slicing for Scan (TensorIterator) inputs for (int64_t i = 0; i < num_scan_inputs; ++i) { const auto in_idx = num_initial_values + i; + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = ov::normalize_axis(node_description, scan_input_axes[i], node_inputs[in_idx + in_offset].get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END if (scan_input_directions[i]) { // reverse direction tensor_iterator->set_sliced_input(body_inputs[in_idx], node_inputs[in_idx + in_offset], -1, -1, 1, 0, axis); } else { // forward direction @@ -100,8 +104,10 @@ OutputVector scan_to_tensor_iterator(const OutputVector& node_inputs, } for (size_t i = 0; i < num_scan_outputs; ++i) { const auto out_idx = num_initial_values + i; + OPENVINO_SUPPRESS_DEPRECATED_START const auto axis = ov::normalize_axis(node_description, scan_output_axes[i], body_outputs[out_idx].get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END if (scan_output_directions[i]) { // reverse direction outputs.push_back(tensor_iterator->get_concatenated_slices(body_outputs[out_idx], -1, -1, 1, 0, axis)); } else { // forward direction diff --git a/src/frontends/onnx/frontend/src/op/squeeze.cpp b/src/frontends/onnx/frontend/src/op/squeeze.cpp index 830c66de848a44..4444f7bf3c4447 100644 --- a/src/frontends/onnx/frontend/src/op/squeeze.cpp +++ b/src/frontends/onnx/frontend/src/op/squeeze.cpp @@ -19,7 +19,9 @@ OutputVector squeeze(const Node& node) { std::vector axes = node.get_attribute_value>("axes", {}); const auto data_rank = data.get_partial_shape().rank(); + OPENVINO_SUPPRESS_DEPRECATED_START std::vector normalized_axes = ngraph::normalize_axes(node.get_description(), axes, data_rank); + OPENVINO_SUPPRESS_DEPRECATED_END auto axes_node = std::make_shared(element::u64, Shape{normalized_axes.size()}, normalized_axes); diff --git a/src/frontends/onnx/frontend/src/op/stft.cpp b/src/frontends/onnx/frontend/src/op/stft.cpp new file mode 100644 index 00000000000000..016aba8a9bbd16 --- /dev/null +++ b/src/frontends/onnx/frontend/src/op/stft.cpp @@ -0,0 +1,124 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op/stft.hpp" + +#include "default_opset.hpp" +#include "exceptions.hpp" +#include "onnx_import/core/null_node.hpp" +#include "utils/common.hpp" +#include "utils/dft.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_17 { + +OutputVector stft(const Node& node) { + const OutputVector ng_inputs{node.get_ng_inputs()}; + auto signal = ng_inputs.at(0); + const auto dft_length_provided = ng_inputs.size() > 3 && !ngraph::op::is_null(ng_inputs[3]); + const auto onesided = node.get_attribute_value("onesided", 1); + const int64_t axis = 1; + + const auto& frame_step_node = ng_inputs.at(1); + CHECK_VALID_NODE(node, + ngraph::op::is_constant(frame_step_node.get_node_shared_ptr()) && + ov::shape_size(frame_step_node.get_shape()) <= 1, + "frame_step input must be a scalar or Shape{1} constant."); + const auto frame_step = + ov::as_type_ptr(frame_step_node.get_node_shared_ptr())->cast_vector()[0]; + const auto signal_param_shape = signal.get_partial_shape(); + CHECK_VALID_NODE(node, + signal_param_shape.is_static() && signal_param_shape.size() == 3, + "Shape of signal input must be static with the rank equal to 3."); + + int64_t frame_length = signal_param_shape[axis].get_length() / frame_step; // default value + if (dft_length_provided) { + const auto& frame_length_node = ng_inputs[3]; + CHECK_VALID_NODE(node, + ngraph::op::is_constant(frame_length_node.get_node_shared_ptr()) && + ov::shape_size(frame_length_node.get_shape()) <= 1, + "frame_length input must be a scalar or Shape{1} constant."); + frame_length = ov::as_type_ptr(frame_length_node.get_node_shared_ptr()) + ->cast_vector()[0]; + } + + const auto window_node_provided = ng_inputs.size() > 2 && !ngraph::op::is_null(ng_inputs[2]); + if (window_node_provided) { // window input provided + if (ng_inputs[2].get_partial_shape().rank().is_static()) { + CHECK_VALID_NODE(node, + ng_inputs[2].get_partial_shape().rank().get_length() == 1, + "The rank of window input must be 1D."); + if (ng_inputs[2].get_partial_shape()[0].is_static()) { + CHECK_VALID_NODE(node, + ng_inputs[2].get_partial_shape()[0].get_length() == frame_length, + "The length of window input must be equal to frame_length."); + } + } + } + const auto is_complex = [](const ov::Output& data) { + return data.get_partial_shape().rank().is_static() && (data.get_partial_shape().cend() - 1)->is_static() && + (data.get_partial_shape().cend() - 1)->get_length() == 2; + }; + if (onesided == 1) { + CHECK_VALID_NODE(node, !is_complex(signal), "If attribute onesided==1, signal input can NOT be complex."); + } + const int64_t batch_size = signal_param_shape[0].get_length(); + const auto nstfts = static_cast((signal_param_shape[axis].get_length() - frame_length) / frame_step) + 1; + const auto axis_const = default_opset::Constant::create(element::i64, {}, {axis}); + const auto zero_const = default_opset::Constant::create(element::i64, {}, {0}); + const auto step = default_opset::Constant::create(element::i64, Shape{2}, {1, 1}); + ov::OutputVector all_signals; + for (int64_t batch = 0; batch < batch_size; ++batch) { + ov::OutputVector signals_in_batch; + for (int64_t sig_idx = 0; sig_idx < nstfts; ++sig_idx) { + const auto start = default_opset::Constant::create(element::i64, + Shape{2}, + std::vector{batch, sig_idx * frame_step}); + const auto stop = + default_opset::Constant::create(element::i64, + Shape{2}, + std::vector{batch + 1, sig_idx * frame_step + frame_length}); + const auto slice_axes = + default_opset::Constant::create(element::i64, Shape{2}, std::vector{0, axis}); + const auto slice = std::make_shared(signal, start, stop, step, slice_axes); + const ov::Output flatten_slice = std::make_shared( + slice, + is_complex(slice) ? default_opset::Constant::create(element::i64, {2}, {-1, 2}) + : (onesided ? default_opset::Constant::create(element::i64, {1}, {-1}) + : default_opset::Constant::create(element::i64, {2}, {-1, 1})), + false); + const auto dft = dft::make_dft( + window_node_provided + ? std::make_shared( + flatten_slice, + is_complex(flatten_slice) + ? std::make_shared( // align window shape with signal shape + std::make_shared( + ng_inputs[2], + default_opset::Constant::create(element::i64, {1}, {1})), + std::make_shared(flatten_slice)) + : ng_inputs[2]) + : flatten_slice, + dft_length_provided ? ng_inputs[3] : std::make_shared(), + 0, + false, + onesided == 1); + signals_in_batch.push_back(std::make_shared(dft, zero_const)); + } + all_signals.push_back( + std::make_shared(std::make_shared(signals_in_batch, 0), + zero_const)); + } + return {std::make_shared(all_signals, 0)}; +} + +} // namespace set_17 + +} // namespace op + +} // namespace onnx_import + +} // namespace ngraph diff --git a/src/frontends/onnx/frontend/src/op/stft.hpp b/src/frontends/onnx/frontend/src/op/stft.hpp new file mode 100644 index 00000000000000..c288c99419e5de --- /dev/null +++ b/src/frontends/onnx/frontend/src/op/stft.hpp @@ -0,0 +1,22 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/node.hpp" +#include "onnx_import/core/node.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_17 { +OutputVector stft(const Node& node); + +} // namespace set_17 + +} // namespace op + +} // namespace onnx_import + +} // namespace ngraph diff --git a/src/frontends/onnx/frontend/src/op/upsample.cpp b/src/frontends/onnx/frontend/src/op/upsample.cpp index 83766e3635138e..2da7bd407c7b9a 100644 --- a/src/frontends/onnx/frontend/src/op/upsample.cpp +++ b/src/frontends/onnx/frontend/src/op/upsample.cpp @@ -38,34 +38,21 @@ void check_mode_support(const onnx_import::Node& node, const std::string& mode, } default_opset::Interpolate::InterpolateAttrs get_attributes(const std::string& mode) { - using InterpolateMode = default_opset::Interpolate::InterpolateMode; - using Transform_mode = default_opset::Interpolate::CoordinateTransformMode; - using ShapeCalcMode = default_opset::Interpolate::ShapeCalcMode; + const auto interpolate_mode = mode == "linear" || mode == "bilinear" + ? default_opset::Interpolate::InterpolateMode::LINEAR_ONNX + : default_opset::Interpolate::InterpolateMode::NEAREST; - const auto interpolate_mode = - (mode == "linear" || mode == "bilinear" ? InterpolateMode::LINEAR_ONNX : InterpolateMode::NEAREST); + auto attrs = default_opset::Interpolate::InterpolateAttrs(interpolate_mode, + default_opset::Interpolate::ShapeCalcMode::SCALES, + {0}, + {0}); - std::vector pad{0}; - auto attrs = default_opset::Interpolate::InterpolateAttrs(interpolate_mode, ShapeCalcMode::SCALES, pad, pad); - - if (attrs.mode == InterpolateMode::LINEAR_ONNX) - attrs.coordinate_transformation_mode = Transform_mode::ASYMMETRIC; + if (attrs.mode == default_opset::Interpolate::InterpolateMode::LINEAR_ONNX) { + attrs.coordinate_transformation_mode = default_opset::Interpolate::CoordinateTransformMode::ASYMMETRIC; + } return attrs; } - -OutputVector create_upsample_subgraph(const Output& data, - const Output& scales, - const std::string& mode) { - const auto shape_of_data = - std::make_shared(std::make_shared(data), ngraph::element::f32); - const auto multiply = std::make_shared(shape_of_data, scales); - const auto output_shape = std::make_shared(std::make_shared(multiply), - ngraph::element::i64); - - return {std::make_shared(data, output_shape, scales, get_attributes(mode))}; -} - } // namespace namespace set_1 { @@ -89,7 +76,7 @@ OutputVector upsample(const onnx_import::Node& node) { const auto scales_const = default_opset::Constant::create(ngraph::element::f32, Shape({scales.size()}), scales); - return create_upsample_subgraph(data, scales_const, mode); + return std::make_shared(data, scales_const, get_attributes(mode))->outputs(); } } // namespace set_1 @@ -110,7 +97,7 @@ OutputVector upsample(const onnx_import::Node& node) { const auto scales_const = default_opset::Constant::create(ngraph::element::f32, Shape({scales.size()}), scales); - return create_upsample_subgraph(data, scales_const, mode); + return std::make_shared(data, scales_const, get_attributes(mode))->outputs(); } } // namespace set_7 @@ -120,17 +107,8 @@ OutputVector upsample(const onnx_import::Node& node) { const auto mode = node.get_attribute_value("mode", "nearest"); check_mode_support(node, mode, version_9); - const auto inputs = node.get_ng_inputs(); - const auto& data = inputs.at(0); - const auto& scales = inputs.at(1); - - const auto& data_shape = data.get_partial_shape(); - const auto& scales_shape = scales.get_partial_shape(); - CHECK_VALID_NODE(node, - (scales_shape.is_static() || data_shape.rank().is_static()), - " Data rank or shape of Scales input is required to be static."); - - return create_upsample_subgraph(data, scales, mode); + const auto& inputs = node.get_ng_inputs(); + return std::make_shared(inputs.at(0), inputs.at(1), get_attributes(mode))->outputs(); } } // namespace set_9 diff --git a/src/frontends/onnx/frontend/src/ops_bridge.cpp b/src/frontends/onnx/frontend/src/ops_bridge.cpp index 655d4efda7a294..c9073f43013c02 100644 --- a/src/frontends/onnx/frontend/src/ops_bridge.cpp +++ b/src/frontends/onnx/frontend/src/ops_bridge.cpp @@ -161,6 +161,7 @@ #include "op/split.hpp" #include "op/sqrt.hpp" #include "op/squeeze.hpp" +#include "op/stft.hpp" #include "op/sub.hpp" #include "op/sum.hpp" #include "op/tan.hpp" @@ -479,6 +480,10 @@ OperatorsBridge::OperatorsBridge() { REGISTER_OPERATOR("SpaceToDepth", 1, space_to_depth); REGISTER_OPERATOR("Split", 1, split); REGISTER_OPERATOR("Split", 13, split); + register_operator("STFT", + VersionRange::single_version_for_all_opsets(), + op::set_17::stft, + "frame_step and frame_length inputs must be constants; signal shape must be static;"); REGISTER_OPERATOR("Sqrt", 1, sqrt); REGISTER_OPERATOR("Squeeze", 1, squeeze); REGISTER_OPERATOR("Squeeze", 13, squeeze); diff --git a/src/frontends/onnx/frontend/src/place.cpp b/src/frontends/onnx/frontend/src/place.cpp index 6b67cf7efcfd03..3430bf4e6b56cb 100644 --- a/src/frontends/onnx/frontend/src/place.cpp +++ b/src/frontends/onnx/frontend/src/place.cpp @@ -317,8 +317,8 @@ ov::frontend::Place::Ptr PlaceOp::get_input_port(const std::string& input_name) std::vector PlaceOp::get_consuming_ports() const { std::vector consuming_ports; - const auto out_ports_number = m_editor->get_output_ports(m_node).size(); - for (size_t out_idx = 0; out_idx < out_ports_number; ++out_idx) { + const auto out_ports_number = static_cast(m_editor->get_output_ports(m_node).size()); + for (int out_idx = 0; out_idx < out_ports_number; ++out_idx) { auto consuming_ops_out = get_output_port(out_idx)->get_consuming_ports(); consuming_ports.insert(consuming_ports.end(), consuming_ops_out.begin(), consuming_ops_out.end()); } diff --git a/src/frontends/onnx/frontend/src/utils/arg_min_max_factory.cpp b/src/frontends/onnx/frontend/src/utils/arg_min_max_factory.cpp index d5a3fdb827061f..ed87102ac20b68 100644 --- a/src/frontends/onnx/frontend/src/utils/arg_min_max_factory.cpp +++ b/src/frontends/onnx/frontend/src/utils/arg_min_max_factory.cpp @@ -53,8 +53,10 @@ std::shared_ptr ArgMinMaxFactory::make_topk_subgraph(default_opset // res_index = dims_on_axis - topk->output(1) = 6 - 3 = 3 // result = res_index - 1 = 3 - 1 = 2 + OPENVINO_SUPPRESS_DEPRECATED_START const int64_t normalized_axis = normalize_axis(m_input_node.get_node(), m_axis, m_input_node.get_partial_shape().rank()); + OPENVINO_SUPPRESS_DEPRECATED_END const auto axis_node = default_opset::Constant::create(ngraph::element::i64, Shape{1}, {normalized_axis}); const auto reverse = std::make_shared(m_input_node, axis_node, opset1::Reverse::Mode::INDEX); diff --git a/src/frontends/onnx/frontend/src/utils/convpool.cpp b/src/frontends/onnx/frontend/src/utils/convpool.cpp index 92f387f102ee9c..5aecbe8a73ce20 100644 --- a/src/frontends/onnx/frontend/src/utils/convpool.cpp +++ b/src/frontends/onnx/frontend/src/utils/convpool.cpp @@ -137,6 +137,7 @@ void calculate_auto_pads(const Shape& data_shape, padding_above.clear(); // Extract kernel shape - remove (N,C) channels Shape kernel_shape(std::next(std::begin(filter_shape), 2), std::end(filter_shape)); + OPENVINO_SUPPRESS_DEPRECATED_START ngraph::infer_auto_padding(data_shape, kernel_shape, strides, @@ -144,6 +145,7 @@ void calculate_auto_pads(const Shape& data_shape, pad_type, padding_above, padding_below); + OPENVINO_SUPPRESS_DEPRECATED_END } } diff --git a/src/frontends/onnx/frontend/src/utils/dft.cpp b/src/frontends/onnx/frontend/src/utils/dft.cpp new file mode 100644 index 00000000000000..d025516d15e57e --- /dev/null +++ b/src/frontends/onnx/frontend/src/utils/dft.cpp @@ -0,0 +1,77 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "dft.hpp" + +#include "default_opset.hpp" +#include "onnx_import/core/null_node.hpp" + +using namespace ngraph::onnx_import; + +namespace ngraph { +namespace onnx_import { +namespace dft { + +namespace { +// For DFT, IDFT, IRDFT cases, if real signal are provided (with shape [D_0, D_1, ..., D_{N-1}, 1]) +// it's needed to fill tensors with zero imaginary part to be aligned with Core ops requirements. +bool try_convert_real_to_complex(ov::Output& signal) { + if (signal.get_partial_shape().rank().is_static()) { + const auto length = signal.get_partial_shape().rank().get_length(); + const auto last_axis_pos = length - 1; + const auto last_dim = signal.get_partial_shape()[last_axis_pos]; + if (last_dim.is_static() && last_dim.get_length() == 1) { + ov::Output imag_part = default_opset::Constant::create(signal.get_element_type(), {}, {0}); + imag_part = + std::make_shared(imag_part, std::make_shared(signal)); + signal = std::make_shared(OutputVector{signal, imag_part}, last_axis_pos); + return true; + } + } + // [D_0, D_1, ..., D_{N-1}, 2] case, so additional transformations not needed or we are not able to check it during + // importing. + return false; +} +} // namespace + +ov::Output make_dft(const ov::Output& signal, + const ov::Output& length, + int64_t axis, + bool is_inversed, + bool is_onesided) { + auto processed_signal = signal; + const auto axis_const = default_opset::Constant::create(element::i64, {1}, {axis}); + bool conversion_to_complex_applied = false; + if (is_inversed || !is_onesided) { // skip for RDFT case + conversion_to_complex_applied = try_convert_real_to_complex(processed_signal); + } + bool dft_length_provided = !ngraph::op::is_null(length); + + ov::Output result; + if (is_inversed) { + if (is_onesided) { + result = dft_length_provided ? std::make_shared(processed_signal, axis_const, length) + : std::make_shared(processed_signal, axis_const); + if (conversion_to_complex_applied) { // align the output shape with a real numbers representation + const auto unsqueeze_axis = default_opset::Constant::create(element::i64, {}, {-1}); + result = std::make_shared(result, unsqueeze_axis); + } + } else { + result = dft_length_provided ? std::make_shared(processed_signal, axis_const, length) + : std::make_shared(processed_signal, axis_const); + } + } else { + if (is_onesided) { + result = dft_length_provided ? std::make_shared(processed_signal, axis_const, length) + : std::make_shared(processed_signal, axis_const); + } else { + result = dft_length_provided ? std::make_shared(processed_signal, axis_const, length) + : std::make_shared(processed_signal, axis_const); + } + } + return {result}; +} +} // namespace dft +} // namespace onnx_import +} // namespace ngraph diff --git a/src/frontends/onnx/frontend/src/utils/dft.hpp b/src/frontends/onnx/frontend/src/utils/dft.hpp new file mode 100644 index 00000000000000..6ab0b01580df43 --- /dev/null +++ b/src/frontends/onnx/frontend/src/utils/dft.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/node.hpp" + +namespace ngraph { +namespace onnx_import { +namespace dft { + +ov::Output make_dft(const ov::Output& signal, + const ov::Output& length, + int64_t axis, + bool is_inversed, + bool is_one_sided); +} // namespace dft +} // namespace onnx_import +} // namespace ngraph diff --git a/src/frontends/onnx/tests/CMakeLists.txt b/src/frontends/onnx/tests/CMakeLists.txt index ac41687b9b47f2..597bdd33a76c0f 100644 --- a/src/frontends/onnx/tests/CMakeLists.txt +++ b/src/frontends/onnx/tests/CMakeLists.txt @@ -6,13 +6,6 @@ set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF) ov_try_use_gold_linker() -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - ie_add_compiler_flags(/wd4244) - ie_add_compiler_flags(/wd4267) - ie_add_compiler_flags(/wd4305) - ie_add_compiler_flags(/wd4756) -endif() - message(STATUS "ONNX frontend test enabled") add_compile_definitions( @@ -75,6 +68,7 @@ set(MULTI_TEST_SRC onnx_import_org_pytorch.in.cpp onnx_import_reshape.in.cpp onnx_import_rnn.in.cpp + onnx_import_signal.in.cpp onnx_import_quant.in.cpp onnx_test_utils.in.cpp onnx_import_with_editor.in.cpp) diff --git a/src/frontends/onnx/tests/models/stft_dynamic_signal_shape.prototxt b/src/frontends/onnx/tests/models/stft_dynamic_signal_shape.prototxt new file mode 100644 index 00000000000000..193223dd12514b --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_dynamic_signal_shape.prototxt @@ -0,0 +1,99 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + output: "frame_length" + name: "frame_length_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 16 + name: "frame_length_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 1 + type: INT + } + } + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + } + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 16 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_given_window_default_length.prototxt b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_given_window_default_length.prototxt new file mode 100644 index 00000000000000..8357764b45a5c2 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_given_window_default_length.prototxt @@ -0,0 +1,85 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 4 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 0 + type: INT + } + } + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 32 + } + dim { + dim_value: 2 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 8 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_given_window_no_default_length.prototxt b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_given_window_no_default_length.prototxt new file mode 100644 index 00000000000000..6ff6e7f25e4c24 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_given_window_no_default_length.prototxt @@ -0,0 +1,101 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 12 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + output: "frame_length" + name: "frame_length_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 10 + name: "frame_length_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 0 + type: INT + } + } + name: "g" + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 48 + } + dim { + dim_value: 2 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 10 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_default_length.prototxt b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_default_length.prototxt new file mode 100644 index 00000000000000..a8308903c9ae85 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_default_length.prototxt @@ -0,0 +1,72 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 0 + type: INT + } + } + name: "g" + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 128 + } + dim { + dim_value: 2 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_default_length_2.prototxt b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_default_length_2.prototxt new file mode 100644 index 00000000000000..f6c615292ea62d --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_default_length_2.prototxt @@ -0,0 +1,72 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 0 + type: INT + } + } + name: "g" + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 64 + } + dim { + dim_value: 2 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_no_default_length.prototxt b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_no_default_length.prototxt new file mode 100644 index 00000000000000..4589ef8fd71b0a --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_no_onesided_complex_input_no_window_no_default_length.prototxt @@ -0,0 +1,87 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + output: "frame_length" + name: "frame_length_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 24 + name: "frame_length_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 0 + type: INT + } + } + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 4 + } + dim { + dim_value: 32 + } + dim { + dim_value: 2 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_no_onesided_real_input_no_window_default_length.prototxt b/src/frontends/onnx/tests/models/stft_no_onesided_real_input_no_window_default_length.prototxt new file mode 100644 index 00000000000000..74e24441fc71ba --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_no_onesided_real_input_no_window_default_length.prototxt @@ -0,0 +1,72 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 0 + type: INT + } + } + name: "g" + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 128 + } + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_non_const_frame_length.prototxt b/src/frontends/onnx/tests/models/stft_non_const_frame_length.prototxt new file mode 100644 index 00000000000000..de4e8479301a81 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_non_const_frame_length.prototxt @@ -0,0 +1,98 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 1 + type: INT + } + } + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 32 + } + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 16 + } + } + } + } + } + input { + name: "frame_length" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 16 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_non_const_frame_step.prototxt b/src/frontends/onnx/tests/models/stft_non_const_frame_step.prototxt new file mode 100644 index 00000000000000..9e52d2d5859de6 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_non_const_frame_step.prototxt @@ -0,0 +1,99 @@ +ir_version: 8 +graph { + node { + output: "frame_length" + name: "frame_length_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 16 + name: "frame_length_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 1 + type: INT + } + } + name: "g" + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 32 + } + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "frame_step" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 16 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_onesided_complex_input.prototxt b/src/frontends/onnx/tests/models/stft_onesided_complex_input.prototxt new file mode 100644 index 00000000000000..adf0584ac97d94 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_onesided_complex_input.prototxt @@ -0,0 +1,100 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + output: "frame_length" + name: "frame_length_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 16 + name: "frame_length_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 1 + type: INT + } + } + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 32 + } + dim { + dim_value: 2 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 16 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_onesided_real_input_given_window_no_default_length.prototxt b/src/frontends/onnx/tests/models/stft_onesided_real_input_given_window_no_default_length.prototxt new file mode 100644 index 00000000000000..dc0e5236524408 --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_onesided_real_input_given_window_no_default_length.prototxt @@ -0,0 +1,100 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + output: "frame_length" + name: "frame_length_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 16 + name: "frame_length_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "window" + input: "frame_length" + output: "Y" + op_type: "STFT" + attribute { + name: "onesided" + i: 1 + type: INT + } + } + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 32 + } + dim { + dim_value: 1 + } + } + } + } + } + input { + name: "window" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 16 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/models/stft_onesided_real_input_no_window_default_length.prototxt b/src/frontends/onnx/tests/models/stft_onesided_real_input_no_window_default_length.prototxt new file mode 100644 index 00000000000000..d736d9b4089f4a --- /dev/null +++ b/src/frontends/onnx/tests/models/stft_onesided_real_input_no_window_default_length.prototxt @@ -0,0 +1,67 @@ +ir_version: 8 +graph { + node { + output: "frame_step" + name: "frame_step_const" + op_type: "Constant" + attribute { + name: "value" + t { + dims: 1 + data_type: 7 + int64_data: 8 + name: "frame_step_tensor" + } + type: TENSOR + } + } + node { + input: "signal" + input: "frame_step" + input: "" + output: "Y" + op_type: "STFT" + } + name: "g" + input { + name: "signal" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 1 + } + dim { + dim_value: 128 + } + dim { + dim_value: 1 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + } + dim { + } + dim { + } + dim { + } + } + } + } + } +} +opset_import { + domain: "" + version: 17 +} diff --git a/src/frontends/onnx/tests/onnx_import.in.cpp b/src/frontends/onnx/tests/onnx_import.in.cpp index 91ce448c02c06f..3a991b69a6a734 100644 --- a/src/frontends/onnx/tests/onnx_import.in.cpp +++ b/src/frontends/onnx/tests/onnx_import.in.cpp @@ -469,7 +469,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_expand_function_greater_or_equal_inside_if) { // expected value == x * y std::vector x(40, 2); std::vector y(40); - std::iota(y.begin(), y.end(), -20); + std::iota(y.begin(), y.end(), -20.f); std::vector expected; std::transform(x.begin(), x.end(), y.begin(), std::back_inserter(expected), [](float i, float j) -> float { return i * j; @@ -489,21 +489,21 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_expand_context_dependent_function) { auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{3, 5}, - {0.54881352186203, - 0.7151893377304077, - 0.6027633547782898, - 0.5448831915855408, - 0.42365479469299316, - 0.6458941102027893, - 0.4375872015953064, - 0.891772985458374, - 0.9636627435684204, - 0.3834415078163147, - 0.7917250394821167, - 0.5288949012756348, - 0.5680445432662964, - 0.9255966544151306, - 0.07103605568408966}); + {0.54881352186203f, + 0.7151893377304077f, + 0.6027633547782898f, + 0.5448831915855408f, + 0.42365479469299316f, + 0.6458941102027893f, + 0.4375872015953064f, + 0.891772985458374f, + 0.9636627435684204f, + 0.3834415078163147f, + 0.7917250394821167f, + 0.5288949012756348f, + 0.5680445432662964f, + 0.9255966544151306f, + 0.07103605568408966f}); test_case.add_input(Shape{3}, {1, 4, 3}); test_case.add_expected_output(Shape{}, {1}); test_case.run(); @@ -797,27 +797,27 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softmax_1D) { auto test_case = test::TestCase(function, s_device); test_case.add_input({-1.0, 0.0, 1.0}); - test_case.add_expected_output({0.09003058, 0.24472848, 0.66524094}); + test_case.add_expected_output({0.09003058f, 0.24472848f, 0.66524094f}); test_case.run(); } namespace { // common input for all Softmax 3D test cases (Shape = {3,4,5}) // clang-format off const std::vector SOFTMAX_INPUT = { - 2.75793882, -0.50841322, 0.82013929, -0.62409912, -0.96136118, - 0.21004745, 1.38337255, 1.19030397, 2.0940445, -0.03551657, - -0.78686039, 1.992782, 0.04300319, -0.29230777, -0.56797112, - -1.26732165, -0.61935399, 0.57670432, 0.92844898, 2.82469233, - - 0.98721677, -0.05100663, -1.21178917, -0.17530157, 1.40051805, - -0.13259761, -1.14313018, 0.2673723, -0.87996154, 1.29053106, - 1.55, 0.8396538, 1.20729817, 0.23727845, -0.89113606, - -1.70909842, 0.26460363, -0.70566808, 2.383518, 1.07024615, - - -1.21722605, 0.82919357, 0.55765697, 0.12657686, 0.63432172, - 0.75425957, -2.43721014, -1.24478184, 2.65316853, 1.19509542, - -0.95523998, 0.5149006, -0.01151649, 0.68327026, -0.4589638, - -0.46554745, 0.21055324, 0.39266729, 2.05098086, 1.83207919}; + 2.75793882f, -0.50841322f, 0.82013929f, -0.62409912f, -0.96136118f, + 0.21004745f, 1.38337255f, 1.19030397f, 2.0940445f, -0.03551657f, + -0.78686039f, 1.992782f, 0.04300319f, -0.29230777f, -0.56797112f, + -1.26732165f, -0.61935399f, 0.57670432f, 0.92844898f, 2.82469233f, + + 0.98721677f, -0.05100663f, -1.21178917f, -0.17530157f, 1.40051805f, + -0.13259761f, -1.14313018f, 0.2673723f, -0.87996154f, 1.29053106f, + 1.55f, 0.8396538f, 1.20729817f, 0.23727845f, -0.89113606f, + -1.70909842f, 0.26460363f, -0.70566808f, 2.383518f, 1.07024615f, + + -1.21722605f, 0.82919357f, 0.55765697f, 0.12657686f, 0.63432172f, + 0.75425957f, -2.43721014f, -1.24478184f, 2.65316853f, 1.19509542f, + -0.95523998f, 0.5149006f, -0.01151649f, 0.68327026f, -0.4589638f, + -0.46554745f, 0.21055324f, 0.39266729f, 2.05098086f, 1.83207919f}; } // namespace // clang-format on @@ -831,20 +831,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softmax_axis_0) { // clang-format off test_case.add_expected_output( Shape{3, 4, 5}, - {0.09683057, 0.00369363, 0.01394559, 0.00329012, 0.00234823, - 0.00757665, 0.02449322, 0.02019284, 0.04985249, 0.00592694, - 0.00279593, 0.04505148, 0.00641108, 0.00458466, 0.00348007, - 0.00172928, 0.00330577, 0.01093237, 0.01554086, 0.10351497, - - 0.01648154, 0.00583583, 0.00182802, 0.00515374, 0.02491679, - 0.00537859, 0.00195794, 0.00802367, 0.00254737, 0.0223216, - 0.02893419, 0.0142204, 0.02053893, 0.00778581, 0.00251907, - 0.00111174, 0.00800149, 0.0030324, 0.06658917, 0.0179084, - - 0.00181811, 0.01407243, 0.01072611, 0.0069699, 0.01158077, - 0.01305647, 0.00053677, 0.0017687, 0.08719896, 0.02028982, - 0.00236265, 0.01027717, 0.0060709, 0.01216173, 0.00388087, - 0.00385541, 0.00758048, 0.00909469, 0.04775123, 0.03836337}); + {0.09683057f, 0.00369363f, 0.01394559f, 0.00329012f, 0.00234823f, + 0.00757665f, 0.02449322f, 0.02019284f, 0.04985249f, 0.00592694f, + 0.00279593f, 0.04505148f, 0.00641108f, 0.00458466f, 0.00348007f, + 0.00172928f, 0.00330577f, 0.01093237f, 0.01554086f, 0.10351497f, + + 0.01648154f, 0.00583583f, 0.00182802f, 0.00515374f, 0.02491679f, + 0.00537859f, 0.00195794f, 0.00802367f, 0.00254737f, 0.0223216f, + 0.02893419f, 0.0142204f, 0.02053893f, 0.00778581f, 0.00251907f, + 0.00111174f, 0.00800149f, 0.0030324f, 0.06658917f, 0.0179084f, + + 0.00181811f, 0.01407243f, 0.01072611f, 0.0069699f, 0.01158077f, + 0.01305647f, 0.00053677f, 0.0017687f, 0.08719896f, 0.02028982f, + 0.00236265f, 0.01027717f, 0.0060709f, 0.01216173f, 0.00388087f, + 0.00385541f, 0.00758048f, 0.00909469f, 0.04775123f, 0.03836337f}); // clang-format on test_case.run(6); @@ -860,20 +860,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softmax_axis_1) { // clang-format off test_case.add_expected_output( Shape{3, 4, 5}, - {0.22757064, 0.00868076, 0.03277484, 0.00773243, 0.0055188, - 0.0178066, 0.05756383, 0.04745709, 0.11716303, 0.01392945, - 0.00657097, 0.10587974, 0.01506727, 0.01077484, 0.00817884, - 0.00406413, 0.00776921, 0.0256932, 0.03652405, 0.24328028, - - 0.06217413, 0.02201481, 0.00689594, 0.01944171, 0.09399488, - 0.02028993, 0.00738604, 0.03026811, 0.00960958, 0.08420492, - 0.10914991, 0.05364435, 0.07748005, 0.02937079, 0.0095028, - 0.00419387, 0.03018442, 0.01143929, 0.2511977, 0.06755678, - - 0.00587593, 0.04548053, 0.0346656, 0.02252594, 0.03742775, - 0.04219705, 0.00173478, 0.00571623, 0.2818174, 0.06557446, - 0.00763582, 0.03321466, 0.01962049, 0.03930537, 0.01254255, - 0.01246025, 0.02449929, 0.02939305, 0.15432668, 0.12398617}); + {0.22757064f, 0.00868076f, 0.03277484f, 0.00773243f, 0.0055188f, + 0.0178066f, 0.05756383f, 0.04745709f, 0.11716303f, 0.01392945f, + 0.00657097f, 0.10587974f, 0.01506727f, 0.01077484f, 0.00817884f, + 0.00406413f, 0.00776921f, 0.0256932f, 0.03652405f, 0.24328028f, + + 0.06217413f, 0.02201481f, 0.00689594f, 0.01944171f, 0.09399488f, + 0.02028993f, 0.00738604f, 0.03026811f, 0.00960958f, 0.08420492f, + 0.10914991f, 0.05364435f, 0.07748005f, 0.02937079f, 0.0095028f, + 0.00419387f, 0.03018442f, 0.01143929f, 0.2511977f, 0.06755678f, + + 0.00587593f, 0.04548053f, 0.0346656f, 0.02252594f, 0.03742775f, + 0.04219705f, 0.00173478f, 0.00571623f, 0.2818174f, 0.06557446f, + 0.00763582f, 0.03321466f, 0.01962049f, 0.03930537f, 0.01254255f, + 0.01246025f, 0.02449929f, 0.02939305f, 0.15432668f, 0.12398617f}); // clang-format on test_case.run(4); @@ -890,20 +890,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softmax_axis_1_opset11) { // clang-format off test_case.add_expected_output( Shape{3, 4, 5}, - {0.88890495, 0.04825497, 0.27088348, 0.04490523, 0.02037154, - 0.06955369, 0.31998834, 0.39223197, 0.68041159, 0.05141776, - 0.02566661, 0.5885689, 0.12453075, 0.06257374, 0.03019055, - 0.01587475, 0.0431878, 0.21235381, 0.21210944, 0.89802015, - - 0.31752626, 0.19442629, 0.0546935, 0.06279221, 0.36823282, - 0.10362164, 0.06523066, 0.24006419, 0.03103672, 0.32987983, - 0.55743381, 0.473766, 0.61451431, 0.09486084, 0.03722801, - 0.02141829, 0.26657706, 0.090728, 0.81131024, 0.26465935, - - 0.08619648, 0.43343993, 0.3877785, 0.04523505, 0.15625437, - 0.61900597, 0.01653285, 0.06394322, 0.56592636, 0.27376196, - 0.11201305, 0.31654337, 0.21947994, 0.07893034, 0.05236297, - 0.18278451, 0.23348385, 0.32879834, 0.30990825, 0.5176207}); + {0.88890495f, 0.04825497f, 0.27088348f, 0.04490523f, 0.02037154f, + 0.06955369f, 0.31998834f, 0.39223197f, 0.68041159f, 0.05141776f, + 0.02566661f, 0.5885689f, 0.12453075f, 0.06257374f, 0.03019055f, + 0.01587475f, 0.0431878f, 0.21235381f, 0.21210944f, 0.89802015f, + + 0.31752626f, 0.19442629f, 0.0546935f, 0.06279221f, 0.36823282f, + 0.10362164f, 0.06523066f, 0.24006419f, 0.03103672f, 0.32987983f, + 0.55743381f, 0.473766f, 0.61451431f, 0.09486084f, 0.03722801f, + 0.02141829f, 0.26657706f, 0.090728f, 0.81131024f, 0.26465935f, + + 0.08619648f, 0.43343993f, 0.3877785f, 0.04523505f, 0.15625437f, + 0.61900597f, 0.01653285f, 0.06394322f, 0.56592636f, 0.27376196f, + 0.11201305f, 0.31654337f, 0.21947994f, 0.07893034f, 0.05236297f, + 0.18278451f, 0.23348385f, 0.32879834f, 0.30990825f, 0.5176207f}); // clang-format on test_case.run(4); @@ -920,20 +920,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softmax_axis_negative_1_opset11) { // clang-format off test_case.add_expected_output( Shape{3, 4, 5}, - {0.80619484, 0.03075256, 0.1161086, 0.027393, 0.01955098, - 0.07012683, 0.22670066, 0.18689778, 0.4614171, 0.05485764, - 0.04486171, 0.7228683, 0.10286818, 0.07356264, 0.05583908, - 0.01280724, 0.02448298, 0.08096659, 0.11509769, 0.76664555, - - 0.30399805, 0.10764059, 0.03371745, 0.09505949, 0.4595844, - 0.13369875, 0.04866969, 0.19944906, 0.0633215, 0.554861, - 0.39101103, 0.19217177, 0.27755913, 0.10521588, 0.03404216, - 0.01150354, 0.08279411, 0.03137731, 0.6890207, 0.18530433, - - 0.0402528, 0.31156224, 0.23747502, 0.15431291, 0.25639707, - 0.10627912, 0.00436928, 0.01439711, 0.7097961, 0.16515835, - 0.06798343, 0.29571748, 0.17468554, 0.34994435, 0.11166911, - 0.03615172, 0.07108136, 0.08527993, 0.4477579, 0.35972902}); + {0.80619484f, 0.03075256f, 0.1161086f, 0.027393f, 0.01955098f, + 0.07012683f, 0.22670066f, 0.18689778f, 0.4614171f, 0.05485764f, + 0.04486171f, 0.7228683f, 0.10286818f, 0.07356264f, 0.05583908f, + 0.01280724f, 0.02448298f, 0.08096659f, 0.11509769f, 0.76664555f, + + 0.30399805f, 0.10764059f, 0.03371745f, 0.09505949f, 0.4595844f, + 0.13369875f, 0.04866969f, 0.19944906f, 0.0633215f, 0.554861f, + 0.39101103f, 0.19217177f, 0.27755913f, 0.10521588f, 0.03404216f, + 0.01150354f, 0.08279411f, 0.03137731f, 0.6890207f, 0.18530433f, + + 0.0402528f, 0.31156224f, 0.23747502f, 0.15431291f, 0.25639707f, + 0.10627912f, 0.00436928f, 0.01439711f, 0.7097961f, 0.16515835f, + 0.06798343f, 0.29571748f, 0.17468554f, 0.34994435f, 0.11166911f, + 0.03615172f, 0.07108136f, 0.08527993f, 0.4477579f, 0.35972902f}); // clang-format on test_case.run(6); @@ -950,20 +950,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softmax_axis_negative_1_opset13) { // clang-format off test_case.add_expected_output( Shape{3, 4, 5}, - {0.80619484, 0.03075256, 0.1161086, 0.027393, 0.01955098, - 0.07012683, 0.22670066, 0.18689778, 0.4614171, 0.05485764, - 0.04486171, 0.7228683, 0.10286818, 0.07356264, 0.05583908, - 0.01280724, 0.02448298, 0.08096659, 0.11509769, 0.76664555, - - 0.30399805, 0.10764059, 0.03371745, 0.09505949, 0.4595844, - 0.13369875, 0.04866969, 0.19944906, 0.0633215, 0.554861, - 0.39101103, 0.19217177, 0.27755913, 0.10521588, 0.03404216, - 0.01150354, 0.08279411, 0.03137731, 0.6890207, 0.18530433, - - 0.0402528, 0.31156224, 0.23747502, 0.15431291, 0.25639707, - 0.10627912, 0.00436928, 0.01439711, 0.7097961, 0.16515835, - 0.06798343, 0.29571748, 0.17468554, 0.34994435, 0.11166911, - 0.03615172, 0.07108136, 0.08527993, 0.4477579, 0.35972902}); + {0.80619484f, 0.03075256f, 0.1161086f, 0.027393f, 0.01955098f, + 0.07012683f, 0.22670066f, 0.18689778f, 0.4614171f, 0.05485764f, + 0.04486171f, 0.7228683f, 0.10286818f, 0.07356264f, 0.05583908f, + 0.01280724f, 0.02448298f, 0.08096659f, 0.11509769f, 0.76664555f, + + 0.30399805f, 0.10764059f, 0.03371745f, 0.09505949f, 0.4595844f, + 0.13369875f, 0.04866969f, 0.19944906f, 0.0633215f, 0.554861f, + 0.39101103f, 0.19217177f, 0.27755913f, 0.10521588f, 0.03404216f, + 0.01150354f, 0.08279411f, 0.03137731f, 0.6890207f, 0.18530433f, + + 0.0402528f, 0.31156224f, 0.23747502f, 0.15431291f, 0.25639707f, + 0.10627912f, 0.00436928f, 0.01439711f, 0.7097961f, 0.16515835f, + 0.06798343f, 0.29571748f, 0.17468554f, 0.34994435f, 0.11166911f, + 0.03615172f, 0.07108136f, 0.08527993f, 0.4477579f, 0.35972902f}); // clang-format on test_case.run(6); @@ -1786,7 +1786,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_resize11_down_sizes_cubic_half_pixel) { test_case.add_input(input_data); test_case.add_expected_output( expected_output_shape, - {1.6307871, 3.0046299, 4.3784733, 7.1261587, 8.5, 9.873844, 12.621532, 13.995373, 15.369216}); + {1.6307871f, 3.0046299f, 4.3784733f, 7.1261587f, 8.5f, 9.873844f, 12.621532f, 13.995373f, 15.369216f}); test_case.run_with_tolerance_as_fp(2.0e-5f); } @@ -1848,18 +1848,18 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_resize11_up_sizes_cubic_half_pixel) { test_case.add_input(input_data); test_case.add_expected_output( expected_output_shape, - {0.45507922f, 0.64057922f, 0.97157922f, 1.42257922f, 1.90732922, 2.22332922f, 2.70807922f, 3.15907922f, - 3.49007922f, 3.67557922, 1.39437963f, 1.57987963f, 1.91087963f, 2.36187963f, 2.84662963, 3.16262963f, - 3.64737963f, 4.09837963f, 4.42937963f, 4.61487963, 2.95130693f, 3.13680693f, 3.46780693f, 3.91880693f, - 4.40355693, 4.71955693f, 5.20430693f, 5.65530693f, 5.98630693f, 6.17180693, 5.20525069f, 5.39075069f, - 5.72175069f, 6.17275069f, 6.65750069, 6.97350069f, 7.45825069f, 7.90925069f, 8.24025069f, 8.42575069, - 6.88975f, 7.07525f, 7.40625f, 7.85725f, 8.342, 8.658f, 9.14275f, 9.59375f, - 9.92475f, 10.11025f, 8.57424931f, 8.75974931f, 9.09074931f, 9.54174931f, 10.02649931, 10.34249931f, - 10.82724931f, 11.27824931f, 11.60924931f, 11.79474931, 10.82819307f, 11.01369307f, 11.34469307f, 11.79569307f, - 12.28044307, 12.59644307f, 13.08119307f, 13.53219307f, 13.86319307f, 14.04869307, 12.38512037f, 12.57062037f, - 12.90162037f, 13.35262037f, 13.83737037, 14.15337037f, 14.63812037f, 15.08912037f, 15.42012037f, 15.60562037, - 13.32442078f, 13.50992078f, 13.84092078f, 14.29192078f, 14.77667078, 15.09267078f, 15.57742078f, 16.02842078f, - 16.35942078f, 16.54492078}); + {0.45507922f, 0.64057922f, 0.97157922f, 1.42257922f, 1.90732922f, 2.22332922f, 2.70807922f, 3.15907922f, + 3.49007922f, 3.67557922f, 1.39437963f, 1.57987963f, 1.91087963f, 2.36187963f, 2.84662963f, 3.16262963f, + 3.64737963f, 4.09837963f, 4.42937963f, 4.61487963f, 2.95130693f, 3.13680693f, 3.46780693f, 3.91880693f, + 4.40355693f, 4.71955693f, 5.20430693f, 5.65530693f, 5.98630693f, 6.17180693f, 5.20525069f, 5.39075069f, + 5.72175069f, 6.17275069f, 6.65750069f, 6.97350069f, 7.45825069f, 7.90925069f, 8.24025069f, 8.42575069f, + 6.88975f, 7.07525f, 7.40625f, 7.85725f, 8.342f, 8.658f, 9.14275f, 9.59375f, + 9.92475f, 10.11025f, 8.57424931f, 8.75974931f, 9.09074931f, 9.54174931f, 10.02649931f, 10.34249931f, + 10.82724931f, 11.27824931f, 11.60924931f, 11.79474931f, 10.82819307f, 11.01369307f, 11.34469307f, 11.79569307f, + 12.28044307f, 12.59644307f, 13.08119307f, 13.53219307f, 13.86319307f, 14.04869307f, 12.38512037f, 12.57062037f, + 12.90162037f, 13.35262037f, 13.83737037f, 14.15337037f, 14.63812037f, 15.08912037f, 15.42012037f, 15.60562037f, + 13.32442078f, 13.50992078f, 13.84092078f, 14.29192078f, 14.77667078f, 15.09267078f, 15.57742078f, 16.02842078f, + 16.35942078f, 16.54492078f}); test_case.run_with_tolerance_as_fp(2.0e-5f); } @@ -1891,18 +1891,18 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_resize11_up_sizes_cubic_half_pixel_dynamic_siz test_case.add_input(std::vector{1, 1, 9, 10}); // sizes test_case.add_expected_output( expected_output_shape, - {0.45507922f, 0.64057922f, 0.97157922f, 1.42257922f, 1.90732922, 2.22332922f, 2.70807922f, 3.15907922f, - 3.49007922f, 3.67557922, 1.39437963f, 1.57987963f, 1.91087963f, 2.36187963f, 2.84662963, 3.16262963f, - 3.64737963f, 4.09837963f, 4.42937963f, 4.61487963, 2.95130693f, 3.13680693f, 3.46780693f, 3.91880693f, - 4.40355693, 4.71955693f, 5.20430693f, 5.65530693f, 5.98630693f, 6.17180693, 5.20525069f, 5.39075069f, - 5.72175069f, 6.17275069f, 6.65750069, 6.97350069f, 7.45825069f, 7.90925069f, 8.24025069f, 8.42575069, - 6.88975f, 7.07525f, 7.40625f, 7.85725f, 8.342, 8.658f, 9.14275f, 9.59375f, - 9.92475f, 10.11025f, 8.57424931f, 8.75974931f, 9.09074931f, 9.54174931f, 10.02649931, 10.34249931f, - 10.82724931f, 11.27824931f, 11.60924931f, 11.79474931, 10.82819307f, 11.01369307f, 11.34469307f, 11.79569307f, - 12.28044307, 12.59644307f, 13.08119307f, 13.53219307f, 13.86319307f, 14.04869307, 12.38512037f, 12.57062037f, - 12.90162037f, 13.35262037f, 13.83737037, 14.15337037f, 14.63812037f, 15.08912037f, 15.42012037f, 15.60562037, - 13.32442078f, 13.50992078f, 13.84092078f, 14.29192078f, 14.77667078, 15.09267078f, 15.57742078f, 16.02842078f, - 16.35942078f, 16.54492078}); + {0.45507922f, 0.64057922f, 0.97157922f, 1.42257922f, 1.90732922f, 2.22332922f, 2.70807922f, 3.15907922f, + 3.49007922f, 3.67557922f, 1.39437963f, 1.57987963f, 1.91087963f, 2.36187963f, 2.84662963f, 3.16262963f, + 3.64737963f, 4.09837963f, 4.42937963f, 4.61487963f, 2.95130693f, 3.13680693f, 3.46780693f, 3.91880693f, + 4.40355693f, 4.71955693f, 5.20430693f, 5.65530693f, 5.98630693f, 6.17180693f, 5.20525069f, 5.39075069f, + 5.72175069f, 6.17275069f, 6.65750069f, 6.97350069f, 7.45825069f, 7.90925069f, 8.24025069f, 8.42575069f, + 6.88975f, 7.07525f, 7.40625f, 7.85725f, 8.342f, 8.658f, 9.14275f, 9.59375f, + 9.92475f, 10.11025f, 8.57424931f, 8.75974931f, 9.09074931f, 9.54174931f, 10.02649931f, 10.34249931f, + 10.82724931f, 11.27824931f, 11.60924931f, 11.79474931f, 10.82819307f, 11.01369307f, 11.34469307f, 11.79569307f, + 12.28044307f, 12.59644307f, 13.08119307f, 13.53219307f, 13.86319307f, 14.04869307f, 12.38512037f, 12.57062037f, + 12.90162037f, 13.35262037f, 13.83737037f, 14.15337037f, 14.63812037f, 15.08912037f, 15.42012037f, 15.60562037f, + 13.32442078f, 13.50992078f, 13.84092078f, 14.29192078f, 14.77667078f, 15.09267078f, 15.57742078f, 16.02842078f, + 16.35942078f, 16.54492078f}); test_case.run_with_tolerance_as_fp(2.0e-5f); } @@ -2177,7 +2177,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_prelu_batch_nd_elementwise) { // Shape{2, 3, 4, 5} std::vector slope(shape_size(Shape{2, 3, 4, 5})); - std::iota(std::begin(slope), std::end(slope), 0); + std::iota(std::begin(slope), std::end(slope), 0.f); inputs.emplace_back(slope); // Shape{2, 3, 4, 5} @@ -2426,19 +2426,19 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softplus) { -FLT_MAX}}; const auto inf = std::numeric_limits::infinity(); - std::vector output{0.3132616579532623291, - 0.6931471824645996094, - 1.313261628150939941, - 10.0000457763671875, - 100.0, - 0.0, - 1000.0, - 0.0, - 0.6931471824645996094, - 0.6931471824645996094, - 0.6931471824645996094, + std::vector output{0.3132616579532623291f, + 0.6931471824645996094f, + 1.313261628150939941f, + 10.0000457763671875f, + 100.0f, + 0.0f, + 1000.0f, + 0.0f, + 0.6931471824645996094f, + 0.6931471824645996094f, + 0.6931471824645996094f, inf, - 0.0}; + 0.0f}; auto test_case = test::TestCase(function, s_device); test_case.add_multiple_inputs(inputs); @@ -2507,7 +2507,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_argmax_float) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/argmax_float.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({4, 0.1, 2, 3, -3, 1, -0.9, 0, 1, 2, 3, 0}); + test_case.add_input({4.f, 0.1f, 2.f, 3.f, -3.f, 1.f, -0.9f, 0.f, 1.f, 2.f, 3.f, 0.f}); test_case.add_expected_output({0, 3, 0}); test_case.run(); } @@ -2517,7 +2517,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_argmin_float) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/argmin_float.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({4, 0.1, 2, 3, -3, 1, -0.9, 0, 1, 2, 3, 0}); + test_case.add_input({4.f, 0.1f, 2.f, 3.f, -3.f, 1.f, -0.9f, 0.f, 1.f, 2.f, 3.f, 0.f}); test_case.add_expected_output({1, 1, 0, 2}); test_case.run(); } @@ -2528,7 +2528,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_argmax_select_last_index) { "onnx/argmax_select_last_index.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{4, 3}, {1, 1, 1, 0.5, 3, 4, 0.5, 1, 1.1, 0, 3, 0}); + test_case.add_input(Shape{4, 3}, {1.f, 1.f, 1.f, 0.5f, 3.f, 4.f, 0.5f, 1.f, 1.1f, 0.f, 3.f, 0.f}); test_case.add_expected_output(Shape{1, 3}, {0, 3, 1}); test_case.run(); } @@ -2539,7 +2539,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_argmin_select_last_index) { "onnx/argmin_select_last_index.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{4, 3}, {1, 1, 1, 2, 3, 4, 2, 1, 1.1, 3, 3, 8}); + test_case.add_input(Shape{4, 3}, {1.f, 1.f, 1.f, 2.f, 3.f, 4.f, 2.f, 1.f, 1.1f, 3.f, 3.f, 8.f}); test_case.add_expected_output(Shape{4}, {2, 0, 1, 1}); test_case.run(); } @@ -2736,10 +2736,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_one_hot_with_axis) { auto function = onnx_import::import_onnx_model( file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/one_hot_axis.onnx")); - Inputs inputs{{1.0, 9.0, 2.0, 4.0}, {1.0, 3.0}}; - std::vector expected_output{{1.0, 1.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, 3.0, 1.0, 1.0, 1.0, 1.0, 3.0, 1.0, 1.0, 1.0, - 1.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}}; + Inputs inputs{{1.0f, 9.0f, 2.0f, 4.0f}, {1.0f, 3.0f}}; + std::vector expected_output{{1.0f, 1.0f, 3.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 3.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 3.0f, 1.0f, 1.0f, 1.0f, 1.0f, 3.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}}; auto test_case = test::TestCase(function, s_device); test_case.add_multiple_inputs(inputs); @@ -2851,7 +2852,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lp_norm_p1) { Shape data_shape{2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); test_case.add_input(data); @@ -2870,7 +2871,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lp_norm_p2) { Shape data_shape{2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); test_case.add_input(data); @@ -2889,7 +2890,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lp_norm_default) { Shape data_shape{2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); test_case.add_input(data); @@ -2909,7 +2910,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lp_norm_default_dynamic) { Shape data_shape{2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); test_case.add_input(data_shape, data); @@ -2928,7 +2929,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_instance_normalization) { Shape data_shape{1, 2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); @@ -2953,7 +2954,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_instance_normalization_dynamic) { std::vector input_data{1.f, 2.f, 3.f}; test_case.add_input(Shape{1, 3, 1, 1}, input_data); test_case.add_expected_output(Shape{1, 3, 1, 1}, - {0.3341970741748809814, 0.3321160078048706055, 0.3407136797904968262}); + {0.3341970741748809814f, 0.3321160078048706055f, 0.3407136797904968262f}); test_case.run(); } @@ -3121,9 +3122,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_mod_sign_fmod_f32) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/mod_sign_fmod_f32.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({-4.3, 7.2, 5.0, 4.3, -7.2, 8.0}); - test_case.add_input({2.1, -3.4, 8.0, -2.1, 3.4, 5.0}); - test_case.add_expected_output(Shape{6}, {-0.10000038, 0.39999962, 5., 0.10000038, -0.39999962, 3.}); + test_case.add_input({-4.3f, 7.2f, 5.0f, 4.3f, -7.2f, 8.0f}); + test_case.add_input({2.1f, -3.4f, 8.0f, -2.1f, 3.4f, 5.0f}); + test_case.add_expected_output(Shape{6}, {-0.10000038f, 0.39999962f, 5.f, 0.10000038f, -0.39999962f, 3.f}); test_case.run(); } @@ -3280,22 +3281,22 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_gather_float_2D_neg_indices) { // clang-format off test_case.add_input(Shape{3, 3}, - { 0.0, 0.1, 0.2, - 1.0, 1.1, 1.2, - 2.0, 2.1, 2.2 }); + { 0.0f, 0.1f, 0.2f, + 1.0f, 1.1f, 1.2f, + 2.0f, 2.1f, 2.2f }); test_case.add_input(Shape{2, 2}, { -1, -2, -3, -2 }); test_case.add_expected_output(Shape{3, 2, 2}, { - 0.2, 0.1, - 0.0, 0.1, + 0.2f, 0.1f, + 0.0f, 0.1f, - 1.2, 1.1, - 1.0, 1.1, + 1.2f, 1.1f, + 1.0f, 1.1f, - 2.2, 2.1, - 2.0, 2.1 }); + 2.2f, 2.1f, + 2.0f, 2.1f }); // clang-format on test_case.run(); @@ -3633,10 +3634,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_upsample8_nearest_infer) { const Shape expected_output_shape{1, 1, 4, 6}; auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f}); test_case.add_expected_output(expected_output_shape, - {1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, - 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0}); + {1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 2.0f, 1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 2.0f, + 3.0f, 3.0f, 3.0f, 4.0f, 4.0f, 4.0f, 3.0f, 3.0f, 3.0f, 4.0f, 4.0f, 4.0f}); test_case.run(); } @@ -3650,10 +3651,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_upsample8_linear_infer) { const Shape expected_output_shape{1, 1, 4, 4}; auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f}); test_case.add_expected_output( expected_output_shape, - {1.0, 1.5, 2.0, 2.0, 2.0, 2.5, 3.0, 3.0, 3.0, 3.5, 4.0, 4.0, 3.0, 3.5, 4.0, 4.0}); + {1.0f, 1.5f, 2.0f, 2.0f, 2.0f, 2.5f, 3.0f, 3.0f, 3.0f, 3.5f, 4.0f, 4.0f, 3.0f, 3.5f, 4.0f, 4.0f}); test_case.run(); } @@ -3669,10 +3670,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_upsample9_scales_const_nearest_infer) { const Shape expected_output_shape{1, 1, 4, 6}; auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f}); test_case.add_expected_output(expected_output_shape, - {1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, - 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0}); + {1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 2.0f, 1.0f, 1.0f, 1.0f, 2.0f, 2.0f, 2.0f, + 3.0f, 3.0f, 3.0f, 4.0f, 4.0f, 4.0f, 3.0f, 3.0f, 3.0f, 4.0f, 4.0f, 4.0f}); test_case.run(); } @@ -3688,10 +3689,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_upsample9_scales_const_linear_infer) { const Shape expected_output_shape{1, 1, 4, 4}; auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f}); test_case.add_expected_output( expected_output_shape, - {1.0, 1.5, 2.0, 2.0, 2.0, 2.5, 3.0, 3.0, 3.0, 3.5, 4.0, 4.0, 3.0, 3.5, 4.0, 4.0}); + {1.0f, 1.5f, 2.0f, 2.0f, 2.0f, 2.5f, 3.0f, 3.0f, 3.0f, 3.5f, 4.0f, 4.0f, 3.0f, 3.5f, 4.0f, 4.0f}); test_case.run(); } @@ -3700,8 +3701,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_image_scaler) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/image_scaler.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0, 40.0}); - test_case.add_expected_output(Shape{1, 2, 2, 2}, {12.0, 14.0, 16.0, 18.0, 21.0, 41.0, 61.0, 81.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f, 10.0f, 20.0f, 30.0f, 40.0f}); + test_case.add_expected_output(Shape{1, 2, 2, 2}, {12.0f, 14.0f, 16.0f, 18.0f, 21.0f, 41.0f, 61.0f, 81.0f}); test_case.run(); } @@ -3710,7 +3711,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_size_op_single) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/size_op_single.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{2, 3}, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}); + test_case.add_input(Shape{2, 3}, {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}); test_case.add_expected_output(Shape{}, {6}); test_case.run(); } @@ -3720,7 +3721,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_size_op_graph_end) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/size_op_graph_end.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f}); test_case.add_expected_output(Shape{}, {4}); test_case.run(); } @@ -3731,8 +3732,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_size_op_graph_middle) { "onnx/size_op_graph_middle.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.0, 2.0, 3.0, 4.0}); - test_case.add_expected_output(Shape{}, {4.0}); + test_case.add_input({1.0f, 2.0f, 3.0f, 4.0f}); + test_case.add_expected_output(Shape{}, {4.0f}); test_case.run(); } @@ -3743,11 +3744,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_size_op_on_input_graph_middle) { "onnx/size_op_on_input_graph_middle.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 2, 4, 1, 3}, {0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.}); + test_case.add_input(Shape{1, 2, 4, 1, 3}, {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}); test_case.add_expected_output(Shape{1, 2, 4, 1, 3}, - {24., 24., 24., 24., 24., 24., 24., 24., 24., 24., 24., 24., - 24., 24., 24., 24., 24., 24., 24., 24., 24., 24., 24., 24.}); + {24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, + 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f, 24.f}); test_case.run(); } @@ -3818,36 +3819,39 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_roialign16_avg_out_half_pixel) { auto test_case = test::TestCase(function, s_device); test_case.add_input( - {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 11., 12.1, 13.2, 14.3, 15.4, 16.5, 17.6, - 18.7, 19.8, 20.9, 22., 23.1, 24.2, 25.3, 26.4, 27.5, 28.6, 29.7, 30.8, 31.9, 33., 34.1, 35.2, - 36.3, 37.4, 38.5, 39.6, 40.7, 41.8, 42.9, 44., 45.1, 46.2, 47.3, 48.4, 49.5, 50.6, 51.7, 52.8, - 53.9, 55., 56.1, 57.2, 58.3, 59.4, 60.5, 61.6, 62.7, 63.8, 64.9, 66., 67.1, 68.2, 69.3, 70.4, - 71.5, 72.6, 73.7, 74.8, 75.9, 77., 78.1, 79.2, 80.3, 81.4, 82.5, 83.6, 84.7, 85.8, 86.9, 88., - 89.1, 90.2, 91.3, 92.4, 93.5, 94.6, 95.7, 96.8, 97.9, 99., 100.1, 101.2, 102.3, 103.4, 104.5, 105.6, - 106.7, 107.8, 108.9, 110., 111.1, 112.2, 113.3, 114.4, 115.5, 116.6, 117.7, 118.8, 119.9, 121., 122.1, 123.2, - 124.3, 125.4, 126.5, 127.6, 128.7, 129.8, 130.9, 132., 133.1, 134.2, 135.3, 136.4, 137.5, 138.6, 139.7, 140.8, - 141.9, 143., 144.1, 145.2, 146.3, 147.4, 148.5, 149.6, 150.7, 151.8, 152.9, 154., 155.1, 156.2, 157.3, 158.4, - 159.5, 160.6, 161.7, 162.8, 163.9, 165., 166.1, 167.2, 168.3, 169.4, 170.5, 171.6, 172.7, 173.8, 174.9, 176., - 177.1, 178.2, 179.3, 180.4, 181.5, 182.6, 183.7, 184.8, 185.9, 187., 188.1, 189.2, 190.3, 191.4, 192.5, 193.6, - 194.7, 195.8, 196.9, 198., 199.1, 200.2, 201.3, 202.4, 203.5, 204.6, 205.7, 206.8, 207.9, 209., 210.1, 211.2, - 212.3, 213.4, 214.5, 215.6, 216.7, 217.8, 218.9, 220., 221.1, 222.2, 223.3, 224.4, 225.5, 226.6, 227.7, 228.8, - 229.9, 231., 232.1, 233.2, 234.3, 235.4, 236.5, 237.6}); - - test_case.add_input({0, 0, 0.75, 2.2, 1.2, 0.5, 2.8, 1.9, 0, 3, 0, 3}); + {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f, 9.9f, 11.f, 12.1f, 13.2f, 14.3f, 15.4f, + 16.5f, 17.6f, 18.7f, 19.8f, 20.9f, 22.f, 23.1f, 24.2f, 25.3f, 26.4f, 27.5f, 28.6f, 29.7f, 30.8f, + 31.9f, 33.f, 34.1f, 35.2f, 36.3f, 37.4f, 38.5f, 39.6f, 40.7f, 41.8f, 42.9f, 44.f, 45.1f, 46.2f, + 47.3f, 48.4f, 49.5f, 50.6f, 51.7f, 52.8f, 53.9f, 55.f, 56.1f, 57.2f, 58.3f, 59.4f, 60.5f, 61.6f, + 62.7f, 63.8f, 64.9f, 66.f, 67.1f, 68.2f, 69.3f, 70.4f, 71.5f, 72.6f, 73.7f, 74.8f, 75.9f, 77.f, + 78.1f, 79.2f, 80.3f, 81.4f, 82.5f, 83.6f, 84.7f, 85.8f, 86.9f, 88.f, 89.1f, 90.2f, 91.3f, 92.4f, + 93.5f, 94.6f, 95.7f, 96.8f, 97.9f, 99.f, 100.1f, 101.2f, 102.3f, 103.4f, 104.5f, 105.6f, 106.7f, 107.8f, + 108.9f, 110.f, 111.1f, 112.2f, 113.3f, 114.4f, 115.5f, 116.6f, 117.7f, 118.8f, 119.9f, 121.f, 122.1f, 123.2f, + 124.3f, 125.4f, 126.5f, 127.6f, 128.7f, 129.8f, 130.9f, 132.f, 133.1f, 134.2f, 135.3f, 136.4f, 137.5f, 138.6f, + 139.7f, 140.8f, 141.9f, 143.f, 144.1f, 145.2f, 146.3f, 147.4f, 148.5f, 149.6f, 150.7f, 151.8f, 152.9f, 154.f, + 155.1f, 156.2f, 157.3f, 158.4f, 159.5f, 160.6f, 161.7f, 162.8f, 163.9f, 165.f, 166.1f, 167.2f, 168.3f, 169.4f, + 170.5f, 171.6f, 172.7f, 173.8f, 174.9f, 176.f, 177.1f, 178.2f, 179.3f, 180.4f, 181.5f, 182.6f, 183.7f, 184.8f, + 185.9f, 187.f, 188.1f, 189.2f, 190.3f, 191.4f, 192.5f, 193.6f, 194.7f, 195.8f, 196.9f, 198.f, 199.1f, 200.2f, + 201.3f, 202.4f, 203.5f, 204.6f, 205.7f, 206.8f, 207.9f, 209.f, 210.1f, 211.2f, 212.3f, 213.4f, 214.5f, 215.6f, + 216.7f, 217.8f, 218.9f, 220.f, 221.1f, 222.2f, 223.3f, 224.4f, 225.5f, 226.6f, 227.7f, 228.8f, 229.9f, 231.f, + 232.1f, 233.2f, 234.3f, 235.4f, 236.5f, 237.6f}); + + test_case.add_input({0.f, 0.f, 0.75f, 2.2f, 1.2f, 0.5f, 2.8f, 1.9f, 0.f, 3.f, 0.f, 3.f}); test_case.add_input({0, 2, 1}); test_case.add_expected_output( Shape{3, 2, 4, 4}, - {2.145, 2.42, 2.6950002, 2.9700003, 3.96, 4.235, 4.51, 4.7850003, 5.775, 6.05, - 6.325, 6.6000004, 7.59, 7.8650007, 8.14, 8.415001, 41.745003, 42.019997, 42.295, 42.57, - 43.56, 43.835, 44.11, 44.385002, 45.375, 45.65, 45.925003, 46.200005, 47.190002, 47.465004, - 47.74, 48.015, 162.77249, 163.0475, 163.32251, 163.5975, 164.42252, 164.69751, 164.9725, 165.2475, - 166.07251, 166.3475, 166.6225, 166.8975, 167.72249, 167.9975, 168.27249, 168.5475, 202.3725, 202.6475, - 202.9225, 203.19751, 204.02252, 204.2975, 204.57251, 204.8475, 205.6725, 205.94751, 206.2225, 206.4975, - 207.32251, 207.5975, 207.8725, 208.1475, 91.162506, 91.4375, 91.7125, 91.9875, 92.8125, 93.0875, - 93.3625, 93.6375, 94.4625, 94.7375, 95.0125, 95.28749, 96.1125, 96.3875, 96.6625, 96.9375, - 130.76251, 131.0375, 131.3125, 131.5875, 132.4125, 132.6875, 132.9625, 133.2375, 134.0625, 134.33751, - 134.6125, 134.88751, 135.7125, 135.9875, 136.26251, 136.53749}); + {2.145f, 2.42f, 2.6950002f, 2.9700003f, 3.96f, 4.235f, 4.51f, 4.7850003f, 5.775f, + 6.05f, 6.325f, 6.6000004f, 7.59f, 7.8650007f, 8.14f, 8.415001f, 41.745003f, 42.019997f, + 42.295f, 42.57f, 43.56f, 43.835f, 44.11f, 44.385002f, 45.375f, 45.65f, 45.925003f, + 46.200005f, 47.190002f, 47.465004f, 47.74f, 48.015f, 162.77249f, 163.0475f, 163.32251f, 163.5975f, + 164.42252f, 164.69751f, 164.9725f, 165.2475f, 166.07251f, 166.3475f, 166.6225f, 166.8975f, 167.72249f, + 167.9975f, 168.27249f, 168.5475f, 202.3725f, 202.6475f, 202.9225f, 203.19751f, 204.02252f, 204.2975f, + 204.57251f, 204.8475f, 205.6725f, 205.94751f, 206.2225f, 206.4975f, 207.32251f, 207.5975f, 207.8725f, + 208.1475f, 91.162506f, 91.4375f, 91.7125f, 91.9875f, 92.8125f, 93.0875f, 93.3625f, 93.6375f, + 94.4625f, 94.7375f, 95.0125f, 95.28749f, 96.1125f, 96.3875f, 96.6625f, 96.9375f, 130.76251f, + 131.0375f, 131.3125f, 131.5875f, 132.4125f, 132.6875f, 132.9625f, 133.2375f, 134.0625f, 134.33751f, + 134.6125f, 134.88751f, 135.7125f, 135.9875f, 136.26251f, 136.53749f}); test_case.run(); } @@ -3858,36 +3862,40 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_roialign16_avg_half_pixel) { auto test_case = test::TestCase(function, s_device); test_case.add_input( - {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 11., 12.1, 13.2, 14.3, 15.4, 16.5, 17.6, - 18.7, 19.8, 20.9, 22., 23.1, 24.2, 25.3, 26.4, 27.5, 28.6, 29.7, 30.8, 31.9, 33., 34.1, 35.2, - 36.3, 37.4, 38.5, 39.6, 40.7, 41.8, 42.9, 44., 45.1, 46.2, 47.3, 48.4, 49.5, 50.6, 51.7, 52.8, - 53.9, 55., 56.1, 57.2, 58.3, 59.4, 60.5, 61.6, 62.7, 63.8, 64.9, 66., 67.1, 68.2, 69.3, 70.4, - 71.5, 72.6, 73.7, 74.8, 75.9, 77., 78.1, 79.2, 80.3, 81.4, 82.5, 83.6, 84.7, 85.8, 86.9, 88., - 89.1, 90.2, 91.3, 92.4, 93.5, 94.6, 95.7, 96.8, 97.9, 99., 100.1, 101.2, 102.3, 103.4, 104.5, 105.6, - 106.7, 107.8, 108.9, 110., 111.1, 112.2, 113.3, 114.4, 115.5, 116.6, 117.7, 118.8, 119.9, 121., 122.1, 123.2, - 124.3, 125.4, 126.5, 127.6, 128.7, 129.8, 130.9, 132., 133.1, 134.2, 135.3, 136.4, 137.5, 138.6, 139.7, 140.8, - 141.9, 143., 144.1, 145.2, 146.3, 147.4, 148.5, 149.6, 150.7, 151.8, 152.9, 154., 155.1, 156.2, 157.3, 158.4, - 159.5, 160.6, 161.7, 162.8, 163.9, 165., 166.1, 167.2, 168.3, 169.4, 170.5, 171.6, 172.7, 173.8, 174.9, 176., - 177.1, 178.2, 179.3, 180.4, 181.5, 182.6, 183.7, 184.8, 185.9, 187., 188.1, 189.2, 190.3, 191.4, 192.5, 193.6, - 194.7, 195.8, 196.9, 198., 199.1, 200.2, 201.3, 202.4, 203.5, 204.6, 205.7, 206.8, 207.9, 209., 210.1, 211.2, - 212.3, 213.4, 214.5, 215.6, 216.7, 217.8, 218.9, 220., 221.1, 222.2, 223.3, 224.4, 225.5, 226.6, 227.7, 228.8, - 229.9, 231., 232.1, 233.2, 234.3, 235.4, 236.5, 237.6}); - - test_case.add_input({0, 0, 0.75, 2.2, 1.2, 0.5, 2.8, 1.9, 0, 3, 0, 3}); + {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f, 9.9f, 11.f, 12.1f, 13.2f, 14.3f, + 15.4f, 16.5f, 17.6f, 18.7f, 19.8f, 20.9f, 22.f, 23.1f, 24.2f, 25.3f, 26.4f, 27.5f, 28.6f, + 29.7f, 30.8f, 31.9f, 33.f, 34.1f, 35.2f, 36.3f, 37.4f, 38.5f, 39.6f, 40.7f, 41.8f, 42.9f, + 44.f, 45.1f, 46.2f, 47.3f, 48.4f, 49.5f, 50.6f, 51.7f, 52.8f, 53.9f, 55.f, 56.1f, 57.2f, + 58.3f, 59.4f, 60.5f, 61.6f, 62.7f, 63.8f, 64.9f, 66.f, 67.1f, 68.2f, 69.3f, 70.4f, 71.5f, + 72.6f, 73.7f, 74.8f, 75.9f, 77.f, 78.1f, 79.2f, 80.3f, 81.4f, 82.5f, 83.6f, 84.7f, 85.8f, + 86.9f, 88.f, 89.1f, 90.2f, 91.3f, 92.4f, 93.5f, 94.6f, 95.7f, 96.8f, 97.9f, 99.f, 100.1f, + 101.2f, 102.3f, 103.4f, 104.5f, 105.6f, 106.7f, 107.8f, 108.9f, 110.f, 111.1f, 112.2f, 113.3f, 114.4f, + 115.5f, 116.6f, 117.7f, 118.8f, 119.9f, 121.f, 122.1f, 123.2f, 124.3f, 125.4f, 126.5f, 127.6f, 128.7f, + 129.8f, 130.9f, 132.f, 133.1f, 134.2f, 135.3f, 136.4f, 137.5f, 138.6f, 139.7f, 140.8f, 141.9f, 143.f, + 144.1f, 145.2f, 146.3f, 147.4f, 148.5f, 149.6f, 150.7f, 151.8f, 152.9f, 154.f, 155.1f, 156.2f, 157.3f, + 158.4f, 159.5f, 160.6f, 161.7f, 162.8f, 163.9f, 165.f, 166.1f, 167.2f, 168.3f, 169.4f, 170.5f, 171.6f, + 172.7f, 173.8f, 174.9f, 176.f, 177.1f, 178.2f, 179.3f, 180.4f, 181.5f, 182.6f, 183.7f, 184.8f, 185.9f, + 187.198f, 188.1f, 189.2f, 190.3f, 191.4f, 192.5f, 193.6f, 194.7f, 195.8f, 196.9f, 198.f, 199.1f, 200.2f, + 201.3f, 202.4f, 203.5f, 204.6f, 205.7f, 206.8f, 207.9f, 209.f, 210.1f, 211.2f, 212.3f, 213.4f, 214.5f, + 215.6f, 216.7f, 217.8f, 218.9f, 220.f, 221.1f, 222.2f, 223.3f, 224.4f, 225.5f, 226.6f, 227.7f, 228.8f, + 229.9f, 231.f, 232.1f, 233.2f, 234.3f, 235.4f, 236.5f, 237.6f}); + + test_case.add_input({0.f, 0.f, 0.75f, 2.2f, 1.2f, 0.5f, 2.8f, 1.9f, 0.f, 3.f, 0.f, 3.f}); test_case.add_input({0, 2, 1}); test_case.add_expected_output( Shape{3, 2, 4, 4}, - {1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 2.3375, 2.3375, - 2.3375, 2.3375, 4.1525, 4.1525, 4.1525, 4.1525, 40.7, 40.7, 40.7, 40.7, - 40.7, 40.7, 40.7, 40.7, 41.9375, 41.9375, 41.9375, 41.9375, 43.7525, 43.7525, - 43.7525, 43.7525, 159.72, 159.94, 160.16, 160.38, 159.90562, 160.12563, 160.34563, 160.56563, - 160.9575, 161.1775, 161.3975, 161.61751, 162.1125, 162.3325, 162.55249, 162.77249, 199.32, 199.54001, - 199.76001, 199.97998, 199.50562, 199.72563, 199.94562, 200.16562, 200.5575, 200.7775, 200.9975, 201.2175, - 201.7125, 201.93251, 202.1525, 202.37251, 86.9, 86.9, 86.9, 86.9, 86.9, 86.9, - 86.9, 86.9, 86.9, 86.9, 86.9, 86.9, 86.9, 86.9, 86.9, 86.9, - 126.5, 126.5, 126.5, 126.5, 126.5, 126.5, 126.5, 126.5, 126.5, 126.5, - 126.5, 126.5, 126.5, 126.5, 126.5, 126.5}); + {1.1f, 1.1f, 1.1f, 1.1f, 1.1f, 1.1f, 1.1f, 1.1f, 2.3375f, + 2.3375f, 2.3375f, 2.3375f, 4.1525f, 4.1525f, 4.1525f, 4.1525f, 40.7f, 40.7f, + 40.7f, 40.7f, 40.7f, 40.7f, 40.7f, 40.7f, 41.9375f, 41.9375f, 41.9375f, + 41.9375f, 43.7525f, 43.7525f, 43.7525f, 43.7525f, 159.72f, 159.94f, 160.16f, 160.38f, + 159.90562f, 160.12563f, 160.34563f, 160.56563f, 160.9575f, 161.1775f, 161.3975f, 161.61751f, 162.1125f, + 162.3325f, 162.55249f, 162.77249f, 199.32f, 199.54001f, 199.76001f, 199.97998f, 199.50562f, 199.72563f, + 199.94562f, 200.16562f, 200.5575f, 200.7775f, 200.9975f, 201.2175f, 201.7125f, 201.93251f, 202.1525f, + 202.37251f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, + 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 86.9f, 126.5f, + 126.5f, 126.5f, 126.5f, 126.5f, 126.5f, 126.5f, 126.5f, 126.5f, 126.5f, + 126.5f, 126.5f, 126.5f, 126.5f, 126.5f, 126.5f}); test_case.run_with_tolerance_as_fp(0.01f); } @@ -3898,9 +3906,9 @@ NGRAPH_TEST(${BACKEND_NAME}, quant_dequant_pattern) { auto test_case = test::TestCase(function, s_device); // scale == 3.0 // zero point == 10 - test_case.add_input({9.0, 10.0, 15.0, 20.0, 30.0}); - test_case.add_input({1}); - test_case.add_expected_output(Shape{5}, {9.0, 9.0, 15.0, 21.0, 30.0}); + test_case.add_input({9.0f, 10.0f, 15.0f, 20.0f, 30.0f}); + test_case.add_input({1.f}); + test_case.add_expected_output(Shape{5}, {9.0f, 9.0f, 15.0f, 21.0f, 30.0f}); test_case.run(); } @@ -3912,9 +3920,9 @@ NGRAPH_TEST(${BACKEND_NAME}, quant_dequant_pattern_axis) { // axis = 1 // scale == {2.0, 3.0, 4.0} // zero point == {10, 20, 30} - test_case.add_input({1.0, 2.0, 3.0, 10.0, 20.0, 30.0, 40.0, 50.0, 100.0}); - test_case.add_expected_output(Shape{3, 3}, {0, 3, 4, 10, 21, 32, 40, 51, 100}); - test_case.add_input({1}); + test_case.add_input({1.0f, 2.0f, 3.0f, 10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 100.0f}); + test_case.add_expected_output(Shape{3, 3}, {0.f, 3.f, 4.f, 10.f, 21.f, 32.f, 40.f, 51.f, 100.f}); + test_case.add_input({1.f}); test_case.run(); } @@ -3923,8 +3931,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_logsoftmax_0D) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/softmax_0D.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({3.141592}); - test_case.add_expected_output({0.0}); + test_case.add_input({3.141592f}); + test_case.add_expected_output({0.0f}); test_case.run(); } @@ -3934,7 +3942,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_logsoftmax_1D) { auto test_case = test::TestCase(function, s_device); test_case.add_input({-1.0f, 0.0f, 1.0f}); - test_case.add_expected_output(Shape{3}, {-2.4076061, -1.407606, -0.407606}); + test_case.add_expected_output(Shape{3}, {-2.4076061f, -1.407606f, -0.407606f}); test_case.run(); } @@ -3944,7 +3952,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_logsoftmax13_1D) { auto test_case = test::TestCase(function, s_device); test_case.add_input({-1.0f, 0.0f, 1.0f}); - test_case.add_expected_output(Shape{3}, {-2.4076061, -1.407606, -0.407606}); + test_case.add_expected_output(Shape{3}, {-2.4076061f, -1.407606f, -0.407606f}); test_case.run(); } @@ -3953,10 +3961,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_logsoftmax13_2D) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/logsoftmax13_2D.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.0f, 1.0f, 2.0f, 3.0f, 10000, 10001, 10002, 10003}); + test_case.add_input({0.0f, 1.0f, 2.0f, 3.0f, 10000.f, 10001.f, 10002.f, 10003.f}); test_case.add_expected_output( Shape{2, 4}, - {-3.4401896, -2.4401896, -1.4401896, -0.44018966, -3.4401896, -2.4401896, -1.4401896, -0.44018966}); + {-3.4401896f, -2.4401896f, -1.4401896f, -0.44018966f, -3.4401896f, -2.4401896f, -1.4401896f, -0.44018966f}); test_case.run_with_tolerance_as_fp(); } @@ -4004,7 +4012,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_mul_v6_broadcast_axis_1) { Shape shape{1, 3, 2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output( @@ -4036,7 +4044,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_mul_v6_broadcast_no_axis) { Shape shape{2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f}); test_case.add_expected_output(shape, {3.0f, 6.0f, 9.0f, 12.0f}); @@ -4061,7 +4069,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_mul_v7_broadcast) { Shape shape{1, 2, 3}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output(shape, {3.0f, 8.0f, 15.0f, 12.0f, 20.0f, 30.0f}); @@ -4076,7 +4084,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_add_v6_broadcast_axis_1) { Shape shape{1, 3, 2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output( @@ -4107,7 +4115,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_add_v6_broadcast_no_axis) { Shape shape{2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f}); test_case.add_expected_output(shape, {4.0f, 5.0f, 6.0f, 7.0f}); @@ -4133,7 +4141,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_sub_v6_broadcast_axis_1) { Shape shape{1, 3, 2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output(shape, @@ -4164,7 +4172,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_sub_v6_broadcast_no_axis) { Shape shape{2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f}); test_case.add_expected_output(shape, {-2.0f, -1.0f, 0.0f, 1.0f}); @@ -4189,7 +4197,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_sub_v7_broadcast) { Shape shape{1, 2, 3}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output(shape, {-2.0f, -2.0f, -2.0f, 1.0f, 1.0f, 1.0f}); @@ -4204,7 +4212,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_div_v6_broadcast_axis_1) { Shape shape{1, 3, 2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output( @@ -4237,7 +4245,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_div_v6_broadcast_no_axis) { Shape shape{2, 2}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({2.0f}); test_case.add_expected_output(shape, {0.5f, 1.0f, 1.5f, 2.0f}); @@ -4262,7 +4270,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_div_v7_broadcast) { Shape shape{1, 2, 3}; std::vector A(shape_size(shape)); - std::iota(A.begin(), A.end(), 1); + std::iota(A.begin(), A.end(), 1.f); test_case.add_input(A); test_case.add_input({3.0f, 4.0f, 5.0f}); test_case.add_expected_output(shape, {0.3333333f, 0.5f, 0.6f, 1.3333333f, 1.25f, 1.2f}); @@ -4299,7 +4307,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_no_min_no_max) { "onnx/clip_no_min_no_max.onnx")); auto test_case = test::TestCase(function, s_device); - const std::vector data{-1.6, -0.1, 10., 0., -10., 1.99, 2.015, 3.}; + const std::vector data{-1.6f, -0.1f, 10.f, 0.f, -10.f, 1.99f, 2.015f, 3.f}; test_case.add_input(data); @@ -4315,12 +4323,12 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_no_min_no_max_inf) { auto test_case = test::TestCase(function, s_device); const std::vector data{std::numeric_limits::infinity(), -std::numeric_limits::infinity(), - static_cast(std::numeric_limits::max()), + static_cast(std::numeric_limits::max()), std::numeric_limits::min(), std::numeric_limits::max(), std::numeric_limits::lowest(), - 0, - -1}; + 0.f, + -1.f}; const std::vector expected_output{std::numeric_limits::max(), std::numeric_limits::lowest(), @@ -4328,13 +4336,13 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_no_min_no_max_inf) { std::numeric_limits::min(), std::numeric_limits::max(), std::numeric_limits::lowest(), - 0, - -1}; + 0.f, + -1.f}; test_case.add_input(data); test_case.add_expected_output(Shape{2, 4}, expected_output); - test_case.run_with_tolerance_as_fp(0); + test_case.run_with_tolerance_as_fp(0.f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_no_min_set_max) { @@ -4343,9 +4351,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_no_min_set_max) { "onnx/clip_no_min_set_max.onnx")); auto test_case = test::TestCase(function, s_device); - const std::vector data{-1.6, -0.1, 10., 0., -10., 1.99, 2.015, 3.}; - const std::vector max_val{2.01}; - const std::vector output{-1.6, -0.1, 2.01, 0., -10., 1.99, 2.01, 2.01}; + const std::vector data{-1.6f, -0.1f, 10.f, 0.f, -10.f, 1.99f, 2.015f, 3.f}; + const std::vector max_val{2.01f}; + const std::vector output{-1.6f, -0.1f, 2.01f, 0.f, -10.f, 1.99f, 2.01f, 2.01f}; test_case.add_input(data); test_case.add_input(max_val); @@ -4360,9 +4368,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_set_min_no_max) { "onnx/clip_set_min_no_max.onnx")); auto test_case = test::TestCase(function, s_device); - const std::vector data{-1.6, -0.1, 10., 0., -10., 1.99, 2.015, 3.}; - const std::vector min_val{-1.59}; - const std::vector output{-1.59, -0.1, 10., 0., -1.59, 1.99, 2.015, 3.}; + const std::vector data{-1.6f, -0.1f, 10.f, 0.f, -10.f, 1.99f, 2.015f, 3.f}; + const std::vector min_val{-1.59f}; + const std::vector output{-1.59f, -0.1f, 10.f, 0.f, -1.59f, 1.99f, 2.015f, 3.f}; test_case.add_input(data); test_case.add_input(min_val); @@ -4408,8 +4416,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_set_min_no_max_initializers) { "onnx/clip_set_min_no_max_initializers.onnx")); auto test_case = test::TestCase(function, s_device); - const std::vector data{-1.6, -0.1, 10., 0., -10., 1.99, 2.015, 3.}; - const std::vector output{-1.59, -0.1, 10., 0., -1.59, 1.99, 2.015, 3.}; + const std::vector data{-1.6f, -0.1f, 10.f, 0.f, -10.f, 1.99f, 2.015f, 3.f}; + const std::vector output{-1.59f, -0.1f, 10.f, 0.f, -1.59f, 1.99f, 2.015f, 3.f}; test_case.add_input(data); @@ -4423,10 +4431,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_set_min_set_max) { "onnx/clip_set_min_set_max.onnx")); auto test_case = test::TestCase(function, s_device); - const std::vector data{-1.6, -0.1, 10., 0., -10., 1.99, 2.015, 3.}; - const std::vector min_val{-1.59}; - const std::vector max_val{2.01}; - const std::vector output{-1.59, -0.1, 2.01, 0., -1.59, 1.99, 2.01, 2.01}; + const std::vector data{-1.6f, -0.1f, 10.f, 0.f, -10.f, 1.99f, 2.015f, 3.f}; + const std::vector min_val{-1.59f}; + const std::vector max_val{2.01f}; + const std::vector output{-1.59f, -0.1f, 2.01f, 0.f, -1.59f, 1.99f, 2.01f, 2.01f}; test_case.add_input(data); test_case.add_input(min_val); @@ -4442,8 +4450,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_clip_set_min_set_max_initializers) { "onnx/clip_set_min_set_max_initializers.onnx")); auto test_case = test::TestCase(function, s_device); - const std::vector data{-1.6, -0.1, 10., 0., -10., 1.99, 2.015, 3.}; - const std::vector output{-1.59, -0.1, 2.01, 0., -1.59, 1.99, 2.01, 2.01}; + const std::vector data{-1.6f, -0.1f, 10.f, 0.f, -10.f, 1.99f, 2.015f, 3.f}; + const std::vector output{-1.59f, -0.1f, 2.01f, 0.f, -1.59f, 1.99f, 2.01f, 2.01f}; test_case.add_input(data); @@ -4456,16 +4464,16 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_mvn_v6) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/mvn_v6.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.8439683, 0.5665144, 0.05836735, 0.02916367, 0.12964272, 0.5060197, 0.79538304, - 0.9411346, 0.9546573, 0.17730942, 0.46192095, 0.26480448, 0.6746842, 0.01665257, - 0.62473077, 0.9240844, 0.9722341, 0.11965699, 0.41356155, 0.9129373, 0.59330076, - 0.81929934, 0.7862604, 0.11799799, 0.69248444, 0.54119414, 0.07513223}); + test_case.add_input({0.8439683f, 0.5665144f, 0.05836735f, 0.02916367f, 0.12964272f, 0.5060197f, 0.79538304f, + 0.9411346f, 0.9546573f, 0.17730942f, 0.46192095f, 0.26480448f, 0.6746842f, 0.01665257f, + 0.62473077f, 0.9240844f, 0.9722341f, 0.11965699f, 0.41356155f, 0.9129373f, 0.59330076f, + 0.81929934f, 0.7862604f, 0.11799799f, 0.69248444f, 0.54119414f, 0.07513223f}); test_case.add_expected_output( Shape{3, 3, 3, 1}, - {1.3546423, 0.33053496, -1.5450814, -1.2106764, -0.8925952, 0.29888135, 0.38083088, - 0.81808794, 0.85865635, -1.1060555, -0.05552877, -0.78310335, 0.83281356, -1.250282, - 0.67467856, 0.7669372, 0.9113869, -1.6463585, -0.23402764, 1.6092131, 0.42940593, - 1.2906139, 1.1860244, -0.92945826, 0.0721334, -0.38174, -1.7799333}); + {1.3546423f, 0.33053496f, -1.5450814f, -1.2106764f, -0.8925952f, 0.29888135f, 0.38083088f, + 0.81808794f, 0.85865635f, -1.1060555f, -0.05552877f, -0.78310335f, 0.83281356f, -1.250282f, + 0.67467856f, 0.7669372f, 0.9113869f, -1.6463585f, -0.23402764f, 1.6092131f, 0.42940593f, + 1.2906139f, 1.1860244f, -0.92945826f, 0.0721334f, -0.38174f, -1.7799333f}); test_case.run(); } @@ -4578,17 +4586,17 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dropout12_not_const_training_mode) { NGRAPH_TEST(${BACKEND_NAME}, onnx_multiple_slices_last_layer) { std::vector data(1 * 30 * 320 * 320); - std::fill(data.begin(), data.end(), 1); + std::fill(data.begin(), data.end(), 1.f); const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/multiple_slices_last_layer.onnx")); auto test_case = test::TestCase(function, s_device); std::vector o1(1 * 320 * 320 * 21); - std::fill(o1.begin(), o1.end(), 1); + std::fill(o1.begin(), o1.end(), 1.f); std::vector o2(1 * 320 * 320 * 9); - std::fill(o2.begin(), o2.end(), 1); + std::fill(o2.begin(), o2.end(), 1.f); test_case.add_input(data); test_case.add_expected_output(Shape{1, 320, 320, 21}, o1); @@ -4613,23 +4621,23 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_softmax_crossentropy_loss_mean) { "onnx/softmax_crossentropy_loss_mean.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.54881352186203, - 0.7151893377304077, - 0.6027633547782898, - 0.5448831915855408, - 0.42365479469299316, - 0.6458941102027893, - 0.4375872015953064, - 0.891772985458374, - 0.9636627435684204, - 0.3834415078163147, - 0.7917250394821167, - 0.5288949012756348, - 0.5680445432662964, - 0.9255966544151306, - 0.07103605568408966}); + test_case.add_input({0.54881352186203f, + 0.7151893377304077f, + 0.6027633547782898f, + 0.5448831915855408f, + 0.42365479469299316f, + 0.6458941102027893f, + 0.4375872015953064f, + 0.891772985458374f, + 0.9636627435684204f, + 0.3834415078163147f, + 0.7917250394821167f, + 0.5288949012756348f, + 0.5680445432662964f, + 0.9255966544151306f, + 0.07103605568408966f}); test_case.add_input({1, 4, 3}); - test_case.add_expected_output(Shape{}, {1.561384797096252441}); + test_case.add_expected_output(Shape{}, {1.561384797096252441f}); test_case.run(); } @@ -4640,15 +4648,15 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_negativelog_likelihood_loss) { auto test_case = test::TestCase(function, s_device); test_case.add_input({ - 0.54881352186203, 0.7151893377304077, 0.6027633547782898, 0.5448831915855408, 0.42365479469299316, - 0.6458941102027893, 0.4375872015953064, 0.891772985458374, 0.9636627435684204, 0.3834415078163147, - 0.7917250394821167, 0.5288949012756348, 0.5680445432662964, 0.9255966544151306, 0.07103605568408966, - 0.08712930232286453, 0.020218396559357643, 0.832619845867157, 0.7781567573547363, 0.8700121641159058, - 0.978618323802948, 0.7991585731506348, 0.4614793658256531, 0.7805292010307312, 0.11827442795038223, - 0.6399210095405579, 0.14335328340530396, 0.9446688890457153, 0.5218483209609985, 0.4146619439125061, + 0.54881352186203f, 0.7151893377304077f, 0.6027633547782898f, 0.5448831915855408f, 0.42365479469299316f, + 0.6458941102027893f, 0.4375872015953064f, 0.891772985458374f, 0.9636627435684204f, 0.3834415078163147f, + 0.7917250394821167f, 0.5288949012756348f, 0.5680445432662964f, 0.9255966544151306f, 0.07103605568408966f, + 0.08712930232286453f, 0.020218396559357643f, 0.832619845867157f, 0.7781567573547363f, 0.8700121641159058f, + 0.978618323802948f, 0.7991585731506348f, 0.4614793658256531f, 0.7805292010307312f, 0.11827442795038223f, + 0.6399210095405579f, 0.14335328340530396f, 0.9446688890457153f, 0.5218483209609985f, 0.4146619439125061f, }); test_case.add_input({3, 3, 2, 4, 2, 0}); - test_case.add_expected_output(Shape{}, {-0.531306922435760498}); + test_case.add_expected_output(Shape{}, {-0.531306922435760498f}); test_case.run(); } @@ -4958,19 +4966,19 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_einsum_sum) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/einsum_sum.onnx")); auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{3, 4}, - {1.764052345967664, - 0.4001572083672233, - 0.9787379841057392, - 2.240893199201458, - 1.8675579901499675, - -0.977277879876411, - 0.9500884175255894, - -0.1513572082976979, - -0.10321885179355784, - 0.41059850193837233, - 0.144043571160878, - 1.454273506962975}); - test_case.add_expected_output(Shape{3}, {5.3838407376420845, 1.689011319501448, 1.9056967282686674}); + {1.764052345967664f, + 0.4001572083672233f, + 0.9787379841057392f, + 2.240893199201458f, + 1.8675579901499675f, + -0.977277879876411f, + 0.9500884175255894f, + -0.1513572082976979f, + -0.10321885179355784f, + 0.41059850193837233f, + 0.144043571160878f, + 1.454273506962975f}); + test_case.add_expected_output(Shape{3}, {5.3838407376420845f, 1.689011319501448f, 1.9056967282686674f}); test_case.run(); } @@ -5074,7 +5082,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_random_uniform) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/random_uniform.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_expected_output(Shape{2, 2}, {43.45518, 48.67585, 42.227386, 40.86294}); + test_case.add_expected_output(Shape{2, 2}, {43.45518f, 48.67585f, 42.227386f, 40.86294f}); test_case.run(); } @@ -5085,7 +5093,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_random_uniform_like) { auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{2, 2}, {41, 42, 43, 44}); - test_case.add_expected_output(Shape{2, 2}, {43.45518, 48.67585, 42.227386, 40.86294}); + test_case.add_expected_output(Shape{2, 2}, {43.45518f, 48.67585f, 42.227386f, 40.86294f}); test_case.run(); } @@ -5094,7 +5102,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_random_normal) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/random_normal.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_expected_output(Shape{2, 2}, {13.459274, 41.75028, -19.311913, 131.79282}); + test_case.add_expected_output(Shape{2, 2}, {13.459274f, 41.75028f, -19.311913f, 131.79282f}); test_case.run(); } @@ -5105,50 +5113,50 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_random_normal_like) { auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{2, 2}, {0, 0, 0, 0}); - test_case.add_expected_output(Shape{2, 2}, {13.459274, 41.75028, -19.311913, 131.79282}); + test_case.add_expected_output(Shape{2, 2}, {13.459274f, 41.75028f, -19.311913f, 131.79282f}); test_case.run(); } -NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_2in) { +NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_2fin) { const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/aten_embedding_sum_packed_2in.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); test_case.add_input(Shape{3, 2}, {0, 2, 1, 2, 3, 4}); // indices - test_case.add_expected_output(Shape{3, 2}, {-2.1, -2.4, -2., -2.2, -0.19999999, 0.8}); + test_case.add_expected_output(Shape{3, 2}, {-2.1f, -2.4f, -2.f, -2.2f, -0.19999999f, 0.8f}); test_case.run(); } -NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_3in_offsets_none) { +NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_3fin_offsets_none) { const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/aten_embedding_sum_packed_3in_offset_none.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); test_case.add_input(Shape{3, 2}, {0, 2, 1, 2, 3, 4}); // indices - test_case.add_expected_output(Shape{3, 2}, {-2.1, -2.4, -2., -2.2, -0.19999999, 0.8}); + test_case.add_expected_output(Shape{3, 2}, {-2.1f, -2.4f, -2.f, -2.2f, -0.19999999f, 0.8f}); test_case.run(); } -NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_4in_per_sample_weights) { +NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_4fin_per_sample_weights) { const auto function = onnx_import::import_onnx_model( file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/aten_embedding_sum_packed_4in_per_sample_weights.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); - test_case.add_input(Shape{3, 2}, {0, 2, 1, 2, 3, 4}); // indices - test_case.add_input(Shape{3, 2}, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}); // per_sample_weights + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); + test_case.add_input(Shape{3, 2}, {0, 2, 1, 2, 3, 4}); // indices + test_case.add_input(Shape{3, 2}, {0.5f, 0.5f, 0.5f, 0.5f, 0.5f, 0.5f}); // per_sample_weights - test_case.add_expected_output(Shape{3, 2}, {-1.05, -1.2, -1., -1.1, -0.09999999, 0.4}); + test_case.add_expected_output(Shape{3, 2}, {-1.05f, -1.2f, -1.f, -1.1f, -0.09999999f, 0.4f}); test_case.run(); } @@ -5159,10 +5167,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_packed_sum_4in_two_none) { "onnx/aten_embedding_sum_packed_4in_two_none.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); test_case.add_input(Shape{3, 2}, {0, 2, 1, 2, 3, 4}); // indices - test_case.add_expected_output(Shape{3, 2}, {-2.1, -2.4, -2., -2.2, -0.19999999, 0.8}); + test_case.add_expected_output(Shape{3, 2}, {-2.1f, -2.4f, -2.f, -2.2f, -0.19999999f, 0.8f}); test_case.run(); } @@ -5173,11 +5181,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_offsets_sum_3in) { "onnx/aten_embedding_sum_offset_3in.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); test_case.add_input(Shape{4}, {0, 2, 3, 4}); // indices test_case.add_input(Shape{3}, {0, 2, 2}); // offsets - test_case.add_expected_output(Shape{3, 2}, {-2.1, -2.4, 0, 0, -0.2, 0.8}); + test_case.add_expected_output(Shape{3, 2}, {-2.1f, -2.4f, 0.f, 0.f, -0.2f, 0.8f}); test_case.run(); } @@ -5188,12 +5196,12 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_offsets_sum_4in) { "onnx/aten_embedding_sum_offset_4in.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); - test_case.add_input(Shape{4}, {0, 2, 3, 4}); // indices - test_case.add_input(Shape{3}, {0, 2, 2}); // offsets - test_case.add_input(Shape{4}, {0.5, 0.5, 0.5, 0.5}); // per_sample_weights + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); + test_case.add_input(Shape{4}, {0, 2, 3, 4}); // indices + test_case.add_input(Shape{3}, {0, 2, 2}); // offsets + test_case.add_input(Shape{4}, {0.5f, 0.5f, 0.5f, 0.5f}); // per_sample_weights - test_case.add_expected_output(Shape{3, 2}, {-1.05, -1.2, 0., 0., -0.09999999, 0.4}); + test_case.add_expected_output(Shape{3, 2}, {-1.05f, -1.2f, 0.f, 0.f, -0.09999999f, 0.4f}); test_case.run(); } @@ -5208,11 +5216,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_aten_embedding_bag_many_node_outputs) { EXPECT_EQ(function->get_results().size(), 1); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{5, 2}, {-0.2, -0.6, -0.1, -0.4, -1.9, -1.8, -1., 1.5, 0.8, -0.7}); + test_case.add_input(Shape{5, 2}, {-0.2f, -0.6f, -0.1f, -0.4f, -1.9f, -1.8f, -1.f, 1.5f, 0.8f, -0.7f}); test_case.add_input(Shape{4}, {0, 2, 3, 4}); // indices test_case.add_input(Shape{3}, {0, 2, 2}); // offsets - test_case.add_expected_output(Shape{3, 2}, {-2.1, -2.4, 0, 0, -0.2, 0.8}); + test_case.add_expected_output(Shape{3, 2}, {-2.1f, -2.4f, 0.f, 0.f, -0.2f, 0.8f}); test_case.run(); } @@ -5388,13 +5396,14 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_fib_like_input_rev) { auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{}, {0}); test_case.add_input(Shape{}, {1}); - test_case.add_input(Shape{10}, std::vector{0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}); + test_case.add_input(Shape{10}, + std::vector{0.f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f}); - test_case.add_expected_output(Shape{}, {0.14897026}); - test_case.add_expected_output(Shape{}, {0.}); + test_case.add_expected_output(Shape{}, {0.14897026f}); + test_case.add_expected_output(Shape{}, {0.f}); test_case.add_expected_output( Shape{10}, - {0.9, 1.52, 1.694, 1.9284, 1.8112, 1.4958401, 0.9921121, 0.49759045, 0.14897026, 0.}); + {0.9f, 1.52f, 1.694f, 1.9284f, 1.8112f, 1.4958401f, 0.9921121f, 0.49759045f, 0.14897026f, 0.f}); test_case.run(); } @@ -5407,13 +5416,14 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_fib_like_input_out_rev) { auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{}, {0}); test_case.add_input(Shape{}, {1}); - test_case.add_input(Shape{10}, std::vector{0., 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}); + test_case.add_input(Shape{10}, + std::vector{0.f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f}); - test_case.add_expected_output(Shape{}, {0.14897026}); + test_case.add_expected_output(Shape{}, {0.14897026f}); test_case.add_expected_output(Shape{}, {0.}); test_case.add_expected_output( Shape{10}, - {0., 0.14897026, 0.49759045, 0.9921121, 1.4958401, 1.8112, 1.9284, 1.694, 1.52, 0.9}); + {0.f, 0.14897026f, 0.49759045f, 0.9921121f, 1.4958401f, 1.8112f, 1.9284f, 1.694f, 1.52f, 0.9f}); test_case.run(); } @@ -5435,27 +5445,29 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_ND_mixed_ones) { test_case.run(); } -NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_ND_mixed_vals) { +NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15f_ND_mixed_vals) { const auto function = onnx_import::import_onnx_model( file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/scan15_ND_mixed.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 3, 2}, {0, 0, 0, 0, 0, 0}); - test_case.add_input(Shape{1, 3, 2}, {1, 1, 1, 1, 1, 1}); - std::vector sequence_vals{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1., 1.1, 1.2, 1.3, 1.4, 1.5, - 1.6, 1.7, 1.8, 1.9, 2., 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.}; + test_case.add_input(Shape{1, 3, 2}, {0.f, 0.f, 0.f, 0.f, 0.f, 0.f}); + test_case.add_input(Shape{1, 3, 2}, {1.f, 1.f, 1.f, 1.f, 1.f, 1.f}); + std::vector sequence_vals{0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.f, + 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f, 1.7f, 1.8f, 1.9f, 2.f, + 2.1f, 2.2f, 2.3f, 2.4f, 2.5f, 2.6f, 2.7f, 2.8f, 2.9f, 3.f}; test_case.add_input(Shape{1, 3, 5, 2}, sequence_vals); // multiply factor (reverse) test_case.add_input(Shape{1, 5, 3, 2}, sequence_vals); // div factor test_case.add_expected_output(Shape{1, 3, 2}, - {2.7327938, 2.1428573, 21.070545, 16.92727, 49.765778, 41.444443}); + {2.7327938f, 2.1428573f, 21.070545f, 16.92727f, 49.765778f, 41.444443f}); test_case.add_expected_output(Shape{1, 3, 2}, - {0.40161943, 0.5274726, 16.80789, 14.025973, 59.98805, 50.518517}); + {0.40161943f, 0.5274726f, 16.80789f, 14.025973f, 59.98805f, 50.518517f}); test_case.add_expected_output( Shape{1, 3, 2, 5}, - {0.40161943, 2.7327938, 7.3076925, 10., 9., 0.5274726, 2.1428573, 4.714286, 6., 5., - 16.80789, 21.070545, 20.185184, 13.851851, 6.333333, 14.025973, 16.92727, 15.799998, 10.799999, 5., - 59.98805, 49.765778, 33.074867, 16.690908, 5.8, 50.518517, 41.444443, 27.444445, 14., 5.}); + {0.40161943f, 2.7327938f, 7.3076925f, 10.f, 9.f, 0.5274726f, 2.1428573f, 4.714286f, + 6.f, 5.f, 16.80789f, 21.070545f, 20.185184f, 13.851851f, 6.333333f, 14.025973f, + 16.92727f, 15.799998f, 10.799999f, 5.f, 59.98805f, 49.765778f, 33.074867f, 16.690908f, + 5.8f, 50.518517f, 41.444443f, 27.444445f, 14.f, 5.f}); test_case.run(); } @@ -5466,22 +5478,24 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_ND_mixed_vals_neg_axes) { "onnx/scan15_ND_mixed_neg_axes.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 3, 2}, {0, 0, 0, 0, 0, 0}); - test_case.add_input(Shape{1, 3, 2}, {1, 1, 1, 1, 1, 1}); - std::vector sequence_vals{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1., 1.1, 1.2, 1.3, 1.4, 1.5, - 1.6, 1.7, 1.8, 1.9, 2., 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.}; + test_case.add_input(Shape{1, 3, 2}, {0.f, 0.f, 0.f, 0.f, 0.f, 0.f}); + test_case.add_input(Shape{1, 3, 2}, {1.f, 1.f, 1.f, 1.f, 1.f, 1.f}); + std::vector sequence_vals{0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.f, + 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f, 1.7f, 1.8f, 1.9f, 2.f, + 2.1f, 2.2f, 2.3f, 2.4f, 2.5f, 2.6f, 2.7f, 2.8f, 2.9f, 3.f}; test_case.add_input(Shape{1, 3, 5, 2}, sequence_vals); // multiply factor (reverse) test_case.add_input(Shape{1, 5, 3, 2}, sequence_vals); // div factor test_case.add_expected_output(Shape{1, 3, 2}, - {2.7327938, 2.1428573, 21.070545, 16.92727, 49.765778, 41.444443}); + {2.7327938f, 2.1428573f, 21.070545f, 16.92727f, 49.765778f, 41.444443f}); test_case.add_expected_output(Shape{1, 3, 2}, - {0.40161943, 0.5274726, 16.80789, 14.025973, 59.98805, 50.518517}); + {0.40161943f, 0.5274726f, 16.80789f, 14.025973f, 59.98805f, 50.518517f}); test_case.add_expected_output( Shape{1, 3, 2, 5}, - {0.40161943, 2.7327938, 7.3076925, 10., 9., 0.5274726, 2.1428573, 4.714286, 6., 5., - 16.80789, 21.070545, 20.185184, 13.851851, 6.333333, 14.025973, 16.92727, 15.799998, 10.799999, 5., - 59.98805, 49.765778, 33.074867, 16.690908, 5.8, 50.518517, 41.444443, 27.444445, 14., 5.}); + {0.40161943f, 2.7327938f, 7.3076925f, 10.f, 9.f, 0.5274726f, 2.1428573f, 4.714286f, + 6.f, 5.f, 16.80789f, 21.070545f, 20.185184f, 13.851851f, 6.333333f, 14.025973f, + 16.92727f, 15.799998f, 10.799999f, 5.f, 59.98805f, 49.765778f, 33.074867f, 16.690908f, + 5.8f, 50.518517f, 41.444443f, 27.444445f, 14.f, 5.f}); test_case.run(); } @@ -5490,22 +5504,24 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_dyn_rank_vals) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/scan15_dyn_rank.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 3, 2}, {0, 0, 0, 0, 0, 0}); - test_case.add_input(Shape{1, 3, 2}, {1, 1, 1, 1, 1, 1}); - std::vector sequence_vals{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1., 1.1, 1.2, 1.3, 1.4, 1.5, - 1.6, 1.7, 1.8, 1.9, 2., 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.}; + test_case.add_input(Shape{1, 3, 2}, {0.f, 0.f, 0.f, 0.f, 0.f, 0.f}); + test_case.add_input(Shape{1, 3, 2}, {1.f, 1.f, 1.f, 1.f, 1.f, 1.f}); + std::vector sequence_vals{0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.f, + 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f, 1.7f, 1.8f, 1.9f, 2.f, + 2.1f, 2.2f, 2.3f, 2.4f, 2.5f, 2.6f, 2.7f, 2.8f, 2.9f, 3.f}; test_case.add_input(Shape{1, 3, 5, 2}, sequence_vals); // multiply factor (reverse) test_case.add_input(Shape{1, 5, 3, 2}, sequence_vals); // div factor test_case.add_expected_output(Shape{1, 3, 2}, - {2.7327938, 2.1428573, 21.070545, 16.92727, 49.765778, 41.444443}); + {2.7327938f, 2.1428573f, 21.070545f, 16.92727f, 49.765778f, 41.444443f}); test_case.add_expected_output(Shape{1, 3, 2}, - {0.40161943, 0.5274726, 16.80789, 14.025973, 59.98805, 50.518517}); + {0.40161943f, 0.5274726f, 16.80789f, 14.025973f, 59.98805f, 50.518517f}); test_case.add_expected_output( Shape{1, 3, 2, 5}, - {0.40161943, 2.7327938, 7.3076925, 10., 9., 0.5274726, 2.1428573, 4.714286, 6., 5., - 16.80789, 21.070545, 20.185184, 13.851851, 6.333333, 14.025973, 16.92727, 15.799998, 10.799999, 5., - 59.98805, 49.765778, 33.074867, 16.690908, 5.8, 50.518517, 41.444443, 27.444445, 14., 5.}); + {0.40161943f, 2.7327938f, 7.3076925f, 10.f, 9.f, 0.5274726f, 2.1428573f, 4.714286f, + 6.f, 5.f, 16.80789f, 21.070545f, 20.185184f, 13.851851f, 6.333333f, 14.025973f, + 16.92727f, 15.799998f, 10.799999f, 5.f, 59.98805f, 49.765778f, 33.074867f, 16.690908f, + 5.8f, 50.518517f, 41.444443f, 27.444445f, 14.f, 5.f}); test_case.run(); } @@ -5529,43 +5545,46 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan15_ND_b4_input_rev_vals) { "onnx/scan15_ND_b4_input_rev.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{4, 3, 2}, std::vector(24, 0)); - test_case.add_input(Shape{4, 3, 2}, std::vector(24, 1)); + test_case.add_input(Shape{4, 3, 2}, std::vector(24, 0.f)); + test_case.add_input(Shape{4, 3, 2}, std::vector(24, 1.f)); std::vector sequence_vals{ - 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1., 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, - 1.9, 2., 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3., 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, - 3.7, 3.8, 3.9, 4., 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5., 5.1, 5.2, 5.3, 5.4, - 5.5, 5.6, 5.7, 5.8, 5.9, 6., 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8, 6.9, 7., 7.1, 7.2, - 7.3, 7.4, 7.5, 7.6, 7.7, 7.8, 7.9, 8., 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9., - 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9, 10., 10.1, 10.2, 10.3, 10.4, 10.5, 10.6, 10.7, 10.8, - 10.9, 11., 11.1, 11.2, 11.3, 11.4, 11.5, 11.6, 11.7, 11.8, 11.9, 12.}; - test_case.add_input(Shape{4, 5, 3, 2}, sequence_vals); // multiply factor (reverse) + 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.f, 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, + 1.6f, 1.7f, 1.8f, 1.9f, 2.f, 2.1f, 2.2f, 2.3f, 2.4f, 2.5f, 2.6f, 2.7f, 2.8f, 2.9f, 3.f, + 3.1f, 3.2f, 3.3f, 3.4f, 3.5f, 3.6f, 3.7f, 3.8f, 3.9f, 4.f, 4.1f, 4.2f, 4.3f, 4.4f, 4.5f, + 4.6f, 4.7f, 4.8f, 4.9f, 5.f, 5.1f, 5.2f, 5.3f, 5.4f, 5.5f, 5.6f, 5.7f, 5.8f, 5.9f, 6.f, + 6.1f, 6.2f, 6.3f, 6.4f, 6.5f, 6.6f, 6.7f, 6.8f, 6.9f, 7.f, 7.1f, 7.2f, 7.3f, 7.4f, 7.5f, + 7.6f, 7.7f, 7.8f, 7.9f, 8.f, 8.1f, 8.2f, 8.3f, 8.4f, 8.5f, 8.6f, 8.7f, 8.8f, 8.9f, 9.f, + 9.1f, 9.2f, 9.3f, 9.4f, 9.5f, 9.6f, 9.7f, 9.8f, 9.9f, 10.f, 10.1f, 10.2f, 10.3f, 10.4f, 10.5f, + 10.6f, 10.7f, 10.8f, 10.9f, 11.f, 11.1f, 11.2f, 11.3f, 11.4f, 11.5f, 11.6f, 11.7f, 11.8f, 11.9f, 12.f}; + test_case.add_input(Shape{4, 5, 3, 2}, sequence_vals); // multiply factor (areverse) test_case.add_input(Shape{4, 5, 3, 2}, sequence_vals); // div factor test_case.add_expected_output( Shape{4, 3, 2}, - {61.210526, 33.2, 23.857145, 19.181818, 16.373913, 14.5, 6.8880844, 6.83, - 6.7754016, 6.7239814, 6.6754713, 6.6296296, 5.9686656, 5.953226, 5.9382715, 5.9237804, - 5.9097314, 5.896105, 5.652082, 5.645059, 5.638186, 5.6314588, 5.624872, 5.618421}); + {61.210526f, 33.2f, 23.857145f, 19.181818f, 16.373913f, 14.5f, 6.8880844f, 6.83f, + 6.7754016f, 6.7239814f, 6.6754713f, 6.6296296f, 5.9686656f, 5.953226f, 5.9382715f, 5.9237804f, + 5.9097314f, 5.896105f, 5.652082f, 5.645059f, 5.638186f, 5.6314588f, 5.624872f, 5.618421f}); test_case.add_expected_output( Shape{4, 3, 2}, - {6.271278, 6.2461543, 6.2433867, 6.2545457, 6.2744985, 6.3, 6.9531364, 6.970527, - 6.987378, 7.003712, 7.019554, 7.034921, 7.30868, 7.3164845, 7.324116, 7.3315806, - 7.338885, 7.346032, 7.485426, 7.489783, 7.494067, 7.49828, 7.5024257, 7.506502}); + {6.271278f, 6.2461543f, 6.2433867f, 6.2545457f, 6.2744985f, 6.3f, 6.9531364f, 6.970527f, + 6.987378f, 7.003712f, 7.019554f, 7.034921f, 7.30868f, 7.3164845f, 7.324116f, 7.3315806f, + 7.338885f, 7.346032f, 7.485426f, 7.489783f, 7.494067f, 7.49828f, 7.5024257f, 7.506502f}); test_case.add_expected_output( Shape{5, 4, 3, 2}, - {25., 13., 9., 7., 5.8, 5., 1.7741936, 1.75, 1.7272727, 1.7058823, - 1.6857144, 1.6666667, 1.3934426, 1.3870969, 1.3809522, 1.375, 1.3692307, 1.3636364, 1.2637362, 1.2608696, - 1.2580644, 1.2553192, 1.2526315, 1.25, 70.57143, 35., 23.333334, 17.6, 14.218181, 12., - 3.6739323, 3.618421, 3.5664334, 3.5176468, 3.471777, 3.4285717, 2.822119, 2.8083491, 2.7950313, 2.7821426, - 2.7696643, 2.757576, 2.543786, 2.5377107, 2.5317693, 2.5259573, 2.520271, 2.514706, 95.57143, 47.999996, - 32.333336, 24.6, 20.01818, 17., 5.448126, 5.368421, 5.293706, 5.223529, 5.157491, 5.0952387, - 4.215562, 4.195446, 4.1759834, 4.1571426, 4.138895, 4.1212125, 3.8075223, 3.7985802, 3.7898335, 3.7812767, - 3.7729027, 3.764706, 61.210526, 33.2, 23.857145, 19.181818, 16.373913, 14.5, 6.8880844, 6.83, - 6.7754016, 6.7239814, 6.6754713, 6.6296296, 5.9686656, 5.953226, 5.9382715, 5.9237804, 5.9097314, 5.896105, - 5.652082, 5.645059, 5.638186, 5.6314588, 5.624872, 5.618421, 6.271278, 6.2461543, 6.2433867, 6.2545457, - 6.2744985, 6.3, 6.9531364, 6.970527, 6.987378, 7.003712, 7.019554, 7.034921, 7.30868, 7.3164845, - 7.324116, 7.3315806, 7.338885, 7.346032, 7.485426, 7.489783, 7.494067, 7.49828, 7.5024257, 7.506502}); + {25.f, 13.f, 9.f, 7.f, 5.8f, 5.f, 1.7741936f, 1.75f, 1.7272727f, + 1.7058823f, 1.6857144f, 1.6666667f, 1.3934426f, 1.3870969f, 1.3809522f, 1.375f, 1.3692307f, 1.3636364f, + 1.2637362f, 1.2608696f, 1.2580644f, 1.2553192f, 1.2526315f, 1.25f, 70.57143f, 35.f, 23.333334f, + 17.6f, 14.218181f, 12.f, 3.6739323f, 3.618421f, 3.5664334f, 3.5176468f, 3.471777f, 3.4285717f, + 2.822119f, 2.8083491f, 2.7950313f, 2.7821426f, 2.7696643f, 2.757576f, 2.543786f, 2.5377107f, 2.5317693f, + 2.5259573f, 2.520271f, 2.514706f, 95.57143f, 47.999996f, 32.333336f, 24.6f, 20.01818f, 17.f, + 5.448126f, 5.368421f, 5.293706f, 5.223529f, 5.157491f, 5.0952387f, 4.215562f, 4.195446f, 4.1759834f, + 4.1571426f, 4.138895f, 4.1212125f, 3.8075223f, 3.7985802f, 3.7898335f, 3.7812767f, 3.7729027f, 3.764706f, + 61.210526f, 33.2f, 23.857145f, 19.181818f, 16.373913f, 14.5f, 6.8880844f, 6.83f, 6.7754016f, + 6.7239814f, 6.6754713f, 6.6296296f, 5.9686656f, 5.953226f, 5.9382715f, 5.9237804f, 5.9097314f, 5.896105f, + 5.652082f, 5.645059f, 5.638186f, 5.6314588f, 5.624872f, 5.618421f, 6.271278f, 6.2461543f, 6.2433867f, + 6.2545457f, 6.2744985f, 6.3f, 6.9531364f, 6.970527f, 6.987378f, 7.003712f, 7.019554f, 7.034921f, + 7.30868f, 7.3164845f, 7.324116f, 7.3315806f, 7.338885f, 7.346032f, 7.485426f, 7.489783f, 7.494067f, + 7.49828f, 7.5024257f, 7.506502f}); test_case.run(); } @@ -5600,43 +5619,46 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_scan8_ND_b4_input_rev_vals) { "onnx/scan8_ND_b4_input_rev.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{4, 3, 2}, std::vector(24, 0)); - test_case.add_input(Shape{4, 3, 2}, std::vector(24, 1)); + test_case.add_input(Shape{4, 3, 2}, std::vector(24, 0.f)); + test_case.add_input(Shape{4, 3, 2}, std::vector(24, 1.f)); std::vector sequence_vals{ - 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1., 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, - 1.9, 2., 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3., 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, - 3.7, 3.8, 3.9, 4., 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5., 5.1, 5.2, 5.3, 5.4, - 5.5, 5.6, 5.7, 5.8, 5.9, 6., 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8, 6.9, 7., 7.1, 7.2, - 7.3, 7.4, 7.5, 7.6, 7.7, 7.8, 7.9, 8., 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9., - 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9, 10., 10.1, 10.2, 10.3, 10.4, 10.5, 10.6, 10.7, 10.8, - 10.9, 11., 11.1, 11.2, 11.3, 11.4, 11.5, 11.6, 11.7, 11.8, 11.9, 12.}; + 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.f, 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, + 1.6f, 1.7f, 1.8f, 1.9f, 2.f, 2.1f, 2.2f, 2.3f, 2.4f, 2.5f, 2.6f, 2.7f, 2.8f, 2.9f, 3.f, + 3.1f, 3.2f, 3.3f, 3.4f, 3.5f, 3.6f, 3.7f, 3.8f, 3.9f, 4.f, 4.1f, 4.2f, 4.3f, 4.4f, 4.5f, + 4.6f, 4.7f, 4.8f, 4.9f, 5.f, 5.1f, 5.2f, 5.3f, 5.4f, 5.5f, 5.6f, 5.7f, 5.8f, 5.9f, 6.f, + 6.1f, 6.2f, 6.3f, 6.4f, 6.5f, 6.6f, 6.7f, 6.8f, 6.9f, 7.f, 7.1f, 7.2f, 7.3f, 7.4f, 7.5f, + 7.6f, 7.7f, 7.8f, 7.9f, 8.f, 8.1f, 8.2f, 8.3f, 8.4f, 8.5f, 8.6f, 8.7f, 8.8f, 8.9f, 9.f, + 9.1f, 9.2f, 9.3f, 9.4f, 9.5f, 9.6f, 9.7f, 9.8f, 9.9f, 10.f, 10.1f, 10.2f, 10.3f, 10.4f, 10.5f, + 10.6f, 10.7f, 10.8f, 10.9f, 11.f, 11.1f, 11.2f, 11.3f, 11.4f, 11.5f, 11.6f, 11.7f, 11.8f, 11.9f, 12.f}; test_case.add_input(Shape{4, 5, 3, 2}, sequence_vals); // multiply factor (reverse) test_case.add_input(Shape{4, 5, 3, 2}, sequence_vals); // div factor test_case.add_expected_output( Shape{4, 3, 2}, - {61.210526, 33.2, 23.857145, 19.181818, 16.373913, 14.5, 6.8880844, 6.83, - 6.7754016, 6.7239814, 6.6754713, 6.6296296, 5.9686656, 5.953226, 5.9382715, 5.9237804, - 5.9097314, 5.896105, 5.652082, 5.645059, 5.638186, 5.6314588, 5.624872, 5.618421}); + {61.210526f, 33.2f, 23.857145f, 19.181818f, 16.373913f, 14.5f, 6.8880844f, 6.83f, + 6.7754016f, 6.7239814f, 6.6754713f, 6.6296296f, 5.9686656f, 5.953226f, 5.9382715f, 5.9237804f, + 5.9097314f, 5.896105f, 5.652082f, 5.645059f, 5.638186f, 5.6314588f, 5.624872f, 5.618421f}); test_case.add_expected_output( Shape{4, 3, 2}, - {6.271278, 6.2461543, 6.2433867, 6.2545457, 6.2744985, 6.3, 6.9531364, 6.970527, - 6.987378, 7.003712, 7.019554, 7.034921, 7.30868, 7.3164845, 7.324116, 7.3315806, - 7.338885, 7.346032, 7.485426, 7.489783, 7.494067, 7.49828, 7.5024257, 7.506502}); + {6.271278f, 6.2461543f, 6.2433867f, 6.2545457f, 6.2744985f, 6.3f, 6.9531364f, 6.970527f, + 6.987378f, 7.003712f, 7.019554f, 7.034921f, 7.30868f, 7.3164845f, 7.324116f, 7.3315806f, + 7.338885f, 7.346032f, 7.485426f, 7.489783f, 7.494067f, 7.49828f, 7.5024257f, 7.506502f}); test_case.add_expected_output( Shape{4, 5, 3, 2}, - {25., 13., 9., 7., 5.8, 5., 70.57143, 35., 23.333334, 17.6, - 14.218181, 12., 95.57143, 47.999996, 32.333336, 24.6, 20.01818, 17., 61.210526, 33.2, - 23.857145, 19.181818, 16.373913, 14.5, 6.271278, 6.2461543, 6.2433867, 6.2545457, 6.2744985, 6.3, - 1.7741936, 1.75, 1.7272727, 1.7058823, 1.6857144, 1.6666667, 3.6739323, 3.618421, 3.5664334, 3.5176468, - 3.471777, 3.4285717, 5.448126, 5.368421, 5.293706, 5.223529, 5.157491, 5.0952387, 6.8880844, 6.83, - 6.7754016, 6.7239814, 6.6754713, 6.6296296, 6.9531364, 6.970527, 6.987378, 7.003712, 7.019554, 7.034921, - 1.3934426, 1.3870969, 1.3809522, 1.375, 1.3692307, 1.3636364, 2.822119, 2.8083491, 2.7950313, 2.7821426, - 2.7696643, 2.757576, 4.215562, 4.195446, 4.1759834, 4.1571426, 4.138895, 4.1212125, 5.9686656, 5.953226, - 5.9382715, 5.9237804, 5.9097314, 5.896105, 7.30868, 7.3164845, 7.324116, 7.3315806, 7.338885, 7.346032, - 1.2637362, 1.2608696, 1.2580644, 1.2553192, 1.2526315, 1.25, 2.543786, 2.5377107, 2.5317693, 2.5259573, - 2.520271, 2.514706, 3.8075223, 3.7985802, 3.7898335, 3.7812767, 3.7729027, 3.764706, 5.652082, 5.645059, - 5.638186, 5.6314588, 5.624872, 5.618421, 7.485426, 7.489783, 7.494067, 7.49828, 7.5024257, 7.506502}); + {25.f, 13.f, 9.f, 7.f, 5.8f, 5.f, 70.57143f, 35.f, 23.333334f, + 17.6f, 14.218181f, 12.f, 95.57143f, 47.999996f, 32.333336f, 24.6f, 20.01818f, 17.f, + 61.210526f, 33.2f, 23.857145f, 19.181818f, 16.373913f, 14.5f, 6.271278f, 6.2461543f, 6.2433867f, + 6.2545457f, 6.2744985f, 6.3f, 1.7741936f, 1.75f, 1.7272727f, 1.7058823f, 1.6857144f, 1.6666667f, + 3.6739323f, 3.618421f, 3.5664334f, 3.5176468f, 3.471777f, 3.4285717f, 5.448126f, 5.368421f, 5.293706f, + 5.223529f, 5.157491f, 5.0952387f, 6.8880844f, 6.83f, 6.7754016f, 6.7239814f, 6.6754713f, 6.6296296f, + 6.9531364f, 6.970527f, 6.987378f, 7.003712f, 7.019554f, 7.034921f, 1.3934426f, 1.3870969f, 1.3809522f, + 1.375f, 1.3692307f, 1.3636364f, 2.822119f, 2.8083491f, 2.7950313f, 2.7821426f, 2.7696643f, 2.757576f, + 4.215562f, 4.195446f, 4.1759834f, 4.1571426f, 4.138895f, 4.1212125f, 5.9686656f, 5.953226f, 5.9382715f, + 5.9237804f, 5.9097314f, 5.896105f, 7.30868f, 7.3164845f, 7.324116f, 7.3315806f, 7.338885f, 7.346032f, + 1.2637362f, 1.2608696f, 1.2580644f, 1.2553192f, 1.2526315f, 1.25f, 2.543786f, 2.5377107f, 2.5317693f, + 2.5259573f, 2.520271f, 2.514706f, 3.8075223f, 3.7985802f, 3.7898335f, 3.7812767f, 3.7729027f, 3.764706f, + 5.652082f, 5.645059f, 5.638186f, 5.6314588f, 5.624872f, 5.618421f, 7.485426f, 7.489783f, 7.494067f, + 7.49828f, 7.5024257f, 7.506502f}); test_case.run(); } @@ -5658,10 +5680,18 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softsign) { auto model = onnx_import::import_onnx_model( file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/softsign.onnx")); - Inputs inputs{std::vector{1.0, 0.1, 20.0, 12.0, -12.0, -0.2, 0.5, 100.0, 0.0, -1.0}}; + Inputs inputs{std::vector{1.0f, 0.1f, 20.0f, 12.0f, -12.0f, -0.2f, 0.5f, 100.0f, 0.0f, -1.0f}}; - std::vector - output{0.5, 0.09090909, 0.95238096, 0.9230769, -0.9230769, -0.16666666, 0.33333334, 0.990099, 0., -0.5}; + std::vector output{0.5f, + 0.09090909f, + 0.95238096f, + 0.9230769f, + -0.9230769f, + -0.16666666f, + 0.33333334f, + 0.990099f, + 0.f, + -0.5f}; auto test_case = test::TestCase(model, s_device); test_case.add_multiple_inputs(inputs); @@ -6215,7 +6245,7 @@ NGRAPH_TEST(${BACKEND_NAME}, castlike_float16_to_uint32) { auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 1, 2, 2}, std::vector{1.5, 2.3, 3, 4}); + test_case.add_input(Shape{1, 1, 2, 2}, std::vector{1.5f, 2.3f, 3.f, 4.f}); test_case.add_input(Shape{4}, {1, 2, 3, 4}); test_case.add_expected_output(std::vector{1, 2, 3, 4}); @@ -6229,7 +6259,7 @@ NGRAPH_TEST(${BACKEND_NAME}, castlike_float16_to_int64) { auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 1, 2, 2}, std::vector{1.5, 2.3, 3, 4}); + test_case.add_input(Shape{1, 1, 2, 2}, std::vector{1.5f, 2.3f, 3.f, 4.f}); test_case.add_input(Shape{4}, {1, 2, 3, 4}); test_case.add_expected_output(std::vector{1, 2, 3, 4}); @@ -6287,7 +6317,7 @@ NGRAPH_TEST(${BACKEND_NAME}, castlike_int32_to_float) { test_case.add_input(Shape{1, 1, 2, 2}, std::vector{-1, 2, 3, 4}); test_case.add_input(Shape{4}, {1, 2, 3, 4}); - test_case.add_expected_output(std::vector{-1.0, 2.0, 3.0, 4.0}); + test_case.add_expected_output(std::vector{-1.0f, 2.0f, 3.0f, 4.0f}); test_case.run(); } @@ -6299,7 +6329,7 @@ NGRAPH_TEST(${BACKEND_NAME}, castlike_float64_to_int32) { auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 1, 2, 2}, std::vector{-107374.9876543, -2.2, 3.3, 4.4}); + test_case.add_input(Shape{1, 1, 2, 2}, std::vector{-107374.9876543f, -2.2f, 3.3f, 4.4f}); test_case.add_input(Shape{4}, {1, 2, 3, 4}); test_case.add_expected_output(std::vector{-107374, -2, 3, 4}); @@ -6313,9 +6343,13 @@ NGRAPH_TEST(${BACKEND_NAME}, DISABLED_castlike_float32_to_bfloat16) { auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 4}, std::vector{121.5, 122.7, 3, 4, 5, 6, 7, 8.8, 9, 10, 11, 12}); - test_case.add_input(Shape{3, 4}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}); - test_case.add_expected_output(std::vector{121.5, 122.7, 3, 4, 5, 6, 7, 8.8, 9, 10, 11, 12}); + test_case.add_input( + Shape{3, 4}, + std::vector{121.5f, 122.7f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.8f, 9.f, 10.f, 11.f, 12.f}); + test_case.add_input(Shape{3, 4}, + {1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f, 9.5f, 10.5f, 11.5f, 12.5f}); + test_case.add_expected_output( + std::vector{121.5f, 122.7f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.8f, 9.f, 10.f, 11.f, 12.f}); test_case.run(); } @@ -6327,9 +6361,12 @@ NGRAPH_TEST(${BACKEND_NAME}, DISABLED_castlike_bfloat16_to_float32) { auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 4}, std::vector{121.5, 122.7, 3, 4, 5, 6, 7, 8.8, 9, 10, 11, 12}); + test_case.add_input( + Shape{3, 4}, + std::vector{121.5f, 122.7f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.8f, 9.f, 10.f, 11.f, 12.f}); test_case.add_input(Shape{3, 4}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}); - test_case.add_expected_output(std::vector{121.5, 122.7, 3, 4, 5, 6, 7, 8.75, 9, 10, 11, 12}); + test_case.add_expected_output( + std::vector{121.5f, 122.7f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.75f, 9.f, 10.f, 11.f, 12.f}); test_case.run(); } @@ -6415,249 +6452,3 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_unique_3d_with_duplicates_and_axis_2) { test_case.run(); } - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft) { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, - 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, - 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, - 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, - 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); - test_case.add_expected_output( - Shape{3, 5, 2}, - {10.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, -2.500000f, -0.812299f, - -2.500000f, -3.440955f, 35.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, - -2.500000f, -0.812299f, -2.500000f, -3.440955f, 60.000000f, 0.000000f, -2.500000f, 3.440955f, - -2.500000f, 0.812299f, -2.500000f, -0.812299f, -2.500000f, -3.440955f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_only_real) { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft_only_real.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 5, 1}, - { - 0.000000f, - 1.000000f, - 2.000000f, - 3.000000f, - 4.000000f, - 5.000000f, - 6.000000f, - 7.000000f, - 8.000000f, - 9.000000f, - 10.000000f, - 11.000000f, - 12.000000f, - 13.000000f, - 14.000000f, - }); - test_case.add_expected_output( - Shape{3, 5, 2}, - {10.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, -2.500000f, -0.812299f, - -2.500000f, -3.440955f, 35.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, - -2.500000f, -0.812299f, -2.500000f, -3.440955f, 60.000000f, 0.000000f, -2.500000f, 3.440955f, - -2.500000f, 0.812299f, -2.500000f, -0.812299f, -2.500000f, -3.440955f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_onesided) { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft_onesided.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input( - Shape{2, 4}, - {0.000000f, 1.000000f, 2.000000f, 3.000000f, 4.000000f, 5.000000f, 6.000000f, 7.000000f}); - test_case.add_expected_output(Shape{2, 3, 2}, - {6.000000f, - 0.000000f, - -2.000000f, - 2.000000f, - -2.000000f, - 0.000000f, - 22.000000f, - 0.000000f, - -2.000000f, - 2.000000f, - -2.000000f, - 0.000000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_onesided_skip_convert_to_complex) { - auto function = - onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_onesided_skip_convert_to_complex.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input( - Shape{2, 4, 1}, - {0.000000f, 1.000000f, 2.000000f, 3.000000f, 4.000000f, 5.000000f, 6.000000f, 7.000000f}); - test_case.add_expected_output(Shape{2, 3, 1, 2}, - {6.000000f, - 0.000000f, - -2.000000f, - 2.000000f, - -2.000000f, - 0.000000f, - 22.000000f, - 0.000000f, - -2.000000f, - 2.000000f, - -2.000000f, - 0.000000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_length_provided) { - auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_lenght_provided.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, - 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, - 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, - 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, - 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); - test_case.add_expected_output( - Shape{1, 5, 2}, - {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, 3.000000f, 0.000000f, 4.000000f, 0.000000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_length_provided_onesided) { - auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_lenght_provided_onesided.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{4, 3}, - {0.000000f, - 1.000000f, - 2.000000f, - 3.000000f, - 4.000000f, - 5.000000f, - 6.000000f, - 7.000000f, - 8.000000f, - 9.000000f, - 10.000000f, - 11.000000f}); - test_case.add_expected_output(Shape{1, 3, 2}, - {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse) { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft_inverse.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, - 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, - 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, - 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, - 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); - test_case.add_expected_output( - Shape{3, 5, 2}, - {2.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, -0.500000f, 0.162460f, - -0.500000f, 0.688191f, 7.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, - -0.500000f, 0.162460f, -0.500000f, 0.688191f, 12.000000f, 0.000000f, -0.500000f, -0.688191f, - -0.500000f, -0.162460f, -0.500000f, 0.162460f, -0.500000f, 0.688191f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_only_real) { - auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_inverse_only_real.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 5, 1}, - {0.000000f, - 1.000000f, - 2.000000f, - 3.000000f, - 4.000000f, - 5.000000f, - 6.000000f, - 7.000000f, - 8.000000f, - 9.000000f, - 10.000000f, - 11.000000f, - 12.000000f, - 13.000000f, - 14.000000f}); - test_case.add_expected_output( - Shape{3, 5, 2}, - {2.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, -0.500000f, 0.162460f, - -0.500000f, 0.688191f, 7.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, - -0.500000f, 0.162460f, -0.500000f, 0.688191f, 12.000000f, 0.000000f, -0.500000f, -0.688191f, - -0.500000f, -0.162460f, -0.500000f, 0.162460f, -0.500000f, 0.688191f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_onesided) { - auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_inverse_onesided.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{2, 3, 2}, - {6.000000f, - 0.000000f, - -2.000000f, - 2.000000f, - -2.000000f, - 0.000000f, - 22.000000f, - 0.000000f, - -2.000000f, - 2.000000f, - -2.000000f, - 0.000000f}); - test_case.add_expected_output( - Shape{2, 4}, - {0.000000f, 1.000000f, 2.000000f, 3.000000f, 4.000000f, 5.000000f, 6.000000f, 7.000000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_onesided_real_input) { - auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_inverse_onesided_real_input.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{2, 3, 1}, {1.000000f, 0.000000f, -1.000000f, 0.5000000f, -0.5000000f, 0.000000f}); - test_case.add_expected_output(Shape{2, 3, 1}, - {0.750000f, -0.250000f, -0.500000f, 0.250000f, 0.250000f, -0.500000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inversed_length_provided) { - auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_inversed_lenght_provided.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, - 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, - 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, - 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, - 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); - test_case.add_expected_output( - Shape{1, 5, 2}, - {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, 3.000000f, 0.000000f, 4.000000f, 0.000000f}); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_length_provided_onesided) { - auto function = - onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/dft_inverse_lenght_provided_onesided.onnx")); - auto test_case = test::TestCase(function, s_device); - test_case.add_input(Shape{1, 3, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f}); - test_case.add_expected_output(Shape{4, 3}, - {0.000000f, - 1.000000f, - 2.000000f, - 3.000000f, - 4.000000f, - 5.000000f, - 6.000000f, - 7.000000f, - 8.000000f, - 9.000000f, - 10.000000f, - 11.000000f}); -} diff --git a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp index 45e4876998acc5..e231334cfc87e8 100644 --- a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp @@ -33,19 +33,27 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_bias_gelu) { "onnx/com.microsoft/bias_gelu.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.5488135, - 0.71518934, - 0.60276335, - 0.5448832, - 0.4236548, - 0.6458941, - 0.4375872, - 0.891773, - 0.96366274, - 0.3834415}); - test_case.add_input({0.79172504, 0.5288949, 0.56804454, 0.92559665, 0.07103606}); - test_case.add_expected_output( - {1.2198428, 1.1112978, 1.0293297, 1.366493, 0.3411342, 1.329408, 0.8051748, 1.354462, 1.8336612, 0.3068893}); + test_case.add_input({0.5488135f, + 0.71518934f, + 0.60276335f, + 0.5448832f, + 0.4236548f, + 0.6458941f, + 0.4375872f, + 0.891773f, + 0.96366274f, + 0.3834415f}); + test_case.add_input({0.79172504f, 0.5288949f, 0.56804454f, 0.92559665f, 0.07103606f}); + test_case.add_expected_output({1.2198428f, + 1.1112978f, + 1.0293297f, + 1.366493f, + 0.3411342f, + 1.329408f, + 0.8051748f, + 1.354462f, + 1.8336612f, + 0.3068893f}); test_case.run(); } @@ -56,19 +64,19 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_with_gamma_beta "onnx/com.microsoft/skip_layer_normalization_with_gamma_beta_bias.onnx")); std::vector input = { - 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, - 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, - 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + 0.54881352f, 0.71518934f, 0.60276335f, 0.54488319f, 0.42365479f, 0.64589411f, 0.43758720f, 0.89177299f, + 0.96366274f, 0.38344151f, 0.79172504f, 0.52889490f, 0.56804454f, 0.92559665f, 0.07103606f, 0.08712930f, + 0.02021840f, 0.83261985f, 0.77815676f, 0.87001216f, 0.97861832f, 0.79915857f, 0.46147937f, 0.78052920f, }; std::vector skip = { - 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, - 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, - 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + 0.11827443f, 0.63992101f, 0.14335328f, 0.94466889f, 0.52184832f, 0.41466194f, 0.26455560f, 0.77423370f, + 0.45615032f, 0.56843394f, 0.01878980f, 0.61763549f, 0.61209571f, 0.61693400f, 0.94374806f, 0.68182027f, + 0.35950789f, 0.43703195f, 0.69763118f, 0.06022547f, 0.66676670f, 0.67063785f, 0.21038257f, 0.12892629f, }; std::vector expected = { - -0.19721794, -0.42944565, 0.18620640, 0.61282152, -0.11097327, -0.59518522, 0.13393641, 0.66901535, - 0.04256713, -0.71902490, 0.23107991, 0.17300847, -0.04390603, -0.31109563, 0.51021838, -0.66914201, - -0.20009395, -0.43313017, 0.67281967, -0.01712347, 0.09767530, -0.43024653, -0.01836969, -0.29238200, + -0.19721794f, -0.42944565f, 0.18620640f, 0.61282152f, -0.11097327f, -0.59518522f, 0.13393641f, 0.66901535f, + 0.04256713f, -0.71902490f, 0.23107991f, 0.17300847f, -0.04390603f, -0.31109563f, 0.51021838f, -0.66914201f, + -0.20009395f, -0.43313017f, 0.67281967f, -0.01712347f, 0.09767530f, -0.43024653f, -0.01836969f, -0.29238200f, }; auto test_case = test::TestCase(function, s_device); test_case.add_input(input); @@ -84,19 +92,19 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_with_gamma_beta "onnx/com.microsoft/skip_layer_normalization_with_gamma_beta.onnx")); std::vector input = { - 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, - 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, - 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + 0.54881352f, 0.71518934f, 0.60276335f, 0.54488319f, 0.42365479f, 0.64589411f, 0.43758720f, 0.89177299f, + 0.96366274f, 0.38344151f, 0.79172504f, 0.52889490f, 0.56804454f, 0.92559665f, 0.07103606f, 0.08712930f, + 0.02021840f, 0.83261985f, 0.77815676f, 0.87001216f, 0.97861832f, 0.79915857f, 0.46147937f, 0.78052920f, }; std::vector skip = { - 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, - 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, - 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + 0.11827443f, 0.63992101f, 0.14335328f, 0.94466889f, 0.52184832f, 0.41466194f, 0.26455560f, 0.77423370f, + 0.45615032f, 0.56843394f, 0.01878980f, 0.61763549f, 0.61209571f, 0.61693400f, 0.94374806f, 0.68182027f, + 0.35950789f, 0.43703195f, 0.69763118f, 0.06022547f, 0.66676670f, 0.67063785f, 0.21038257f, 0.12892629f, }; std::vector expected = { - -0.17974678, -0.23946194, -0.04376268, 0.46959469, -0.11171167, -0.41859278, -0.11082965, 0.64513868, - 0.07773457, -0.51403606, -0.13661698, 0.11262375, -0.05096011, -0.10416907, 0.10070466, -0.50876135, - -0.22290939, -0.27663514, 0.55416691, -0.08064821, 0.04857478, -0.25121087, -0.15912610, -0.26637587, + -0.17974678f, -0.23946194f, -0.04376268f, 0.46959469f, -0.11171167f, -0.41859278f, -0.11082965f, 0.64513868f, + 0.07773457f, -0.51403606f, -0.13661698f, 0.11262375f, -0.05096011f, -0.10416907f, 0.10070466f, -0.50876135f, + -0.22290939f, -0.27663514f, 0.55416691f, -0.08064821f, 0.04857478f, -0.25121087f, -0.15912610f, -0.26637587f, }; auto test_case = test::TestCase(function, s_device); test_case.add_input(input); @@ -112,19 +120,19 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_with_gamma) { "onnx/com.microsoft/skip_layer_normalization_with_gamma.onnx")); std::vector input = { - 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, - 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, - 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + 0.54881352f, 0.71518934f, 0.60276335f, 0.54488319f, 0.42365479f, 0.64589411f, 0.43758720f, 0.89177299f, + 0.96366274f, 0.38344151f, 0.79172504f, 0.52889490f, 0.56804454f, 0.92559665f, 0.07103606f, 0.08712930f, + 0.02021840f, 0.83261985f, 0.77815676f, 0.87001216f, 0.97861832f, 0.79915857f, 0.46147937f, 0.78052920f, }; std::vector skip = { - 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, - 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, - 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + 0.11827443f, 0.63992101f, 0.14335328f, 0.94466889f, 0.52184832f, 0.41466194f, 0.26455560f, 0.77423370f, + 0.45615032f, 0.56843394f, 0.01878980f, 0.61763549f, 0.61209571f, 0.61693400f, 0.94374806f, 0.68182027f, + 0.35950789f, 0.43703195f, 0.69763118f, 0.06022547f, 0.66676670f, 0.67063785f, 0.21038257f, 0.12892629f, }; std::vector expected = { - -0.10974677, 0.16053806, -0.26376268, 0.46959469, -0.04171166, -0.01859277, -0.33082965, 0.64513868, - 0.14773457, -0.11403608, -0.35661697, 0.11262375, 0.01903989, 0.29583094, -0.11929534, -0.50876135, - -0.15290938, 0.12336487, 0.33416691, -0.08064821, 0.11857478, 0.14878914, -0.37912610, -0.26637587, + -0.10974677f, 0.16053806f, -0.26376268f, 0.46959469f, -0.04171166f, -0.01859277f, -0.33082965f, 0.64513868f, + 0.14773457f, -0.11403608f, -0.35661697f, 0.11262375f, 0.01903989f, 0.29583094f, -0.11929534f, -0.50876135f, + -0.15290938f, 0.12336487f, 0.33416691f, -0.08064821f, 0.11857478f, 0.14878914f, -0.37912610f, -0.26637587f, }; auto test_case = test::TestCase(function, s_device); test_case.add_input(input); @@ -140,37 +148,37 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_dynamic_shapes) "onnx/com.microsoft/skip_layer_normalization_dynamic_shapes.onnx")); std::vector input = { - 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, - 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, - 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + 0.54881352f, 0.71518934f, 0.60276335f, 0.54488319f, 0.42365479f, 0.64589411f, 0.43758720f, 0.89177299f, + 0.96366274f, 0.38344151f, 0.79172504f, 0.52889490f, 0.56804454f, 0.92559665f, 0.07103606f, 0.08712930f, + 0.02021840f, 0.83261985f, 0.77815676f, 0.87001216f, 0.97861832f, 0.79915857f, 0.46147937f, 0.78052920f, }; std::vector skip = { - 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, - 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, - 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + 0.11827443f, 0.63992101f, 0.14335328f, 0.94466889f, 0.52184832f, 0.41466194f, 0.26455560f, 0.77423370f, + 0.45615032f, 0.56843394f, 0.01878980f, 0.61763549f, 0.61209571f, 0.61693400f, 0.94374806f, 0.68182027f, + 0.35950789f, 0.43703195f, 0.69763118f, 0.06022547f, 0.66676670f, 0.67063785f, 0.21038257f, 0.12892629f, }; std::vector gamma = { - 0.31542835, - 0.36371076, - 0.57019675, - 0.43860152, + 0.31542835f, + 0.36371076f, + 0.57019675f, + 0.43860152f, }; std::vector beta = { - 0.98837382, - 0.10204481, - 0.20887676, - 0.16130951, + 0.98837382f, + 0.10204481f, + 0.20887676f, + 0.16130951f, }; std::vector bias = { - 0.65310830, - 0.25329161, - 0.46631077, - 0.24442559, + 0.65310830f, + 0.25329161f, + 0.46631077f, + 0.24442559f, }; std::vector expected = { - 0.76600611, 0.34308332, -0.48470584, 0.71335256, 1.10028172, -0.13354334, -0.45232186, 0.79840088, - 1.52454257, -0.19450217, -0.13759643, 0.03988872, 1.27861762, 0.39529073, 0.12247884, -0.52944231, - 0.64228040, 0.21059875, 1.05966032, -0.14278713, 1.46366918, 0.21215858, -0.31640187, -0.22832340, + 0.76600611f, 0.34308332f, -0.48470584f, 0.71335256f, 1.10028172f, -0.13354334f, -0.45232186f, 0.79840088f, + 1.52454257f, -0.19450217f, -0.13759643f, 0.03988872f, 1.27861762f, 0.39529073f, 0.12247884f, -0.52944231f, + 0.64228040f, 0.21059875f, 1.05966032f, -0.14278713f, 1.46366918f, 0.21215858f, -0.31640187f, -0.22832340f, }; auto test_case = test::TestCase(function, s_device); @@ -193,21 +201,21 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization) { 8, 1, 5, 9, 8, 9, 4, 3, 0, 3, 5, 0, 2, 3, 8, 1, 3, 3, 3, 7, 0, 1, 9, 9, }; std::vector expected_output = { - -0.06615843, -0.18040463, 0.02199928, 0.01868065, 0.05397778, -0.11761580, -0.09138932, -0.02506775, - -0.02368510, -0.10373901, -0.05551499, -0.20972314, 0.01365213, 0.01132561, -0.08603337, -0.08906764, - 0.09692993, -0.04444099, -0.02037602, -0.03453060, -0.10214549, -0.13331436, -0.02665862, -0.01228805, - -0.14232540, -0.07032782, 0.05511986, -0.00120272, -0.04875736, -0.13051267, -0.05709254, 0.17854357, - -0.01759873, -0.01819968, 0.07573269, 0.00557164, 0.06232717, 0.00530490, -0.01565807, -0.14841977, - -0.02299280, 0.02038561, -0.00049481, 0.02575402, 0.10081697, -0.12517214, -0.09316762, -0.00974943, - -0.03093284, -0.06309240, -0.05551499, -0.20972314, 0.01365213, 0.01132561, -0.08603337, -0.06176658, - 0.08304203, -0.05025182, 0.00383657, -0.02288112, -0.11407227, -0.01386134, -0.04411830, -0.00537948, - 0.00164397, -0.03739140, 0.09941526, 0.00333974, -0.04251949, -0.12992151, -0.09509478, -0.11811313, - -0.03307065, -0.00866115, -0.15162414, 0.01106802, 0.06037656, 0.00035292, -0.00223284, -0.11215645, - -0.01390734, 0.07064321, 0.04028325, -0.00290875, 0.12875907, -0.12517214, -0.09316762, -0.00974943, - -0.03093284, -0.06309240, -0.08723789, 0.03130914, 0.03131931, -0.01526242, 0.20811458, -0.05696163, - 0.16304255, -0.02407495, -0.02955675, -0.03086288, -0.08130091, -0.05001551, -0.04875683, 0.00143666, - -0.12153473, -0.00018507, 0.10957482, -0.00416618, -0.01612359, -0.11605026, -0.08593204, 0.09055272, - -0.03054028, -0.03603891, -0.08479506, -0.00034568, 0.03713699, 0.00163411, -0.01738501, -0.18267182, + -0.06615843f, -0.18040463f, 0.02199928f, 0.01868065f, 0.05397778f, -0.11761580f, -0.09138932f, -0.02506775f, + -0.02368510f, -0.10373901f, -0.05551499f, -0.20972314f, 0.01365213f, 0.01132561f, -0.08603337f, -0.08906764f, + 0.09692993f, -0.04444099f, -0.02037602f, -0.03453060f, -0.10214549f, -0.13331436f, -0.02665862f, -0.01228805f, + -0.14232540f, -0.07032782f, 0.05511986f, -0.00120272f, -0.04875736f, -0.13051267f, -0.05709254f, 0.17854357f, + -0.01759873f, -0.01819968f, 0.07573269f, 0.00557164f, 0.06232717f, 0.00530490f, -0.01565807f, -0.14841977f, + -0.02299280f, 0.02038561f, -0.00049481f, 0.02575402f, 0.10081697f, -0.12517214f, -0.09316762f, -0.00974943f, + -0.03093284f, -0.06309240f, -0.05551499f, -0.20972314f, 0.01365213f, 0.01132561f, -0.08603337f, -0.06176658f, + 0.08304203f, -0.05025182f, 0.00383657f, -0.02288112f, -0.11407227f, -0.01386134f, -0.04411830f, -0.00537948f, + 0.00164397f, -0.03739140f, 0.09941526f, 0.00333974f, -0.04251949f, -0.12992151f, -0.09509478f, -0.11811313f, + -0.03307065f, -0.00866115f, -0.15162414f, 0.01106802f, 0.06037656f, 0.00035292f, -0.00223284f, -0.11215645f, + -0.01390734f, 0.07064321f, 0.04028325f, -0.00290875f, 0.12875907f, -0.12517214f, -0.09316762f, -0.00974943f, + -0.03093284f, -0.06309240f, -0.08723789f, 0.03130914f, 0.03131931f, -0.01526242f, 0.20811458f, -0.05696163f, + 0.16304255f, -0.02407495f, -0.02955675f, -0.03086288f, -0.08130091f, -0.05001551f, -0.04875683f, 0.00143666f, + -0.12153473f, -0.00018507f, 0.10957482f, -0.00416618f, -0.01612359f, -0.11605026f, -0.08593204f, 0.09055272f, + -0.03054028f, -0.03603891f, -0.08479506f, -0.00034568f, 0.03713699f, 0.00163411f, -0.01738501f, -0.18267182f, }; auto test_case = test::TestCase(function, s_device); @@ -229,21 +237,21 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_segment_e 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 2, 0, 1, 1, 1, }; std::vector expected_output = { - -0.06044213, -0.14845914, 0.02457689, 0.02091519, 0.09514004, -0.10280035, -0.02087995, -0.03323204, - -0.02967127, -0.13447416, -0.05191760, -0.16518904, 0.02340531, 0.02176395, 0.04972410, -0.07360736, - 0.12192874, -0.04081530, -0.02338044, -0.05671440, -0.09475864, -0.08944942, -0.03362993, -0.01683486, - -0.16770349, -0.07382569, 0.06230322, 0.02215859, -0.05212611, -0.03934773, -0.04748865, 0.18134241, - -0.01965741, -0.02202452, 0.01973994, 0.01575558, 0.04300199, 0.01436110, -0.00198062, -0.09065692, - -0.02923042, -0.00748686, 0.00717049, 0.02638642, 0.12174864, -0.12973398, -0.11872391, -0.00549398, - -0.02386289, -0.02210563, -0.03590920, -0.13728066, -0.01337939, 0.01538021, -0.14687485, -0.05033565, - 0.03818212, -0.04939338, 0.00961064, -0.07407621, -0.09624685, 0.05594898, -0.04948713, -0.01305631, - -0.03779668, -0.01469170, 0.12346989, 0.02082030, -0.03449103, -0.06029151, -0.09300473, -0.16308543, - -0.02370042, 0.01066893, -0.06523034, 0.00497636, 0.01933458, -0.00900802, 0.00430878, -0.13999483, - -0.02377289, 0.01760014, 0.03896973, 0.00831112, 0.15634246, -0.11109130, -0.11997811, -0.02304414, - -0.01989413, -0.12763791, -0.05698400, 0.17125534, 0.00499324, -0.02953288, 0.09178342, -0.05001877, - 0.16157132, -0.02312993, -0.02932195, -0.04914058, -0.07994118, -0.07199102, -0.04517454, 0.01249476, - -0.07525793, -0.00207180, 0.03993115, -0.01676321, -0.00214832, -0.16074482, -0.05012497, -0.00552153, - -0.04302063, -0.00549224, -0.18399858, -0.00767871, -0.02209404, -0.01383207, -0.00082931, -0.19533031, + -0.06044213f, -0.14845914f, 0.02457689f, 0.02091519f, 0.09514004f, -0.10280035f, -0.02087995f, -0.03323204f, + -0.02967127f, -0.13447416f, -0.05191760f, -0.16518904f, 0.02340531f, 0.02176395f, 0.04972410f, -0.07360736f, + 0.12192874f, -0.04081530f, -0.02338044f, -0.05671440f, -0.09475864f, -0.08944942f, -0.03362993f, -0.01683486f, + -0.16770349f, -0.07382569f, 0.06230322f, 0.02215859f, -0.05212611f, -0.03934773f, -0.04748865f, 0.18134241f, + -0.01965741f, -0.02202452f, 0.01973994f, 0.01575558f, 0.04300199f, 0.01436110f, -0.00198062f, -0.09065692f, + -0.02923042f, -0.00748686f, 0.00717049f, 0.02638642f, 0.12174864f, -0.12973398f, -0.11872391f, -0.00549398f, + -0.02386289f, -0.02210563f, -0.03590920f, -0.13728066f, -0.01337939f, 0.01538021f, -0.14687485f, -0.05033565f, + 0.03818212f, -0.04939338f, 0.00961064f, -0.07407621f, -0.09624685f, 0.05594898f, -0.04948713f, -0.01305631f, + -0.03779668f, -0.01469170f, 0.12346989f, 0.02082030f, -0.03449103f, -0.06029151f, -0.09300473f, -0.16308543f, + -0.02370042f, 0.01066893f, -0.06523034f, 0.00497636f, 0.01933458f, -0.00900802f, 0.00430878f, -0.13999483f, + -0.02377289f, 0.01760014f, 0.03896973f, 0.00831112f, 0.15634246f, -0.11109130f, -0.11997811f, -0.02304414f, + -0.01989413f, -0.12763791f, -0.05698400f, 0.17125534f, 0.00499324f, -0.02953288f, 0.09178342f, -0.05001877f, + 0.16157132f, -0.02312993f, -0.02932195f, -0.04914058f, -0.07994118f, -0.07199102f, -0.04517454f, 0.01249476f, + -0.07525793f, -0.00207180f, 0.03993115f, -0.01676321f, -0.00214832f, -0.16074482f, -0.05012497f, -0.00552153f, + -0.04302063f, -0.00549224f, -0.18399858f, -0.00767871f, -0.02209404f, -0.01383207f, -0.00082931f, -0.19533031f, }; std::vector expected_mask_index = { @@ -257,7 +265,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_segment_e test_case.add_input(segment_ids); test_case.add_expected_output(expected_output); test_case.add_expected_output(expected_mask_index); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_segment_embedding_and_mask) { @@ -276,21 +284,21 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_segment_e 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, }; std::vector expected_output = { - -0.06044213, -0.14845914, 0.02457689, 0.02091519, 0.09514004, -0.10280035, -0.02087995, -0.03323204, - -0.02967127, -0.13447416, -0.05191760, -0.16518904, 0.02340531, 0.02176395, 0.04972410, -0.07360736, - 0.12192874, -0.04081530, -0.02338044, -0.05671440, -0.09475864, -0.08944942, -0.03362993, -0.01683486, - -0.16770349, -0.07382569, 0.06230322, 0.02215859, -0.05212611, -0.03934773, -0.04748865, 0.18134241, - -0.01965741, -0.02202452, 0.01973994, 0.01575558, 0.04300199, 0.01436110, -0.00198062, -0.09065692, - -0.02923042, -0.00748686, 0.00717049, 0.02638642, 0.12174864, -0.12973398, -0.11872391, -0.00549398, - -0.02386289, -0.02210563, -0.03590920, -0.13728066, -0.01337939, 0.01538021, -0.14687485, -0.05033565, - 0.03818212, -0.04939338, 0.00961064, -0.07407621, -0.09624685, 0.05594898, -0.04948713, -0.01305631, - -0.03779668, -0.01469170, 0.12346989, 0.02082030, -0.03449103, -0.06029151, -0.09300473, -0.16308543, - -0.02370042, 0.01066893, -0.06523034, 0.00497636, 0.01933458, -0.00900802, 0.00430878, -0.13999483, - -0.02377289, 0.01760014, 0.03896973, 0.00831112, 0.15634246, -0.11109130, -0.11997811, -0.02304414, - -0.01989413, -0.12763791, -0.05698400, 0.17125534, 0.00499324, -0.02953288, 0.09178342, -0.05001877, - 0.16157132, -0.02312993, -0.02932195, -0.04914058, -0.07994118, -0.07199102, -0.04517454, 0.01249476, - -0.07525793, -0.00207180, 0.03993115, -0.01676321, -0.00214832, -0.16074482, -0.05012497, -0.00552153, - -0.04302063, -0.00549224, -0.18399858, -0.00767871, -0.02209404, -0.01383207, -0.00082931, -0.19533031, + -0.06044213f, -0.14845914f, 0.02457689f, 0.02091519f, 0.09514004f, -0.10280035f, -0.02087995f, -0.03323204f, + -0.02967127f, -0.13447416f, -0.05191760f, -0.16518904f, 0.02340531f, 0.02176395f, 0.04972410f, -0.07360736f, + 0.12192874f, -0.04081530f, -0.02338044f, -0.05671440f, -0.09475864f, -0.08944942f, -0.03362993f, -0.01683486f, + -0.16770349f, -0.07382569f, 0.06230322f, 0.02215859f, -0.05212611f, -0.03934773f, -0.04748865f, 0.18134241f, + -0.01965741f, -0.02202452f, 0.01973994f, 0.01575558f, 0.04300199f, 0.01436110f, -0.00198062f, -0.09065692f, + -0.02923042f, -0.00748686f, 0.00717049f, 0.02638642f, 0.12174864f, -0.12973398f, -0.11872391f, -0.00549398f, + -0.02386289f, -0.02210563f, -0.03590920f, -0.13728066f, -0.01337939f, 0.01538021f, -0.14687485f, -0.05033565f, + 0.03818212f, -0.04939338f, 0.00961064f, -0.07407621f, -0.09624685f, 0.05594898f, -0.04948713f, -0.01305631f, + -0.03779668f, -0.01469170f, 0.12346989f, 0.02082030f, -0.03449103f, -0.06029151f, -0.09300473f, -0.16308543f, + -0.02370042f, 0.01066893f, -0.06523034f, 0.00497636f, 0.01933458f, -0.00900802f, 0.00430878f, -0.13999483f, + -0.02377289f, 0.01760014f, 0.03896973f, 0.00831112f, 0.15634246f, -0.11109130f, -0.11997811f, -0.02304414f, + -0.01989413f, -0.12763791f, -0.05698400f, 0.17125534f, 0.00499324f, -0.02953288f, 0.09178342f, -0.05001877f, + 0.16157132f, -0.02312993f, -0.02932195f, -0.04914058f, -0.07994118f, -0.07199102f, -0.04517454f, 0.01249476f, + -0.07525793f, -0.00207180f, 0.03993115f, -0.01676321f, -0.00214832f, -0.16074482f, -0.05012497f, -0.00552153f, + -0.04302063f, -0.00549224f, -0.18399858f, -0.00767871f, -0.02209404f, -0.01383207f, -0.00082931f, -0.19533031f, }; std::vector expected_mask_index = { 5, @@ -304,7 +312,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_segment_e test_case.add_input(mask); test_case.add_expected_output(expected_output); test_case.add_expected_output(expected_mask_index); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_dynamic_shapes) { @@ -320,70 +328,71 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_dynamic_shapes 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 2, 0, 1, 1, 1, }; std::vector word_embeddings = { - 0.96980906, 0.65314001, 0.17090958, 0.35815218, 0.75068617, 0.60783064, 0.32504722, 0.03842543, 0.63427407, - 0.95894927, 0.65279031, 0.63505888, 0.99529958, 0.58185035, 0.41436860, 0.47469750, 0.62351012, 0.33800763, - 0.67475230, 0.31720173, 0.77834547, 0.94957107, 0.66252685, 0.01357164, 0.62284607, 0.67365962, 0.97194499, - 0.87819350, 0.50962436, 0.05571469, 0.45115921, 0.01998767, 0.44171092, 0.97958672, 0.35944447, 0.48089352, - 0.68866116, 0.88047588, 0.91823548, 0.21682213, 0.56518888, 0.86510259, 0.50896895, 0.91672295, 0.92115760, - 0.08311249, 0.27771857, 0.00935670, 0.84234208, 0.64717412, + 0.96980906f, 0.65314001f, 0.17090958f, 0.35815218f, 0.75068617f, 0.60783064f, 0.32504722f, 0.03842543f, + 0.63427407f, 0.95894927f, 0.65279031f, 0.63505888f, 0.99529958f, 0.58185035f, 0.41436860f, 0.47469750f, + 0.62351012f, 0.33800763f, 0.67475230f, 0.31720173f, 0.77834547f, 0.94957107f, 0.66252685f, 0.01357164f, + 0.62284607f, 0.67365962f, 0.97194499f, 0.87819350f, 0.50962436f, 0.05571469f, 0.45115921f, 0.01998767f, + 0.44171092f, 0.97958672f, 0.35944447f, 0.48089352f, 0.68866116f, 0.88047588f, 0.91823548f, 0.21682213f, + 0.56518888f, 0.86510259f, 0.50896895f, 0.91672295f, 0.92115760f, 0.08311249f, 0.27771857f, 0.00935670f, + 0.84234208f, 0.64717412f, }; std::vector position_embeddings = { - 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, 0.36980811, 0.14644176, 0.56961840, - 0.70373726, 0.28847644, 0.43328807, 0.75610667, 0.39609829, 0.89603841, 0.63892108, 0.89155442, - 0.68005556, 0.44919774, 0.97857094, 0.11620191, 0.76702368, 0.41182014, 0.67543906, 0.24979627, - 0.31321833, 0.96541619, 0.58846509, 0.65966839, 0.53320622, 0.23053302, 0.39486930, 0.61880857, - 0.47486752, 0.47013220, 0.71607453, 0.28799102, 0.38346222, 0.74916983, 0.87845218, 0.10286336, + 0.84138614f, 0.26473016f, 0.39782074f, 0.55282146f, 0.16494046f, 0.36980811f, 0.14644176f, 0.56961840f, + 0.70373726f, 0.28847644f, 0.43328807f, 0.75610667f, 0.39609829f, 0.89603841f, 0.63892108f, 0.89155442f, + 0.68005556f, 0.44919774f, 0.97857094f, 0.11620191f, 0.76702368f, 0.41182014f, 0.67543906f, 0.24979627f, + 0.31321833f, 0.96541619f, 0.58846509f, 0.65966839f, 0.53320622f, 0.23053302f, 0.39486930f, 0.61880857f, + 0.47486752f, 0.47013220f, 0.71607453f, 0.28799102f, 0.38346222f, 0.74916983f, 0.87845218f, 0.10286336f, }; std::vector segment_embeddings = { - 0.09237389, - 0.35404667, - 0.55181628, - 0.03362509, - 0.96896178, - 0.32099724, - 0.22126268, - 0.14126390, - 0.09725992, - 0.98404223, - 0.26034093, - 0.53702253, - 0.44792616, - 0.09956909, - 0.35231167, + 0.09237389f, + 0.35404667f, + 0.55181628f, + 0.03362509f, + 0.96896178f, + 0.32099724f, + 0.22126268f, + 0.14126390f, + 0.09725992f, + 0.98404223f, + 0.26034093f, + 0.53702253f, + 0.44792616f, + 0.09956909f, + 0.35231167f, }; std::vector gamma = { - 0.46924916, - 0.84114015, - 0.90464777, - 0.03755938, - 0.50831544, + 0.46924916f, + 0.84114015f, + 0.90464777f, + 0.03755938f, + 0.50831544f, }; std::vector beta = { - 0.16684751, - 0.77905101, - 0.86493331, - 0.41139671, - 0.13997258, + 0.16684751f, + 0.77905101f, + 0.86493331f, + 0.41139671f, + 0.13997258f, }; std::vector mask = { 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, }; std::vector expected_output = { - -0.04089922, 0.35108989, 0.30442458, 0.39546335, 1.15422225, 0.10419128, -0.19301927, 0.01070970, - 0.43977541, 0.89119899, -0.51436460, 1.99256825, 1.41077507, 0.38642293, 0.17583044, 0.03320138, - 1.16508031, -0.24356931, 0.47440714, -0.17844005, 0.20463173, 1.90038323, 1.14138567, 0.34504607, - 0.16403235, -0.24976699, 0.29362509, 0.34502214, 0.41751838, 1.09390712, 0.12354189, 1.83025289, - 1.05569196, 0.34413773, 0.35469764, -0.69760042, 0.76338542, 1.75443077, 0.44126555, 0.18181801, - 0.73277575, 0.45443264, 0.17068321, 0.36591727, 0.72869974, -0.56090516, 0.14415455, 1.47314119, - 0.42908576, 0.73084539, -0.22373237, 2.26550221, 0.05606699, 0.39417523, 0.35234636, 0.78569502, - 0.77521765, -0.65131050, 0.40168875, 0.45527256, 0.38715565, 0.98521245, 2.21446753, 0.36345237, - -0.33269632, 0.36558092, 1.36846578, 1.37523413, 0.33698002, 0.28889543, -0.40639281, 1.01643157, - 0.59668219, 0.39197800, 1.03101778, 0.02551098, -0.03612846, -0.01371557, 0.43444607, 0.96746695, - 0.60583955, -0.10362893, 0.40574494, 0.38046724, 0.87445319, -0.00880148, -0.15437943, 0.08118075, - 0.44650543, 0.85956848, -0.27865338, 2.10837507, 0.04798460, 0.43948367, -0.10185169, 0.19978794, - 1.32323360, 1.20525467, 0.44288942, -0.84200430, 0.52563053, 0.69949460, 0.73987913, 0.34668452, - 0.74545687, 0.57696682, 0.22452033, -0.27099937, 0.39649010, 0.87083614, -0.18965788, 0.58206403, - -0.08108193, 0.42067638, 1.05117214, -0.34287399, 0.20424896, 0.27994895, 0.46011117, 0.70890665, + -0.04089922f, 0.35108989f, 0.30442458f, 0.39546335f, 1.15422225f, 0.10419128f, -0.19301927f, 0.01070970f, + 0.43977541f, 0.89119899f, -0.51436460f, 1.99256825f, 1.41077507f, 0.38642293f, 0.17583044f, 0.03320138f, + 1.16508031f, -0.24356931f, 0.47440714f, -0.17844005f, 0.20463173f, 1.90038323f, 1.14138567f, 0.34504607f, + 0.16403235f, -0.24976699f, 0.29362509f, 0.34502214f, 0.41751838f, 1.09390712f, 0.12354189f, 1.83025289f, + 1.05569196f, 0.34413773f, 0.35469764f, -0.69760042f, 0.76338542f, 1.75443077f, 0.44126555f, 0.18181801f, + 0.73277575f, 0.45443264f, 0.17068321f, 0.36591727f, 0.72869974f, -0.56090516f, 0.14415455f, 1.47314119f, + 0.42908576f, 0.73084539f, -0.22373237f, 2.26550221f, 0.05606699f, 0.39417523f, 0.35234636f, 0.78569502f, + 0.77521765f, -0.65131050f, 0.40168875f, 0.45527256f, 0.38715565f, 0.98521245f, 2.21446753f, 0.36345237f, + -0.33269632f, 0.36558092f, 1.36846578f, 1.37523413f, 0.33698002f, 0.28889543f, -0.40639281f, 1.01643157f, + 0.59668219f, 0.39197800f, 1.03101778f, 0.02551098f, -0.03612846f, -0.01371557f, 0.43444607f, 0.96746695f, + 0.60583955f, -0.10362893f, 0.40574494f, 0.38046724f, 0.87445319f, -0.00880148f, -0.15437943f, 0.08118075f, + 0.44650543f, 0.85956848f, -0.27865338f, 2.10837507f, 0.04798460f, 0.43948367f, -0.10185169f, 0.19978794f, + 1.32323360f, 1.20525467f, 0.44288942f, -0.84200430f, 0.52563053f, 0.69949460f, 0.73987913f, 0.34668452f, + 0.74545687f, 0.57696682f, 0.22452033f, -0.27099937f, 0.39649010f, 0.87083614f, -0.18965788f, 0.58206403f, + -0.08108193f, 0.42067638f, 1.05117214f, -0.34287399f, 0.20424896f, 0.27994895f, 0.46011117f, 0.70890665f, }; std::vector expected_mask_index = { 6, @@ -402,7 +411,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_dynamic_shapes test_case.add_input(Shape{3, 8}, mask); test_case.add_expected_output(Shape{3, 8, 5}, expected_output); test_case.add_expected_output(Shape{3}, expected_mask_index); - test_case.run_with_tolerance_as_fp(1e-6); + test_case.run_with_tolerance_as_fp(1e-6f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_diff_seq_len_pos_embed_len) { @@ -418,70 +427,72 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_diff_seq_len_p 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 2, 0, 1, 1, 1, }; std::vector word_embeddings = { - 0.96980906, 0.65314001, 0.17090958, 0.35815218, 0.75068617, 0.60783064, 0.32504722, 0.03842543, 0.63427407, - 0.95894927, 0.65279031, 0.63505888, 0.99529958, 0.58185035, 0.41436860, 0.47469750, 0.62351012, 0.33800763, - 0.67475230, 0.31720173, 0.77834547, 0.94957107, 0.66252685, 0.01357164, 0.62284607, 0.67365962, 0.97194499, - 0.87819350, 0.50962436, 0.05571469, 0.45115921, 0.01998767, 0.44171092, 0.97958672, 0.35944447, 0.48089352, - 0.68866116, 0.88047588, 0.91823548, 0.21682213, 0.56518888, 0.86510259, 0.50896895, 0.91672295, 0.92115760, - 0.08311249, 0.27771857, 0.00935670, 0.84234208, 0.64717412, + 0.96980906f, 0.65314001f, 0.17090958f, 0.35815218f, 0.75068617f, 0.60783064f, 0.32504722f, 0.03842543f, + 0.63427407f, 0.95894927f, 0.65279031f, 0.63505888f, 0.99529958f, 0.58185035f, 0.41436860f, 0.47469750f, + 0.62351012f, 0.33800763f, 0.67475230f, 0.31720173f, 0.77834547f, 0.94957107f, 0.66252685f, 0.01357164f, + 0.62284607f, 0.67365962f, 0.97194499f, 0.87819350f, 0.50962436f, 0.05571469f, 0.45115921f, 0.01998767f, + 0.44171092f, 0.97958672f, 0.35944447f, 0.48089352f, 0.68866116f, 0.88047588f, 0.91823548f, 0.21682213f, + 0.56518888f, 0.86510259f, 0.50896895f, 0.91672295f, 0.92115760f, 0.08311249f, 0.27771857f, 0.00935670f, + 0.84234208f, 0.64717412f, }; std::vector position_embeddings = { - 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, 0.36980811, 0.14644176, 0.56961840, 0.70373726, - 0.28847644, 0.43328807, 0.75610667, 0.39609829, 0.89603841, 0.63892108, 0.89155442, 0.68005556, 0.44919774, - 0.97857094, 0.11620191, 0.76702368, 0.41182014, 0.67543906, 0.24979627, 0.31321833, 0.96541619, 0.58846509, - 0.65966839, 0.53320622, 0.23053302, 0.39486930, 0.61880857, 0.47486752, 0.47013220, 0.71607453, 0.28799102, - 0.38346222, 0.74916983, 0.87845218, 0.10286336, 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, + 0.84138614f, 0.26473016f, 0.39782074f, 0.55282146f, 0.16494046f, 0.36980811f, 0.14644176f, 0.56961840f, + 0.70373726f, 0.28847644f, 0.43328807f, 0.75610667f, 0.39609829f, 0.89603841f, 0.63892108f, 0.89155442f, + 0.68005556f, 0.44919774f, 0.97857094f, 0.11620191f, 0.76702368f, 0.41182014f, 0.67543906f, 0.24979627f, + 0.31321833f, 0.96541619f, 0.58846509f, 0.65966839f, 0.53320622f, 0.23053302f, 0.39486930f, 0.61880857f, + 0.47486752f, 0.47013220f, 0.71607453f, 0.28799102f, 0.38346222f, 0.74916983f, 0.87845218f, 0.10286336f, + 0.84138614f, 0.26473016f, 0.39782074f, 0.55282146f, 0.16494046f, }; std::vector segment_embeddings = { - 0.09237389, - 0.35404667, - 0.55181628, - 0.03362509, - 0.96896178, - 0.32099724, - 0.22126268, - 0.14126390, - 0.09725992, - 0.98404223, - 0.26034093, - 0.53702253, - 0.44792616, - 0.09956909, - 0.35231167, + 0.09237389f, + 0.35404667f, + 0.55181628f, + 0.03362509f, + 0.96896178f, + 0.32099724f, + 0.22126268f, + 0.14126390f, + 0.09725992f, + 0.98404223f, + 0.26034093f, + 0.53702253f, + 0.44792616f, + 0.09956909f, + 0.35231167f, }; std::vector gamma = { - 0.46924916, - 0.84114015, - 0.90464777, - 0.03755938, - 0.50831544, + 0.46924916f, + 0.84114015f, + 0.90464777f, + 0.03755938f, + 0.50831544f, }; std::vector beta = { - 0.16684751, - 0.77905101, - 0.86493331, - 0.41139671, - 0.13997258, + 0.16684751f, + 0.77905101f, + 0.86493331f, + 0.41139671f, + 0.13997258f, }; std::vector mask = { 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, }; std::vector expected_output = { - -0.04089922, 0.35108989, 0.30442458, 0.39546335, 1.15422225, 0.10419128, -0.19301927, 0.01070970, - 0.43977541, 0.89119899, -0.51436460, 1.99256825, 1.41077507, 0.38642293, 0.17583044, 0.03320138, - 1.16508031, -0.24356931, 0.47440714, -0.17844005, 0.20463173, 1.90038323, 1.14138567, 0.34504607, - 0.16403235, -0.24976699, 0.29362509, 0.34502214, 0.41751838, 1.09390712, 0.12354189, 1.83025289, - 1.05569196, 0.34413773, 0.35469764, -0.69760042, 0.76338542, 1.75443077, 0.44126555, 0.18181801, - 0.73277575, 0.45443264, 0.17068321, 0.36591727, 0.72869974, -0.56090516, 0.14415455, 1.47314119, - 0.42908576, 0.73084539, -0.22373237, 2.26550221, 0.05606699, 0.39417523, 0.35234636, 0.78569502, - 0.77521765, -0.65131050, 0.40168875, 0.45527256, 0.38715565, 0.98521245, 2.21446753, 0.36345237, - -0.33269632, 0.36558092, 1.36846578, 1.37523413, 0.33698002, 0.28889543, -0.40639281, 1.01643157, - 0.59668219, 0.39197800, 1.03101778, 0.02551098, -0.03612846, -0.01371557, 0.43444607, 0.96746695, - 0.60583955, -0.10362893, 0.40574494, 0.38046724, 0.87445319, -0.00880148, -0.15437943, 0.08118075, - 0.44650543, 0.85956848, -0.27865338, 2.10837507, 0.04798460, 0.43948367, -0.10185169, 0.19978794, - 1.32323360, 1.20525467, 0.44288942, -0.84200430, 0.52563053, 0.69949460, 0.73987913, 0.34668452, - 0.74545687, 0.57696682, 0.22452033, -0.27099937, 0.39649010, 0.87083614, -0.18965788, 0.58206403, - -0.08108193, 0.42067638, 1.05117214, -0.34287399, 0.20424896, 0.27994895, 0.46011117, 0.70890665, + -0.04089922f, 0.35108989f, 0.30442458f, 0.39546335f, 1.15422225f, 0.10419128f, -0.19301927f, 0.01070970f, + 0.43977541f, 0.89119899f, -0.51436460f, 1.99256825f, 1.41077507f, 0.38642293f, 0.17583044f, 0.03320138f, + 1.16508031f, -0.24356931f, 0.47440714f, -0.17844005f, 0.20463173f, 1.90038323f, 1.14138567f, 0.34504607f, + 0.16403235f, -0.24976699f, 0.29362509f, 0.34502214f, 0.41751838f, 1.09390712f, 0.12354189f, 1.83025289f, + 1.05569196f, 0.34413773f, 0.35469764f, -0.69760042f, 0.76338542f, 1.75443077f, 0.44126555f, 0.18181801f, + 0.73277575f, 0.45443264f, 0.17068321f, 0.36591727f, 0.72869974f, -0.56090516f, 0.14415455f, 1.47314119f, + 0.42908576f, 0.73084539f, -0.22373237f, 2.26550221f, 0.05606699f, 0.39417523f, 0.35234636f, 0.78569502f, + 0.77521765f, -0.65131050f, 0.40168875f, 0.45527256f, 0.38715565f, 0.98521245f, 2.21446753f, 0.36345237f, + -0.33269632f, 0.36558092f, 1.36846578f, 1.37523413f, 0.33698002f, 0.28889543f, -0.40639281f, 1.01643157f, + 0.59668219f, 0.39197800f, 1.03101778f, 0.02551098f, -0.03612846f, -0.01371557f, 0.43444607f, 0.96746695f, + 0.60583955f, -0.10362893f, 0.40574494f, 0.38046724f, 0.87445319f, -0.00880148f, -0.15437943f, 0.08118075f, + 0.44650543f, 0.85956848f, -0.27865338f, 2.10837507f, 0.04798460f, 0.43948367f, -0.10185169f, 0.19978794f, + 1.32323360f, 1.20525467f, 0.44288942f, -0.84200430f, 0.52563053f, 0.69949460f, 0.73987913f, 0.34668452f, + 0.74545687f, 0.57696682f, 0.22452033f, -0.27099937f, 0.39649010f, 0.87083614f, -0.18965788f, 0.58206403f, + -0.08108193f, 0.42067638f, 1.05117214f, -0.34287399f, 0.20424896f, 0.27994895f, 0.46011117f, 0.70890665f, }; std::vector expected_mask_index = { 6, @@ -500,7 +511,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_diff_seq_len_p test_case.add_input(Shape{3, 8}, mask); test_case.add_expected_output(Shape{3, 8, 5}, expected_output); test_case.add_expected_output(Shape{3}, expected_mask_index); - test_case.run_with_tolerance_as_fp(1e-6); + test_case.run_with_tolerance_as_fp(1e-6f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_position_ids) { @@ -516,50 +527,52 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_position_ 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 2, 0, 1, 1, 1, }; std::vector word_embeddings = { - 0.96980906, 0.65314001, 0.17090958, 0.35815218, 0.75068617, 0.60783064, 0.32504722, 0.03842543, 0.63427407, - 0.95894927, 0.65279031, 0.63505888, 0.99529958, 0.58185035, 0.41436860, 0.47469750, 0.62351012, 0.33800763, - 0.67475230, 0.31720173, 0.77834547, 0.94957107, 0.66252685, 0.01357164, 0.62284607, 0.67365962, 0.97194499, - 0.87819350, 0.50962436, 0.05571469, 0.45115921, 0.01998767, 0.44171092, 0.97958672, 0.35944447, 0.48089352, - 0.68866116, 0.88047588, 0.91823548, 0.21682213, 0.56518888, 0.86510259, 0.50896895, 0.91672295, 0.92115760, - 0.08311249, 0.27771857, 0.00935670, 0.84234208, 0.64717412, + 0.96980906f, 0.65314001f, 0.17090958f, 0.35815218f, 0.75068617f, 0.60783064f, 0.32504722f, 0.03842543f, + 0.63427407f, 0.95894927f, 0.65279031f, 0.63505888f, 0.99529958f, 0.58185035f, 0.41436860f, 0.47469750f, + 0.62351012f, 0.33800763f, 0.67475230f, 0.31720173f, 0.77834547f, 0.94957107f, 0.66252685f, 0.01357164f, + 0.62284607f, 0.67365962f, 0.97194499f, 0.87819350f, 0.50962436f, 0.05571469f, 0.45115921f, 0.01998767f, + 0.44171092f, 0.97958672f, 0.35944447f, 0.48089352f, 0.68866116f, 0.88047588f, 0.91823548f, 0.21682213f, + 0.56518888f, 0.86510259f, 0.50896895f, 0.91672295f, 0.92115760f, 0.08311249f, 0.27771857f, 0.00935670f, + 0.84234208f, 0.64717412f, }; std::vector position_embeddings = { - 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, 0.43328807, 0.75610667, 0.39609829, 0.89603841, - 0.63892108, 0.36980811, 0.14644176, 0.56961840, 0.70373726, 0.28847644, 0.89155442, 0.68005556, 0.44919774, - 0.97857094, 0.11620191, 0.76702368, 0.41182014, 0.67543906, 0.24979627, 0.31321833, 0.28799102, 0.38346222, - 0.74916983, 0.87845218, 0.10286336, 0.96541619, 0.58846509, 0.65966839, 0.53320622, 0.23053302, 0.39486930, - 0.61880857, 0.47486752, 0.47013220, 0.71607453, 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, + 0.84138614f, 0.26473016f, 0.39782074f, 0.55282146f, 0.16494046f, 0.43328807f, 0.75610667f, 0.39609829f, + 0.89603841f, 0.63892108f, 0.36980811f, 0.14644176f, 0.56961840f, 0.70373726f, 0.28847644f, 0.89155442f, + 0.68005556f, 0.44919774f, 0.97857094f, 0.11620191f, 0.76702368f, 0.41182014f, 0.67543906f, 0.24979627f, + 0.31321833f, 0.28799102f, 0.38346222f, 0.74916983f, 0.87845218f, 0.10286336f, 0.96541619f, 0.58846509f, + 0.65966839f, 0.53320622f, 0.23053302f, 0.39486930f, 0.61880857f, 0.47486752f, 0.47013220f, 0.71607453f, + 0.84138614f, 0.26473016f, 0.39782074f, 0.55282146f, 0.16494046f, }; std::vector segment_embeddings = { - 0.09237389, - 0.35404667, - 0.55181628, - 0.03362509, - 0.96896178, - 0.32099724, - 0.22126268, - 0.14126390, - 0.09725992, - 0.98404223, - 0.26034093, - 0.53702253, - 0.44792616, - 0.09956909, - 0.35231167, + 0.09237389f, + 0.35404667f, + 0.55181628f, + 0.03362509f, + 0.96896178f, + 0.32099724f, + 0.22126268f, + 0.14126390f, + 0.09725992f, + 0.98404223f, + 0.26034093f, + 0.53702253f, + 0.44792616f, + 0.09956909f, + 0.35231167f, }; std::vector gamma = { - 0.46924916, - 0.84114015, - 0.90464777, - 0.03755938, - 0.50831544, + 0.46924916f, + 0.84114015f, + 0.90464777f, + 0.03755938f, + 0.50831544f, }; std::vector beta = { - 0.16684751, - 0.77905101, - 0.86493331, - 0.41139671, - 0.13997258, + 0.16684751f, + 0.77905101f, + 0.86493331f, + 0.41139671f, + 0.13997258f, }; std::vector mask = { 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, @@ -568,21 +581,21 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_position_ 0, 2, 1, 3, 4, 6, 7, 5, 8, 2, 1, 3, 4, 6, 7, 5, 0, 2, 1, 3, 4, 6, 7, 5, }; std::vector expected_output = { - -0.04089922, 0.35108989, 0.30442458, 0.39546335, 1.15422225, 0.10419128, -0.19301927, 0.01070970, - 0.43977541, 0.89119899, -0.51436460, 1.99256825, 1.41077507, 0.38642293, 0.17583044, 0.03320138, - 1.16508031, -0.24356931, 0.47440714, -0.17844005, 0.20463173, 1.90038323, 1.14138567, 0.34504607, - 0.16403235, -0.24976699, 0.29362509, 0.34502214, 0.41751838, 1.09390712, 0.12354189, 1.83025289, - 1.05569196, 0.34413773, 0.35469764, -0.69760042, 0.76338542, 1.75443077, 0.44126555, 0.18181801, - 0.73277575, 0.45443264, 0.17068321, 0.36591727, 0.72869974, -0.56090516, 0.14415455, 1.47314119, - 0.42908576, 0.73084539, -0.22373237, 2.26550221, 0.05606699, 0.39417523, 0.35234636, 0.78569502, - 0.77521765, -0.65131050, 0.40168875, 0.45527256, 0.38715565, 0.98521245, 2.21446753, 0.36345237, - -0.33269632, 0.36558092, 1.36846578, 1.37523413, 0.33698002, 0.28889543, -0.40639281, 1.01643157, - 0.59668219, 0.39197800, 1.03101778, 0.02551098, -0.03612846, -0.01371557, 0.43444607, 0.96746695, - 0.60583955, -0.10362893, 0.40574494, 0.38046724, 0.87445319, -0.00880148, -0.15437943, 0.08118075, - 0.44650543, 0.85956848, -0.27865338, 2.10837507, 0.04798460, 0.43948367, -0.10185169, 0.19978794, - 1.32323360, 1.20525467, 0.44288942, -0.84200430, 0.52563053, 0.69949460, 0.73987913, 0.34668452, - 0.74545687, 0.57696682, 0.22452033, -0.27099937, 0.39649010, 0.87083614, -0.18965788, 0.58206403, - -0.08108193, 0.42067638, 1.05117214, -0.34287399, 0.20424896, 0.27994895, 0.46011117, 0.70890665, + -0.04089922f, 0.35108989f, 0.30442458f, 0.39546335f, 1.15422225f, 0.10419128f, -0.19301927f, 0.01070970f, + 0.43977541f, 0.89119899f, -0.51436460f, 1.99256825f, 1.41077507f, 0.38642293f, 0.17583044f, 0.03320138f, + 1.16508031f, -0.24356931f, 0.47440714f, -0.17844005f, 0.20463173f, 1.90038323f, 1.14138567f, 0.34504607f, + 0.16403235f, -0.24976699f, 0.29362509f, 0.34502214f, 0.41751838f, 1.09390712f, 0.12354189f, 1.83025289f, + 1.05569196f, 0.34413773f, 0.35469764f, -0.69760042f, 0.76338542f, 1.75443077f, 0.44126555f, 0.18181801f, + 0.73277575f, 0.45443264f, 0.17068321f, 0.36591727f, 0.72869974f, -0.56090516f, 0.14415455f, 1.47314119f, + 0.42908576f, 0.73084539f, -0.22373237f, 2.26550221f, 0.05606699f, 0.39417523f, 0.35234636f, 0.78569502f, + 0.77521765f, -0.65131050f, 0.40168875f, 0.45527256f, 0.38715565f, 0.98521245f, 2.21446753f, 0.36345237f, + -0.33269632f, 0.36558092f, 1.36846578f, 1.37523413f, 0.33698002f, 0.28889543f, -0.40639281f, 1.01643157f, + 0.59668219f, 0.39197800f, 1.03101778f, 0.02551098f, -0.03612846f, -0.01371557f, 0.43444607f, 0.96746695f, + 0.60583955f, -0.10362893f, 0.40574494f, 0.38046724f, 0.87445319f, -0.00880148f, -0.15437943f, 0.08118075f, + 0.44650543f, 0.85956848f, -0.27865338f, 2.10837507f, 0.04798460f, 0.43948367f, -0.10185169f, 0.19978794f, + 1.32323360f, 1.20525467f, 0.44288942f, -0.84200430f, 0.52563053f, 0.69949460f, 0.73987913f, 0.34668452f, + 0.74545687f, 0.57696682f, 0.22452033f, -0.27099937f, 0.39649010f, 0.87083614f, -0.18965788f, 0.58206403f, + -0.08108193f, 0.42067638f, 1.05117214f, -0.34287399f, 0.20424896f, 0.27994895f, 0.46011117f, 0.70890665f, }; std::vector expected_mask_index = { 6, @@ -602,7 +615,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_position_ test_case.add_input(Shape{3, 8}, position_ids); test_case.add_expected_output(Shape{3, 8, 5}, expected_output); test_case.add_expected_output(Shape{3}, expected_mask_index); - test_case.run_with_tolerance_as_fp(1e-6); + test_case.run_with_tolerance_as_fp(1e-6f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention) { @@ -612,20 +625,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.91475844, 0.91523546, 0.82536930, 0.37491974, 0.22384071, 0.05941105, 0.01902100, 0.70131350, - 0.09603709, 0.44200060, 0.53106076, 0.79464376, 0.35469049, 0.25225943, 0.25179818, 0.29592562, - 0.24836586, 0.65088797, 0.93126643, 0.67980725, 0.85708112, 0.59808528, 0.46321425, 0.19301885, + 0.91475844f, 0.91523546f, 0.82536930f, 0.37491974f, 0.22384071f, 0.05941105f, 0.01902100f, 0.70131350f, + 0.09603709f, 0.44200060f, 0.53106076f, 0.79464376f, 0.35469049f, 0.25225943f, 0.25179818f, 0.29592562f, + 0.24836586f, 0.65088797f, 0.93126643f, 0.67980725f, 0.85708112f, 0.59808528f, 0.46321425f, 0.19301885f, }; std::vector output = { - 0.07966283, 0.10783536, -0.19424979, 0.54514766, 0.07965867, 0.10783093, -0.19424866, 0.54510003, - 0.07965846, 0.10783067, -0.19424550, 0.54509139, 0.07966217, 0.10783640, -0.19424903, 0.54512268, - 0.06940663, 0.10962760, -0.19698445, 0.53492010, 0.06940675, 0.10962828, -0.19698484, 0.53492326, - 0.06940714, 0.10963022, -0.19698712, 0.53494006, 0.06940673, 0.10962812, -0.19698519, 0.53492481, + 0.07966283f, 0.10783536f, -0.19424979f, 0.54514766f, 0.07965867f, 0.10783093f, -0.19424866f, 0.54510003f, + 0.07965846f, 0.10783067f, -0.19424550f, 0.54509139f, 0.07966217f, 0.10783640f, -0.19424903f, 0.54512268f, + 0.06940663f, 0.10962760f, -0.19698445f, 0.53492010f, 0.06940675f, 0.10962828f, -0.19698484f, 0.53492326f, + 0.06940714f, 0.10963022f, -0.19698712f, 0.53494006f, 0.06940673f, 0.10962812f, -0.19698519f, 0.53492481f, }; test_case.add_input(input); test_case.add_expected_output(output); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_qkv_hidden_sizes) { @@ -636,24 +649,24 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_qkv_hidden_sizes) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.56477863, 0.60309958, 0.35158035, 0.03123519, 0.81918180, 0.76905495, 0.47219241, 0.72016627, - 0.59377003, 0.91380632, 0.56797302, 0.34846428, 0.83839595, 0.16394103, 0.34676281, 0.09161621, - 0.45562279, 0.23317528, 0.37197968, 0.06727808, 0.08500192, 0.84915495, 0.68266946, 0.00227691, + 0.56477863f, 0.60309958f, 0.35158035f, 0.03123519f, 0.81918180f, 0.76905495f, 0.47219241f, 0.72016627f, + 0.59377003f, 0.91380632f, 0.56797302f, 0.34846428f, 0.83839595f, 0.16394103f, 0.34676281f, 0.09161621f, + 0.45562279f, 0.23317528f, 0.37197968f, 0.06727808f, 0.08500192f, 0.84915495f, 0.68266946f, 0.00227691f, }; std::vector output = { - -0.59370947, -0.30300471, 0.12048547, -0.09029539, 0.08041390, 0.10250041, -0.19381392, 0.55126983, - -0.59370828, -0.30301332, 0.12049319, -0.09029691, 0.08041921, 0.10250521, -0.19381438, 0.55127531, - -0.59370869, -0.30301058, 0.12049074, -0.09029643, 0.08041564, 0.10250199, -0.19381410, 0.55127168, - -0.59370929, -0.30300608, 0.12048667, -0.09029562, 0.08041184, 0.10249855, -0.19381374, 0.55126774, - -0.59681994, -0.26327702, 0.07638434, -0.06311120, 0.06671587, 0.10916986, -0.19412412, 0.51977092, - -0.59682053, -0.26328400, 0.07638102, -0.06311222, 0.06671817, 0.10917170, -0.19412397, 0.51977223, - -0.59682077, -0.26328647, 0.07637984, -0.06311259, 0.06671739, 0.10917108, -0.19412403, 0.51977175, - -0.59682101, -0.26328778, 0.07637922, -0.06311278, 0.06671065, 0.10916568, -0.19412443, 0.51976782, + -0.59370947f, -0.30300471f, 0.12048547f, -0.09029539f, 0.08041390f, 0.10250041f, -0.19381392f, 0.55126983f, + -0.59370828f, -0.30301332f, 0.12049319f, -0.09029691f, 0.08041921f, 0.10250521f, -0.19381438f, 0.55127531f, + -0.59370869f, -0.30301058f, 0.12049074f, -0.09029643f, 0.08041564f, 0.10250199f, -0.19381410f, 0.55127168f, + -0.59370929f, -0.30300608f, 0.12048667f, -0.09029562f, 0.08041184f, 0.10249855f, -0.19381374f, 0.55126774f, + -0.59681994f, -0.26327702f, 0.07638434f, -0.06311120f, 0.06671587f, 0.10916986f, -0.19412412f, 0.51977092f, + -0.59682053f, -0.26328400f, 0.07638102f, -0.06311222f, 0.06671817f, 0.10917170f, -0.19412397f, 0.51977223f, + -0.59682077f, -0.26328647f, 0.07637984f, -0.06311259f, 0.06671739f, 0.10917108f, -0.19412403f, 0.51977175f, + -0.59682101f, -0.26328778f, 0.07637922f, -0.06311278f, 0.06671065f, 0.10916568f, -0.19412443f, 0.51976782f, }; test_case.add_input(input); test_case.add_expected_output(output); - test_case.run_with_tolerance_as_fp(1e-4); + test_case.run_with_tolerance_as_fp(1e-4f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_unidirectional) { @@ -664,34 +677,34 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_unidirectional) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.89578921, 0.42421508, 0.35630688, 0.77461642, 0.65753633, 0.09723099, 0.62597734, 0.72117692, - 0.57636845, 0.17104276, 0.13245547, 0.59879875, 0.15624641, 0.44903454, 0.50483286, 0.92975074, - 0.36934483, 0.29919949, 0.57185954, 0.83036488, 0.08384345, 0.20378476, 0.74684393, 0.46716982, + 0.89578921f, 0.42421508f, 0.35630688f, 0.77461642f, 0.65753633f, 0.09723099f, 0.62597734f, 0.72117692f, + 0.57636845f, 0.17104276f, 0.13245547f, 0.59879875f, 0.15624641f, 0.44903454f, 0.50483286f, 0.92975074f, + 0.36934483f, 0.29919949f, 0.57185954f, 0.83036488f, 0.08384345f, 0.20378476f, 0.74684393f, 0.46716982f, }; std::vector output = { - 0.05604819, 0.09000472, -0.19437021, 0.52487367, 0.06211422, 0.08740954, -0.19139624, 0.52762908, - 0.06708897, 0.08992603, -0.19214047, 0.53631783, 0.06896879, 0.10248676, -0.19485690, 0.53477794, - 0.08577005, 0.12807365, -0.19762954, 0.54432857, 0.06929274, 0.10893210, -0.19599904, 0.53184807, - 0.07348281, 0.10215081, -0.19280069, 0.53552240, 0.07861833, 0.10517240, -0.19285706, 0.54126489, + 0.05604819f, 0.09000472f, -0.19437021f, 0.52487367f, 0.06211422f, 0.08740954f, -0.19139624f, 0.52762908f, + 0.06708897f, 0.08992603f, -0.19214047f, 0.53631783f, 0.06896879f, 0.10248676f, -0.19485690f, 0.53477794f, + 0.08577005f, 0.12807365f, -0.19762954f, 0.54432857f, 0.06929274f, 0.10893210f, -0.19599904f, 0.53184807f, + 0.07348281f, 0.10215081f, -0.19280069f, 0.53552240f, 0.07861833f, 0.10517240f, -0.19285706f, 0.54126489f, }; std::vector present = { - -0.60427380, -0.25958878, -0.59609234, -0.24055196, -0.59613681, -0.30088067, -0.59633607, -0.33270463, - 0.06899665, -0.09284544, 0.08059876, -0.06146053, 0.11841078, -0.10019838, 0.10605468, -0.09273906, - -0.59036821, -0.32410735, -0.60532302, -0.25127757, -0.58926487, -0.25271094, -0.58640373, -0.31730092, - 0.12509561, -0.07968873, 0.06005794, -0.08937149, 0.10523240, -0.05083811, 0.14162725, -0.07438751, - 0.05604819, 0.09000472, 0.06819826, 0.08480665, 0.07700446, 0.09494394, 0.07459175, 0.14003153, - -0.19437021, 0.52487367, -0.18843602, 0.53037173, -0.19362189, 0.55360907, -0.20299932, 0.53020388, - 0.08577005, 0.12807365, 0.05276009, 0.08972625, 0.08190014, 0.08852972, 0.09400313, 0.11423884, - -0.19762954, 0.54432857, -0.19435294, 0.51924801, -0.18643703, 0.54280555, -0.19302703, 0.55837619, + -0.60427380f, -0.25958878f, -0.59609234f, -0.24055196f, -0.59613681f, -0.30088067f, -0.59633607f, -0.33270463f, + 0.06899665f, -0.09284544f, 0.08059876f, -0.06146053f, 0.11841078f, -0.10019838f, 0.10605468f, -0.09273906f, + -0.59036821f, -0.32410735f, -0.60532302f, -0.25127757f, -0.58926487f, -0.25271094f, -0.58640373f, -0.31730092f, + 0.12509561f, -0.07968873f, 0.06005794f, -0.08937149f, 0.10523240f, -0.05083811f, 0.14162725f, -0.07438751f, + 0.05604819f, 0.09000472f, 0.06819826f, 0.08480665f, 0.07700446f, 0.09494394f, 0.07459175f, 0.14003153f, + -0.19437021f, 0.52487367f, -0.18843602f, 0.53037173f, -0.19362189f, 0.55360907f, -0.20299932f, 0.53020388f, + 0.08577005f, 0.12807365f, 0.05276009f, 0.08972625f, 0.08190014f, 0.08852972f, 0.09400313f, 0.11423884f, + -0.19762954f, 0.54432857f, -0.19435294f, 0.51924801f, -0.18643703f, 0.54280555f, -0.19302703f, 0.55837619f, }; test_case.add_input(input); test_case.add_expected_output(output); test_case.add_expected_output(present); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_1) { +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_1f) { const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, @@ -699,29 +712,29 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_1) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.02841483, 0.47845092, 0.14633700, 0.54597300, 0.40160629, 0.55281311, 0.14931096, 0.64483738, - 0.96559167, 0.05262021, 0.12391864, 0.20093553, 0.74290562, 0.19367455, 0.19253619, 0.41593507, - 0.91188699, 0.61606920, 0.72673517, 0.86981291, 0.19963337, 0.22747350, 0.34308898, 0.57267183, + 0.02841483f, 0.47845092f, 0.14633700f, 0.54597300f, 0.40160629f, 0.55281311f, 0.14931096f, 0.64483738f, + 0.96559167f, 0.05262021f, 0.12391864f, 0.20093553f, 0.74290562f, 0.19367455f, 0.19253619f, 0.41593507f, + 0.91188699f, 0.61606920f, 0.72673517f, 0.86981291f, 0.19963337f, 0.22747350f, 0.34308898f, 0.57267183f, }; std::vector mask_index = { 0, 1, }; std::vector output = { - 0.08298690, 0.12711772, -0.19757506, 0.54029012, 0.08298548, 0.12711433, -0.19757731, 0.54031140, - 0.08298430, 0.12711799, -0.19757695, 0.54031777, 0.08298548, 0.12711433, -0.19757444, 0.54028159, - 0.05380550, 0.10459180, -0.19593412, 0.50907606, 0.05380550, 0.10459180, -0.19593412, 0.50907606, - 0.05380550, 0.10459180, -0.19593412, 0.50907606, 0.05380550, 0.10459180, -0.19593412, 0.50907606, + 0.08298690f, 0.12711772f, -0.19757506f, 0.54029012f, 0.08298548f, 0.12711433f, -0.19757731f, 0.54031140f, + 0.08298430f, 0.12711799f, -0.19757695f, 0.54031777f, 0.08298548f, 0.12711433f, -0.19757444f, 0.54028159f, + 0.05380550f, 0.10459180f, -0.19593412f, 0.50907606f, 0.05380550f, 0.10459180f, -0.19593412f, 0.50907606f, + 0.05380550f, 0.10459180f, -0.19593412f, 0.50907606f, 0.05380550f, 0.10459180f, -0.19593412f, 0.50907606f, }; std::vector present = { - -0.58437425, -0.29483819, -0.59927911, -0.30336475, -0.59104657, -0.37327260, -0.59078789, -0.29863101, - 0.11751597, -0.04114649, 0.09933343, -0.09884726, 0.16250694, -0.12028439, 0.09319257, -0.05129660, - -0.60341775, -0.25221461, -0.58933026, -0.31912822, -0.59271193, -0.25470981, -0.59399152, -0.32643768, - 0.05398282, -0.07468132, 0.14743008, -0.09407346, 0.10399222, -0.06682440, 0.11632499, -0.08986320, - 0.09104910, 0.12973849, 0.06917210, 0.11059431, 0.09356256, 0.12594685, 0.07814129, 0.14221822, - -0.19329809, 0.53526556, -0.19787431, 0.53673857, -0.20045389, 0.57165766, -0.19869246, 0.51749766, - 0.05380550, 0.10459180, 0.09169570, 0.09892380, 0.07746917, 0.08042616, 0.07953370, 0.12909687, - -0.19593412, 0.50907606, -0.19202785, 0.56904894, -0.18689045, 0.54643762, -0.19969353, 0.53976399, + -0.58437425f, -0.29483819f, -0.59927911f, -0.30336475f, -0.59104657f, -0.37327260f, -0.59078789f, -0.29863101f, + 0.11751597f, -0.04114649f, 0.09933343f, -0.09884726f, 0.16250694f, -0.12028439f, 0.09319257f, -0.05129660f, + -0.60341775f, -0.25221461f, -0.58933026f, -0.31912822f, -0.59271193f, -0.25470981f, -0.59399152f, -0.32643768f, + 0.05398282f, -0.07468132f, 0.14743008f, -0.09407346f, 0.10399222f, -0.06682440f, 0.11632499f, -0.08986320f, + 0.09104910f, 0.12973849f, 0.06917210f, 0.11059431f, 0.09356256f, 0.12594685f, 0.07814129f, 0.14221822f, + -0.19329809f, 0.53526556f, -0.19787431f, 0.53673857f, -0.20045389f, 0.57165766f, -0.19869246f, 0.51749766f, + 0.05380550f, 0.10459180f, 0.09169570f, 0.09892380f, 0.07746917f, 0.08042616f, 0.07953370f, 0.12909687f, + -0.19593412f, 0.50907606f, -0.19202785f, 0.56904894f, -0.18689045f, 0.54643762f, -0.19969353f, 0.53976399f, }; test_case.add_input(input); @@ -739,9 +752,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_2) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.75259578, 0.81492645, 0.46713001, 0.29483622, 0.06768602, 0.95105755, 0.32065326, 0.52417183, - 0.73136383, 0.77176476, 0.60997742, 0.64625764, 0.16311000, 0.89680773, 0.01331447, 0.42468646, - 0.58711547, 0.00345124, 0.13053808, 0.46278623, 0.13786320, 0.65182054, 0.74864876, 0.81506181, + 0.75259578f, 0.81492645f, 0.46713001f, 0.29483622f, 0.06768602f, 0.95105755f, 0.32065326f, 0.52417183f, + 0.73136383f, 0.77176476f, 0.60997742f, 0.64625764f, 0.16311000f, 0.89680773f, 0.01331447f, 0.42468646f, + 0.58711547f, 0.00345124f, 0.13053808f, 0.46278623f, 0.13786320f, 0.65182054f, 0.74864876f, 0.81506181f, }; std::vector mask_index = { 3, @@ -750,20 +763,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_2) { 1, }; std::vector output = { - 0.07524174, 0.11320241, -0.19909523, 0.54785377, 0.06825337, 0.13981669, -0.20774621, 0.53718704, - 0.07531278, 0.12957911, -0.20330518, 0.54547405, 0.07531209, 0.12958010, -0.20330583, 0.54547292, - 0.08900890, 0.11150353, -0.18931937, 0.53757656, 0.07915881, 0.10416336, -0.18914750, 0.52921104, - 0.08285815, 0.11462159, -0.19115375, 0.53077918, 0.08285838, 0.11462225, -0.19115454, 0.53077984, + 0.07524174f, 0.11320241f, -0.19909523f, 0.54785377f, 0.06825337f, 0.13981669f, -0.20774621f, 0.53718704f, + 0.07531278f, 0.12957911f, -0.20330518f, 0.54547405f, 0.07531209f, 0.12958010f, -0.20330583f, 0.54547292f, + 0.08900890f, 0.11150353f, -0.18931937f, 0.53757656f, 0.07915881f, 0.10416336f, -0.18914750f, 0.52921104f, + 0.08285815f, 0.11462159f, -0.19115375f, 0.53077918f, 0.08285838f, 0.11462225f, -0.19115454f, 0.53077984f, }; std::vector present = { - -0.59630549, -0.28110915, -0.60274345, -0.36154836, -0.59437746, -0.33717164, -0.60134649, -0.29849592, - 0.11169122, -0.09345293, 0.11103803, -0.13096604, 0.13131849, -0.10597084, 0.10463209, -0.11332577, - -0.57949269, -0.27235535, -0.58941406, -0.25372508, -0.58658379, -0.28718373, -0.59821802, -0.32433146, - 0.13244939, -0.02865628, 0.09308393, -0.04083736, 0.10948701, -0.04423397, 0.13060363, -0.12316251, - 0.07509718, 0.08392500, 0.06825337, 0.13981669, 0.08239168, 0.11931328, 0.06770951, 0.09240761, - -0.19074154, 0.55260652, -0.20774621, 0.53718704, -0.19888818, 0.55371630, -0.19559640, 0.54754448, - 0.09983939, 0.10603377, 0.07915881, 0.10416336, 0.08655046, 0.12505992, 0.07738422, 0.09509270, - -0.18571433, 0.55095005, -0.18914750, 0.52921104, -0.19315663, 0.53234470, -0.19601485, 0.56322992, + -0.59630549f, -0.28110915f, -0.60274345f, -0.36154836f, -0.59437746f, -0.33717164f, -0.60134649f, -0.29849592f, + 0.11169122f, -0.09345293f, 0.11103803f, -0.13096604f, 0.13131849f, -0.10597084f, 0.10463209f, -0.11332577f, + -0.57949269f, -0.27235535f, -0.58941406f, -0.25372508f, -0.58658379f, -0.28718373f, -0.59821802f, -0.32433146f, + 0.13244939f, -0.02865628f, 0.09308393f, -0.04083736f, 0.10948701f, -0.04423397f, 0.13060363f, -0.12316251f, + 0.07509718f, 0.08392500f, 0.06825337f, 0.13981669f, 0.08239168f, 0.11931328f, 0.06770951f, 0.09240761f, + -0.19074154f, 0.55260652f, -0.20774621f, 0.53718704f, -0.19888818f, 0.55371630f, -0.19559640f, 0.54754448f, + 0.09983939f, 0.10603377f, 0.07915881f, 0.10416336f, 0.08655046f, 0.12505992f, 0.07738422f, 0.09509270f, + -0.18571433f, 0.55095005f, -0.18914750f, 0.52921104f, -0.19315663f, 0.53234470f, -0.19601485f, 0.56322992f, }; test_case.add_input(input); @@ -781,9 +794,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_3) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.33093750, 0.39181390, 0.14586255, 0.39709702, 0.98086524, 0.03891133, 0.72234219, 0.21966648, - 0.79986620, 0.97251678, 0.04131543, 0.43971965, 0.50185394, 0.11452501, 0.88111717, 0.76076663, - 0.31870860, 0.54107893, 0.91756296, 0.58112669, 0.99117357, 0.00256292, 0.58885485, 0.93481058, + 0.33093750f, 0.39181390f, 0.14586255f, 0.39709702f, 0.98086524f, 0.03891133f, 0.72234219f, 0.21966648f, + 0.79986620f, 0.97251678f, 0.04131543f, 0.43971965f, 0.50185394f, 0.11452501f, 0.88111717f, 0.76076663f, + 0.31870860f, 0.54107893f, 0.91756296f, 0.58112669f, 0.99117357f, 0.00256292f, 0.58885485f, 0.93481058f, }; std::vector mask = { 1, @@ -796,27 +809,27 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_3) { 1, }; std::vector output = { - 0.07551830, 0.10666487, -0.19357042, 0.53683108, 0.07551410, 0.10666656, -0.19356072, 0.53684169, - 0.07552745, 0.10666100, -0.19358172, 0.53682435, 0.07552218, 0.10666317, -0.19358677, 0.53681952, - 0.09727416, 0.13513327, -0.20121223, 0.57003713, 0.09727416, 0.13513327, -0.20121223, 0.57003713, - 0.09727416, 0.13513327, -0.20121223, 0.57003713, 0.09727416, 0.13513327, -0.20121223, 0.57003713, + 0.07551830f, 0.10666487f, -0.19357042f, 0.53683108f, 0.07551410f, 0.10666656f, -0.19356072f, 0.53684169f, + 0.07552745f, 0.10666100f, -0.19358172f, 0.53682435f, 0.07552218f, 0.10666317f, -0.19358677f, 0.53681952f, + 0.09727416f, 0.13513327f, -0.20121223f, 0.57003713f, 0.09727416f, 0.13513327f, -0.20121223f, 0.57003713f, + 0.09727416f, 0.13513327f, -0.20121223f, 0.57003713f, 0.09727416f, 0.13513327f, -0.20121223f, 0.57003713f, }; std::vector present = { - -0.59174627, -0.27471560, -0.58307797, -0.25967693, -0.60766846, -0.31754097, -0.61241394, -0.26291698, - 0.09206123, -0.05307099, 0.12491645, -0.03853742, 0.08732655, -0.13050151, 0.04073093, -0.10792807, - -0.60556883, -0.34055573, -0.60474855, -0.28785610, -0.60757709, -0.32514900, -0.58872569, -0.37967020, - 0.09779400, -0.13136166, 0.07915612, -0.10649752, 0.11043755, -0.15124020, 0.16626491, -0.11274654, - 0.07639833, 0.11762549, 0.09370039, 0.09133558, 0.05661478, 0.11096847, 0.04019671, 0.10117501, - -0.19371650, 0.52530587, -0.18429738, 0.55240726, -0.20283231, 0.53265429, -0.20036045, 0.50568837, - 0.06171235, 0.12687264, 0.05802051, 0.10266830, 0.06172965, 0.08967118, 0.09727416, 0.13513327, - -0.20576829, 0.53365225, -0.19832623, 0.52809310, -0.19971462, 0.55584043, -0.20121223, 0.57003713, + -0.59174627f, -0.27471560f, -0.58307797f, -0.25967693f, -0.60766846f, -0.31754097f, -0.61241394f, -0.26291698f, + 0.09206123f, -0.05307099f, 0.12491645f, -0.03853742f, 0.08732655f, -0.13050151f, 0.04073093f, -0.10792807f, + -0.60556883f, -0.34055573f, -0.60474855f, -0.28785610f, -0.60757709f, -0.32514900f, -0.58872569f, -0.37967020f, + 0.09779400f, -0.13136166f, 0.07915612f, -0.10649752f, 0.11043755f, -0.15124020f, 0.16626491f, -0.11274654f, + 0.07639833f, 0.11762549f, 0.09370039f, 0.09133558f, 0.05661478f, 0.11096847f, 0.04019671f, 0.10117501f, + -0.19371650f, 0.52530587f, -0.18429738f, 0.55240726f, -0.20283231f, 0.53265429f, -0.20036045f, 0.50568837f, + 0.06171235f, 0.12687264f, 0.05802051f, 0.10266830f, 0.06172965f, 0.08967118f, 0.09727416f, 0.13513327f, + -0.20576829f, 0.53365225f, -0.19832623f, 0.52809310f, -0.19971462f, 0.55584043f, -0.20121223f, 0.57003713f, }; test_case.add_input(input); test_case.add_input(mask); test_case.add_expected_output(output); test_case.add_expected_output(present); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_4) { @@ -827,35 +840,35 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_mask_index_4) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.23565151, 0.58627969, 0.75137484, 0.68586946, 0.62750375, 0.13284931, 0.13347220, 0.36357051, - 0.56910241, 0.48275986, 0.49440190, 0.45483324, 0.63547862, 0.97893149, 0.40630588, 0.38783622, - 0.07172249, 0.46385381, 0.99764502, 0.22219376, 0.67735291, 0.40799847, 0.74337566, 0.87263006, + 0.23565151f, 0.58627969f, 0.75137484f, 0.68586946f, 0.62750375f, 0.13284931f, 0.13347220f, 0.36357051f, + 0.56910241f, 0.48275986f, 0.49440190f, 0.45483324f, 0.63547862f, 0.97893149f, 0.40630588f, 0.38783622f, + 0.07172249f, 0.46385381f, 0.99764502f, 0.22219376f, 0.67735291f, 0.40799847f, 0.74337566f, 0.87263006f, }; std::vector mask = { 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, }; std::vector output = { - 0.07771622, 0.10724538, -0.19453585, 0.54342043, 0.07459468, 0.10934003, -0.19561143, 0.53936625, - 0.07927690, 0.10619678, -0.19399606, 0.54543519, 0.07459468, 0.10934003, -0.19561143, 0.53936625, - 0.05485561, 0.11278091, -0.20117569, 0.52096349, 0.06629646, 0.10195158, -0.19900991, 0.54654449, - 0.06491723, 0.10292297, -0.19678673, 0.53451663, 0.06549793, 0.11126325, -0.19989857, 0.53717279, + 0.07771622f, 0.10724538f, -0.19453585f, 0.54342043f, 0.07459468f, 0.10934003f, -0.19561143f, 0.53936625f, + 0.07927690f, 0.10619678f, -0.19399606f, 0.54543519f, 0.07459468f, 0.10934003f, -0.19561143f, 0.53936625f, + 0.05485561f, 0.11278091f, -0.20117569f, 0.52096349f, 0.06629646f, 0.10195158f, -0.19900991f, 0.54654449f, + 0.06491723f, 0.10292297f, -0.19678673f, 0.53451663f, 0.06549793f, 0.11126325f, -0.19989857f, 0.53717279f, }; std::vector present = { - -0.59188855, -0.34495637, -0.59508181, -0.25013468, -0.59176934, -0.33229247, -0.59576762, -0.29731843, - 0.14217430, -0.10403840, 0.08584045, -0.06193545, 0.12358667, -0.08588549, 0.10515238, -0.08629489, - -0.59092808, -0.28260738, -0.60047609, -0.30411413, -0.61210287, -0.28645760, -0.59391296, -0.34649473, - 0.12789863, -0.08159252, 0.08122411, -0.08866425, 0.06395009, -0.12896645, 0.14855847, -0.11978809, - 0.08783118, 0.12152332, 0.07067389, 0.09078297, 0.08385989, 0.13306075, 0.07459468, 0.10934003, - -0.19849420, 0.55928540, -0.18948570, 0.53154731, -0.19960676, 0.54237455, -0.19561143, 0.53936625, - 0.08509844, 0.08314656, 0.06388859, 0.12990499, 0.04582624, 0.09566365, 0.08674107, 0.10823163, - -0.18808734, 0.56137776, -0.20168513, 0.51830697, -0.20066255, 0.52363914, -0.19737384, 0.56921995, + -0.59188855f, -0.34495637f, -0.59508181f, -0.25013468f, -0.59176934f, -0.33229247f, -0.59576762f, -0.29731843f, + 0.14217430f, -0.10403840f, 0.08584045f, -0.06193545f, 0.12358667f, -0.08588549f, 0.10515238f, -0.08629489f, + -0.59092808f, -0.28260738f, -0.60047609f, -0.30411413f, -0.61210287f, -0.28645760f, -0.59391296f, -0.34649473f, + 0.12789863f, -0.08159252f, 0.08122411f, -0.08866425f, 0.06395009f, -0.12896645f, 0.14855847f, -0.11978809f, + 0.08783118f, 0.12152332f, 0.07067389f, 0.09078297f, 0.08385989f, 0.13306075f, 0.07459468f, 0.10934003f, + -0.19849420f, 0.55928540f, -0.18948570f, 0.53154731f, -0.19960676f, 0.54237455f, -0.19561143f, 0.53936625f, + 0.08509844f, 0.08314656f, 0.06388859f, 0.12990499f, 0.04582624f, 0.09566365f, 0.08674107f, 0.10823163f, + -0.18808734f, 0.56137776f, -0.20168513f, 0.51830697f, -0.20066255f, 0.52363914f, -0.19737384f, 0.56921995f, }; test_case.add_input(input); test_case.add_input(mask); test_case.add_expected_output(output); test_case.add_expected_output(present); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_past) { @@ -866,9 +879,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_past) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.82966000, 0.77751911, 0.08977074, 0.06076468, 0.40659550, 0.19995944, 0.55544919, 0.83971608, - 0.86254036, 0.30894691, 0.80156928, 0.83092463, 0.14506543, 0.32196075, 0.42209163, 0.24465553, - 0.93944097, 0.73528159, 0.23347616, 0.60544974, 0.93329269, 0.67604774, 0.56349903, 0.26199624, + 0.82966000f, 0.77751911f, 0.08977074f, 0.06076468f, 0.40659550f, 0.19995944f, 0.55544919f, 0.83971608f, + 0.86254036f, 0.30894691f, 0.80156928f, 0.83092463f, 0.14506543f, 0.32196075f, 0.42209163f, 0.24465553f, + 0.93944097f, 0.73528159f, 0.23347616f, 0.60544974f, 0.93329269f, 0.67604774f, 0.56349903f, 0.26199624f, }; std::vector mask = { 1, @@ -891,41 +904,42 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_past) { 1, }; std::vector past = { - 0.92467678, 0.79873562, 0.00939191, 0.34891853, 0.35521412, 0.21872006, 0.89974332, 0.74132687, 0.73566031, - 0.75168055, 0.06773245, 0.85702997, 0.76256698, 0.51739877, 0.91567177, 0.66617578, 0.88056499, 0.08436447, - 0.54744655, 0.25466520, 0.08500137, 0.19271941, 0.86525357, 0.21717627, 0.97158766, 0.42288730, 0.09890039, - 0.01148765, 0.97024685, 0.19697112, 0.67671591, 0.67960924, 0.46656516, 0.30850092, 0.73536104, 0.73938161, - 0.91650903, 0.57628596, 0.51164514, 0.11695814, 0.79792547, 0.97192264, 0.29246020, 0.41030061, 0.19014873, - 0.90233624, 0.84986305, 0.26141909, 0.84528726, 0.81416380, 0.00429944, 0.31476986, 0.00440918, 0.77413058, - 0.13409913, 0.20965169, 0.61764991, 0.55266041, 0.56107825, 0.42051074, 0.16804738, 0.80362344, 0.52392679, - 0.27550557, 0.66738850, 0.39348483, 0.31801429, 0.30325863, 0.37068403, 0.92767614, 0.60799408, 0.01458820, - 0.24194679, 0.59596598, 0.81762302, 0.38094005, 0.16618672, 0.92488551, 0.84298438, 0.21752745, + 0.92467678f, 0.79873562f, 0.00939191f, 0.34891853f, 0.35521412f, 0.21872006f, 0.89974332f, 0.74132687f, + 0.73566031f, 0.75168055f, 0.06773245f, 0.85702997f, 0.76256698f, 0.51739877f, 0.91567177f, 0.66617578f, + 0.88056499f, 0.08436447f, 0.54744655f, 0.25466520f, 0.08500137f, 0.19271941f, 0.86525357f, 0.21717627f, + 0.97158766f, 0.42288730f, 0.09890039f, 0.01148765f, 0.97024685f, 0.19697112f, 0.67671591f, 0.67960924f, + 0.46656516f, 0.30850092f, 0.73536104f, 0.73938161f, 0.91650903f, 0.57628596f, 0.51164514f, 0.11695814f, + 0.79792547f, 0.97192264f, 0.29246020f, 0.41030061f, 0.19014873f, 0.90233624f, 0.84986305f, 0.26141909f, + 0.84528726f, 0.81416380f, 0.00429944f, 0.31476986f, 0.00440918f, 0.77413058f, 0.13409913f, 0.20965169f, + 0.61764991f, 0.55266041f, 0.56107825f, 0.42051074f, 0.16804738f, 0.80362344f, 0.52392679f, 0.27550557f, + 0.66738850f, 0.39348483f, 0.31801429f, 0.30325863f, 0.37068403f, 0.92767614f, 0.60799408f, 0.01458820f, + 0.24194679f, 0.59596598f, 0.81762302f, 0.38094005f, 0.16618672f, 0.92488551f, 0.84298438f, 0.21752745f, }; std::vector output = { - 0.26186451, 0.45950246, -0.04001215, 0.47680017, 0.26333901, 0.46158865, -0.04006424, 0.47588652, - 0.26875457, 0.47031689, -0.03951600, 0.47674999, 0.26851410, 0.46987134, -0.03919901, 0.47629333, - 0.18083976, 0.16579385, -0.05161894, 0.63075018, 0.18228555, 0.16642828, -0.04873618, 0.63316816, - 0.18362364, 0.16702136, -0.05045432, 0.63178891, 0.18000112, 0.16541445, -0.05139139, 0.63105792, + 0.26186451f, 0.45950246f, -0.04001215f, 0.47680017f, 0.26333901f, 0.46158865f, -0.04006424f, 0.47588652f, + 0.26875457f, 0.47031689f, -0.03951600f, 0.47674999f, 0.26851410f, 0.46987134f, -0.03919901f, 0.47629333f, + 0.18083976f, 0.16579385f, -0.05161894f, 0.63075018f, 0.18228555f, 0.16642828f, -0.04873618f, 0.63316816f, + 0.18362364f, 0.16702136f, -0.05045432f, 0.63178891f, 0.18000112f, 0.16541445f, -0.05139139f, 0.63105792f, }; std::vector present = { - 0.92467678, 0.79873562, 0.00939191, 0.34891853, 0.35521412, 0.21872006, 0.89974332, 0.74132687, - 0.73566031, 0.75168055, -0.59527576, -0.23625080, -0.58657664, -0.29827437, -0.59528387, -0.33578828, - -0.59068960, -0.34870598, 0.06773245, 0.85702997, 0.76256698, 0.51739877, 0.91567177, 0.66617578, - 0.88056499, 0.08436447, 0.54744655, 0.25466520, 0.08536442, -0.06134639, 0.11295843, -0.04818217, - 0.14562836, -0.12305059, 0.15695867, -0.11161390, 0.08500137, 0.19271941, 0.86525357, 0.21717627, - 0.97158766, 0.42288730, 0.09890039, 0.01148765, 0.97024685, 0.19697112, -0.59141791, -0.31600696, - -0.58647990, -0.34302223, -0.59306550, -0.36427227, -0.59695083, -0.26431620, 0.67671591, 0.67960924, - 0.46656516, 0.30850092, 0.73536104, 0.73938161, 0.91650903, 0.57628596, 0.51164514, 0.11695814, - 0.11255538, -0.07302766, 0.16620418, -0.09871224, 0.15272795, -0.12076923, 0.08827571, -0.07442430, - 0.79792547, 0.97192264, 0.29246020, 0.41030061, 0.19014873, 0.90233624, 0.84986305, 0.26141909, - 0.84528726, 0.81416380, 0.07014155, 0.07749540, 0.08745074, 0.13131952, 0.08430066, 0.09709007, - 0.09247591, 0.11065811, 0.00429944, 0.31476986, 0.00440918, 0.77413058, 0.13409913, 0.20965169, - 0.61764991, 0.55266041, 0.56107825, 0.42051074, -0.18658412, 0.53568852, -0.19482780, 0.53271860, - -0.19558203, 0.57155901, -0.19633618, 0.57260245, 0.16804738, 0.80362344, 0.52392679, 0.27550557, - 0.66738850, 0.39348483, 0.31801429, 0.30325863, 0.37068403, 0.92767614, 0.08172131, 0.13249113, - 0.09947956, 0.10781212, 0.08890627, 0.12280971, 0.06911418, 0.09499176, 0.60799408, 0.01458820, - 0.24194679, 0.59596598, 0.81762302, 0.38094005, 0.16618672, 0.92488551, 0.84298438, 0.21752745, - -0.19839945, 0.53462923, -0.19349247, 0.57778782, -0.20039621, 0.56689924, -0.19190890, 0.53286803, + 0.92467678f, 0.79873562f, 0.00939191f, 0.34891853f, 0.35521412f, 0.21872006f, 0.89974332f, 0.74132687f, + 0.73566031f, 0.75168055f, -0.59527576f, -0.23625080f, -0.58657664f, -0.29827437f, -0.59528387f, -0.33578828f, + -0.59068960f, -0.34870598f, 0.06773245f, 0.85702997f, 0.76256698f, 0.51739877f, 0.91567177f, 0.66617578f, + 0.88056499f, 0.08436447f, 0.54744655f, 0.25466520f, 0.08536442f, -0.06134639f, 0.11295843f, -0.04818217f, + 0.14562836f, -0.12305059f, 0.15695867f, -0.11161390f, 0.08500137f, 0.19271941f, 0.86525357f, 0.21717627f, + 0.97158766f, 0.42288730f, 0.09890039f, 0.01148765f, 0.97024685f, 0.19697112f, -0.59141791f, -0.31600696f, + -0.58647990f, -0.34302223f, -0.59306550f, -0.36427227f, -0.59695083f, -0.26431620f, 0.67671591f, 0.67960924f, + 0.46656516f, 0.30850092f, 0.73536104f, 0.73938161f, 0.91650903f, 0.57628596f, 0.51164514f, 0.11695814f, + 0.11255538f, -0.07302766f, 0.16620418f, -0.09871224f, 0.15272795f, -0.12076923f, 0.08827571f, -0.07442430f, + 0.79792547f, 0.97192264f, 0.29246020f, 0.41030061f, 0.19014873f, 0.90233624f, 0.84986305f, 0.26141909f, + 0.84528726f, 0.81416380f, 0.07014155f, 0.07749540f, 0.08745074f, 0.13131952f, 0.08430066f, 0.09709007f, + 0.09247591f, 0.11065811f, 0.00429944f, 0.31476986f, 0.00440918f, 0.77413058f, 0.13409913f, 0.20965169f, + 0.61764991f, 0.55266041f, 0.56107825f, 0.42051074f, -0.18658412f, 0.53568852f, -0.19482780f, 0.53271860f, + -0.19558203f, 0.57155901f, -0.19633618f, 0.57260245f, 0.16804738f, 0.80362344f, 0.52392679f, 0.27550557f, + 0.66738850f, 0.39348483f, 0.31801429f, 0.30325863f, 0.37068403f, 0.92767614f, 0.08172131f, 0.13249113f, + 0.09947956f, 0.10781212f, 0.08890627f, 0.12280971f, 0.06911418f, 0.09499176f, 0.60799408f, 0.01458820f, + 0.24194679f, 0.59596598f, 0.81762302f, 0.38094005f, 0.16618672f, 0.92488551f, 0.84298438f, 0.21752745f, + -0.19839945f, 0.53462923f, -0.19349247f, 0.57778782f, -0.20039621f, 0.56689924f, -0.19190890f, 0.53286803f, }; test_case.add_input(input); @@ -933,7 +947,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_past) { test_case.add_input(past); test_case.add_expected_output(output); test_case.add_expected_output(present); - test_case.run_with_tolerance_as_fp(1e-6); + test_case.run_with_tolerance_as_fp(1e-6f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_extra_add) { @@ -944,9 +958,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_extra_add) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.14930259, 0.11199699, 0.81292826, 0.08368169, 0.05704883, 0.41276145, 0.38760167, 0.00146112, - 0.14275745, 0.54254925, 0.07962929, 0.31023681, 0.09597706, 0.60583973, 0.90233743, 0.33360451, - 0.18193199, 0.19159532, 0.07869831, 0.86026299, 0.20683478, 0.40150928, 0.93124926, 0.31805834, + 0.14930259f, 0.11199699f, 0.81292826f, 0.08368169f, 0.05704883f, 0.41276145f, 0.38760167f, 0.00146112f, + 0.14275745f, 0.54254925f, 0.07962929f, 0.31023681f, 0.09597706f, 0.60583973f, 0.90233743f, 0.33360451f, + 0.18193199f, 0.19159532f, 0.07869831f, 0.86026299f, 0.20683478f, 0.40150928f, 0.93124926f, 0.31805834f, }; std::vector mask = { 0, @@ -959,30 +973,30 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_extra_add) { 0, }; std::vector extra_add = { - 0.73230380, 0.61824518, 0.19738488, 0.57034588, 0.22331032, 0.53262889, 0.60098642, 0.72943515, - 0.09009175, 0.81116527, 0.47240964, 0.49679127, 0.41110733, 0.29418564, 0.93818313, 0.64175284, - 0.06807775, 0.66733366, 0.78848422, 0.48788327, 0.38806340, 0.14002480, 0.72263688, 0.22772972, - 0.24000823, 0.75820386, 0.64254439, 0.19385594, 0.95595860, 0.59840417, 0.93769604, 0.62474734, - 0.36690548, 0.76047903, 0.62352085, 0.58574778, 0.64251810, 0.78072041, 0.43344691, 0.75383639, - 0.73950553, 0.92625278, 0.05066428, 0.08448382, 0.25980917, 0.50312829, 0.97800279, 0.05422170, - 0.05171391, 0.82828254, 0.42234898, 0.95752198, 0.96325767, 0.97909677, 0.35578200, 0.48091716, - 0.03637243, 0.91552693, 0.43403026, 0.94275808, 0.51182085, 0.86773109, 0.38459453, 0.87822068, + 0.73230380f, 0.61824518f, 0.19738488f, 0.57034588f, 0.22331032f, 0.53262889f, 0.60098642f, 0.72943515f, + 0.09009175f, 0.81116527f, 0.47240964f, 0.49679127f, 0.41110733f, 0.29418564f, 0.93818313f, 0.64175284f, + 0.06807775f, 0.66733366f, 0.78848422f, 0.48788327f, 0.38806340f, 0.14002480f, 0.72263688f, 0.22772972f, + 0.24000823f, 0.75820386f, 0.64254439f, 0.19385594f, 0.95595860f, 0.59840417f, 0.93769604f, 0.62474734f, + 0.36690548f, 0.76047903f, 0.62352085f, 0.58574778f, 0.64251810f, 0.78072041f, 0.43344691f, 0.75383639f, + 0.73950553f, 0.92625278f, 0.05066428f, 0.08448382f, 0.25980917f, 0.50312829f, 0.97800279f, 0.05422170f, + 0.05171391f, 0.82828254f, 0.42234898f, 0.95752198f, 0.96325767f, 0.97909677f, 0.35578200f, 0.48091716f, + 0.03637243f, 0.91552693f, 0.43403026f, 0.94275808f, 0.51182085f, 0.86773109f, 0.38459453f, 0.87822068f, }; std::vector output = { - 0.06090815, 0.12919067, -0.19883196, 0.50295448, 0.06090815, 0.12919067, -0.19883196, 0.50295448, - 0.06090815, 0.12919067, -0.19883196, 0.50295448, 0.06090815, 0.12919067, -0.19883196, 0.50295448, - 0.08714182, 0.12259886, -0.19516067, 0.54010558, 0.08671370, 0.12369543, -0.19658084, 0.54502594, - 0.08458151, 0.12488046, -0.19519810, 0.53906947, 0.09063499, 0.12088943, -0.19583938, 0.54266596, + 0.06090815f, 0.12919067f, -0.19883196f, 0.50295448f, 0.06090815f, 0.12919067f, -0.19883196f, 0.50295448f, + 0.06090815f, 0.12919067f, -0.19883196f, 0.50295448f, 0.06090815f, 0.12919067f, -0.19883196f, 0.50295448f, + 0.08714182f, 0.12259886f, -0.19516067f, 0.54010558f, 0.08671370f, 0.12369543f, -0.19658084f, 0.54502594f, + 0.08458151f, 0.12488046f, -0.19519810f, 0.53906947f, 0.09063499f, 0.12088943f, -0.19583938f, 0.54266596f, }; std::vector present = { - -0.59800303, -0.35666457, -0.59420627, -0.31881350, -0.59887993, -0.27025288, -0.60216135, -0.27772796, - 0.11659990, -0.11224300, 0.09693416, -0.07304113, 0.06023501, -0.05941332, 0.06434284, -0.07978789, - -0.59005713, -0.37009716, -0.59542215, -0.27914333, -0.57998544, -0.29826957, -0.58625919, -0.28872511, - 0.15994480, -0.11288825, 0.07906821, -0.05991337, 0.14479136, -0.04415035, 0.13493451, -0.06541853, - 0.07513385, 0.14411135, 0.07505661, 0.14532046, 0.06090815, 0.12919067, 0.05788904, 0.12018456, - -0.20586906, 0.53715372, -0.20203318, 0.52092510, -0.19883196, 0.50295448, -0.19937295, 0.51055026, - 0.09417956, 0.12943678, 0.06923291, 0.12574309, 0.10221909, 0.11366953, 0.09235901, 0.09584601, - -0.20036517, 0.56818324, -0.19709785, 0.51547027, -0.18871340, 0.55736589, -0.18826833, 0.55965197, + -0.59800303f, -0.35666457f, -0.59420627f, -0.31881350f, -0.59887993f, -0.27025288f, -0.60216135f, -0.27772796f, + 0.11659990f, -0.11224300f, 0.09693416f, -0.07304113f, 0.06023501f, -0.05941332f, 0.06434284f, -0.07978789f, + -0.59005713f, -0.37009716f, -0.59542215f, -0.27914333f, -0.57998544f, -0.29826957f, -0.58625919f, -0.28872511f, + 0.15994480f, -0.11288825f, 0.07906821f, -0.05991337f, 0.14479136f, -0.04415035f, 0.13493451f, -0.06541853f, + 0.07513385f, 0.14411135f, 0.07505661f, 0.14532046f, 0.06090815f, 0.12919067f, 0.05788904f, 0.12018456f, + -0.20586906f, 0.53715372f, -0.20203318f, 0.52092510f, -0.19883196f, 0.50295448f, -0.19937295f, 0.51055026f, + 0.09417956f, 0.12943678f, 0.06923291f, 0.12574309f, 0.10221909f, 0.11366953f, 0.09235901f, 0.09584601f, + -0.20036517f, 0.56818324f, -0.19709785f, 0.51547027f, -0.18871340f, 0.55736589f, -0.18826833f, 0.55965197f, }; test_case.add_input(input); @@ -990,7 +1004,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_extra_add) { test_case.add_input(extra_add); test_case.add_expected_output(output); test_case.add_expected_output(present); - test_case.run_with_tolerance_as_fp(1e-7); + test_case.run_with_tolerance_as_fp(1e-7f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_dynamic_shapes) { @@ -1001,29 +1015,30 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_dynamic_shapes) { auto test_case = test::TestCase(function, s_device); std::vector input = { - 0.42226878, 0.50984067, 0.80440795, 0.68040705, 0.93614250, 0.45104721, 0.71767306, 0.48596525, - 0.70076728, 0.04500086, 0.28930107, 0.77435863, 0.19392140, 0.90290719, 0.91955870, 0.58811885, - 0.76795286, 0.62884814, 0.23377730, 0.49212688, 0.87256873, 0.11944817, 0.57715887, 0.91886938, + 0.42226878f, 0.50984067f, 0.80440795f, 0.68040705f, 0.93614250f, 0.45104721f, 0.71767306f, 0.48596525f, + 0.70076728f, 0.04500086f, 0.28930107f, 0.77435863f, 0.19392140f, 0.90290719f, 0.91955870f, 0.58811885f, + 0.76795286f, 0.62884814f, 0.23377730f, 0.49212688f, 0.87256873f, 0.11944817f, 0.57715887f, 0.91886938f, }; std::vector weights = { - 0.99377930, 0.22733542, 0.43217131, 0.60717988, 0.97224706, 0.70020503, 0.92439449, 0.41512674, 0.47728160, - 0.40306625, 0.72619593, 0.37954643, 0.36950976, 0.84305370, 0.61671126, 0.22251014, 0.73839295, 0.73471880, - 0.37428924, 0.80240524, 0.23120961, 0.06072779, 0.92840081, 0.71558088, 0.08719950, 0.51666921, 0.53768843, - 0.48113129, 0.46389169, 0.01036468, 0.37341005, 0.67195475, 0.53599644, 0.41795707, 0.58081782, 0.97939289, + 0.99377930f, 0.22733542f, 0.43217131f, 0.60717988f, 0.97224706f, 0.70020503f, 0.92439449f, 0.41512674f, + 0.47728160f, 0.40306625f, 0.72619593f, 0.37954643f, 0.36950976f, 0.84305370f, 0.61671126f, 0.22251014f, + 0.73839295f, 0.73471880f, 0.37428924f, 0.80240524f, 0.23120961f, 0.06072779f, 0.92840081f, 0.71558088f, + 0.08719950f, 0.51666921f, 0.53768843f, 0.48113129f, 0.46389169f, 0.01036468f, 0.37341005f, 0.67195475f, + 0.53599644f, 0.41795707f, 0.58081782f, 0.97939289f, }; std::vector bias = { - 0.77122736, - 0.75600564, - 0.86177206, - 0.69982684, - 0.74719858, - 0.78054035, - 0.80007398, - 0.74902135, - 0.81258053, - 0.01575289, - 0.08463049, - 0.39671996, + 0.77122736f, + 0.75600564f, + 0.86177206f, + 0.69982684f, + 0.74719858f, + 0.78054035f, + 0.80007398f, + 0.74902135f, + 0.81258053f, + 0.01575289f, + 0.08463049f, + 0.39671996f, }; std::vector mask = { 0, @@ -1046,39 +1061,42 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_dynamic_shapes) { 0, }; std::vector past = { - 0.27759778, 0.18458818, 0.63114458, 0.09953160, 0.59739488, 0.63917851, 0.18828323, 0.65625650, 0.84574437, - 0.91846281, 0.55102497, 0.27506110, 0.06816208, 0.82616585, 0.85912132, 0.88682729, 0.14730524, 0.61618829, - 0.89891797, 0.27753425, 0.57438278, 0.33753166, 0.88768929, 0.35533753, 0.30193496, 0.81678063, 0.26569194, - 0.62769043, 0.61990744, 0.59077013, 0.11058200, 0.97370809, 0.81339806, 0.57207322, 0.80417949, 0.54185718, - 0.80831683, 0.29390740, 0.29051417, 0.51964313, 0.04341308, 0.05925354, 0.82397246, 0.55753845, 0.61247689, - 0.98571628, 0.07566493, 0.37537411, 0.42080343, 0.21715857, 0.57869565, 0.55962265, 0.82500041, 0.60776925, - 0.19367239, 0.88382334, 0.20328504, 0.58192456, 0.94542676, 0.98562658, 0.64355153, 0.69856495, 0.30377558, - 0.02857198, 0.96969068, 0.48450547, 0.98341352, 0.03546083, 0.84963584, 0.94460547, 0.90907097, 0.22525074, - 0.12530145, 0.52223104, 0.09549426, 0.93127102, 0.93429947, 0.01428344, 0.74249738, 0.22606593, + 0.27759778f, 0.18458818f, 0.63114458f, 0.09953160f, 0.59739488f, 0.63917851f, 0.18828323f, 0.65625650f, + 0.84574437f, 0.91846281f, 0.55102497f, 0.27506110f, 0.06816208f, 0.82616585f, 0.85912132f, 0.88682729f, + 0.14730524f, 0.61618829f, 0.89891797f, 0.27753425f, 0.57438278f, 0.33753166f, 0.88768929f, 0.35533753f, + 0.30193496f, 0.81678063f, 0.26569194f, 0.62769043f, 0.61990744f, 0.59077013f, 0.11058200f, 0.97370809f, + 0.81339806f, 0.57207322f, 0.80417949f, 0.54185718f, 0.80831683f, 0.29390740f, 0.29051417f, 0.51964313f, + 0.04341308f, 0.05925354f, 0.82397246f, 0.55753845f, 0.61247689f, 0.98571628f, 0.07566493f, 0.37537411f, + 0.42080343f, 0.21715857f, 0.57869565f, 0.55962265f, 0.82500041f, 0.60776925f, 0.19367239f, 0.88382334f, + 0.20328504f, 0.58192456f, 0.94542676f, 0.98562658f, 0.64355153f, 0.69856495f, 0.30377558f, 0.02857198f, + 0.96969068f, 0.48450547f, 0.98341352f, 0.03546083f, 0.84963584f, 0.94460547f, 0.90907097f, 0.22525074f, + 0.12530145f, 0.52223104f, 0.09549426f, 0.93127102f, 0.93429947f, 0.01428344f, 0.74249738f, 0.22606593f, }; std::vector output = { - 1.47439122, 0.50951630, 1.17974961, 1.58501005, 1.49403512, 0.51560062, 1.18972027, 1.59668207, - 1.48384988, 0.51248586, 1.18596375, 1.59219086, 1.44181466, 0.50219649, 1.15537691, 1.55348074, - 0.83429223, 0.59521818, 0.87688094, 0.13611843, 0.82936716, 0.61004817, 0.87633312, 0.13887596, - 0.83155584, 0.59382534, 0.87496555, 0.14041223, 0.83309680, 0.58982348, 0.87517864, 0.13930768, + 1.47439122f, 0.50951630f, 1.17974961f, 1.58501005f, 1.49403512f, 0.51560062f, 1.18972027f, 1.59668207f, + 1.48384988f, 0.51248586f, 1.18596375f, 1.59219086f, 1.44181466f, 0.50219649f, 1.15537691f, 1.55348074f, + 0.83429223f, 0.59521818f, 0.87688094f, 0.13611843f, 0.82936716f, 0.61004817f, 0.87633312f, 0.13887596f, + 0.83155584f, 0.59382534f, 0.87496555f, 0.14041223f, 0.83309680f, 0.58982348f, 0.87517864f, 0.13930768f, }; std::vector present = { - 0.27759778, 0.18458818, 0.63114458, 0.09953160, 0.59739488, 0.63917851, 0.18828323, 0.65625650, 0.84574437, - 0.91846281, 1.90736914, 1.45914197, 2.30920029, 1.94944119, 2.12886763, 1.64736962, 1.36378694, 1.03263116, - 0.55102497, 0.27506110, 0.06816208, 0.82616585, 0.85912132, 0.88682729, 0.14730524, 0.61618829, 0.89891797, - 0.27753425, 1.68161881, 1.87394094, 1.94785213, 2.08572555, 1.90705216, 1.90777159, 1.23910809, 1.52017307, - 0.57438278, 0.33753166, 0.88768929, 0.35533753, 0.30193496, 0.81678063, 0.26569194, 0.62769043, 0.61990744, - 0.59077013, 2.02901411, 1.58923888, 2.17776394, 1.76309133, 1.74264824, 1.31485105, 1.71575761, 1.29775190, - 0.11058200, 0.97370809, 0.81339806, 0.57207322, 0.80417949, 0.54185718, 0.80831683, 0.29390740, 0.29051417, - 0.51964313, 1.66065478, 2.17192268, 1.86598253, 2.03193212, 1.52620018, 1.82728052, 1.46963060, 1.87916136, - 0.04341308, 0.05925354, 0.82397246, 0.55753845, 0.61247689, 0.98571628, 0.07566493, 0.37537411, 0.42080343, - 0.21715857, 1.56316149, 0.55312467, 1.59553123, 0.53537023, 1.64308119, 0.62742490, 1.31600118, 0.37510848, - 0.57869565, 0.55962265, 0.82500041, 0.60776925, 0.19367239, 0.88382334, 0.20328504, 0.58192456, 0.94542676, - 0.98562658, 1.33183134, 1.70965421, 1.70983100, 1.76660407, 1.46399045, 1.70318413, 0.83565855, 1.37921953, - 0.64355153, 0.69856495, 0.30377558, 0.02857198, 0.96969068, 0.48450547, 0.98341352, 0.03546083, 0.84963584, - 0.94460547, 1.60677671, 0.53308368, 1.60789728, 0.56227136, 1.50563633, 0.50456268, 1.49554634, 0.48299593, - 0.90907097, 0.22525074, 0.12530145, 0.52223104, 0.09549426, 0.93127102, 0.93429947, 0.01428344, 0.74249738, - 0.22606593, 1.59781134, 2.01703453, 1.58993423, 1.78536010, 1.21809304, 1.69219351, 1.24090374, 1.75499403, + 0.27759778f, 0.18458818f, 0.63114458f, 0.09953160f, 0.59739488f, 0.63917851f, 0.18828323f, 0.65625650f, + 0.84574437f, 0.91846281f, 1.90736914f, 1.45914197f, 2.30920029f, 1.94944119f, 2.12886763f, 1.64736962f, + 1.36378694f, 1.03263116f, 0.55102497f, 0.27506110f, 0.06816208f, 0.82616585f, 0.85912132f, 0.88682729f, + 0.14730524f, 0.61618829f, 0.89891797f, 0.27753425f, 1.68161881f, 1.87394094f, 1.94785213f, 2.08572555f, + 1.90705216f, 1.90777159f, 1.23910809f, 1.52017307f, 0.57438278f, 0.33753166f, 0.88768929f, 0.35533753f, + 0.30193496f, 0.81678063f, 0.26569194f, 0.62769043f, 0.61990744f, 0.59077013f, 2.02901411f, 1.58923888f, + 2.17776394f, 1.76309133f, 1.74264824f, 1.31485105f, 1.71575761f, 1.29775190f, 0.11058200f, 0.97370809f, + 0.81339806f, 0.57207322f, 0.80417949f, 0.54185718f, 0.80831683f, 0.29390740f, 0.29051417f, 0.51964313f, + 1.66065478f, 2.17192268f, 1.86598253f, 2.03193212f, 1.52620018f, 1.82728052f, 1.46963060f, 1.87916136f, + 0.04341308f, 0.05925354f, 0.82397246f, 0.55753845f, 0.61247689f, 0.98571628f, 0.07566493f, 0.37537411f, + 0.42080343f, 0.21715857f, 1.56316149f, 0.55312467f, 1.59553123f, 0.53537023f, 1.64308119f, 0.62742490f, + 1.31600118f, 0.37510848f, 0.57869565f, 0.55962265f, 0.82500041f, 0.60776925f, 0.19367239f, 0.88382334f, + 0.20328504f, 0.58192456f, 0.94542676f, 0.98562658f, 1.33183134f, 1.70965421f, 1.70983100f, 1.76660407f, + 1.46399045f, 1.70318413f, 0.83565855f, 1.37921953f, 0.64355153f, 0.69856495f, 0.30377558f, 0.02857198f, + 0.96969068f, 0.48450547f, 0.98341352f, 0.03546083f, 0.84963584f, 0.94460547f, 1.60677671f, 0.53308368f, + 1.60789728f, 0.56227136f, 1.50563633f, 0.50456268f, 1.49554634f, 0.48299593f, 0.90907097f, 0.22525074f, + 0.12530145f, 0.52223104f, 0.09549426f, 0.93127102f, 0.93429947f, 0.01428344f, 0.74249738f, 0.22606593f, + 1.59781134f, 2.01703453f, 1.58993423f, 1.78536010f, 1.21809304f, 1.69219351f, 1.24090374f, 1.75499403f, }; test_case.add_input(Shape{2, 4, 3}, input); @@ -1088,7 +1106,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention_dynamic_shapes) { test_case.add_input(Shape{2, 2, 2, 5, 2}, past); test_case.add_expected_output(Shape{2, 4, 4}, output); test_case.add_expected_output(Shape{2, 2, 2, 9, 2}, present); - test_case.run_with_tolerance_as_fp(1e-6); + test_case.run_with_tolerance_as_fp(1e-6f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_model_fusedgemm_abc) { @@ -1098,67 +1116,68 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_fusedgemm_abc) { auto test_case = test::TestCase(function, s_device); std::vector inputA = { - 0.760289272, - 0.155913759, - 0.781790674, - -0.916164881, - -0.599392663, - 0.264654594, - 0.793851873, - 0.177088557, - 0.082737454, - 0.070692121, - -0.811413035, - -0.098108588, - 0.650090827, - -0.987659751, - -0.815909968, - -0.375566031, - -0.192777789, - -0.843511765, + 0.760289272f, + 0.155913759f, + 0.781790674f, + -0.916164881f, + -0.599392663f, + 0.264654594f, + 0.793851873f, + 0.177088557f, + 0.082737454f, + 0.070692121f, + -0.811413035f, + -0.098108588f, + 0.650090827f, + -0.987659751f, + -0.815909968f, + -0.375566031f, + -0.192777789f, + -0.843511765f, }; std::vector inputB = { - -0.599338344, -0.893724541, -0.362130441, -0.510642812, -0.943908814, -0.247790266, -0.732624930, 0.660286910, - -0.264866660, -0.907203793, 0.339617010, -0.322529173, 0.714601048, 0.581729832, -0.609115490, -0.369882312, - -0.462432785, -0.554824440, -0.833489997, -0.899945507, -0.088337136, -0.253637339, -0.443307744, -0.677004897, + -0.599338344f, -0.893724541f, -0.362130441f, -0.510642812f, -0.943908814f, -0.247790266f, + -0.732624930f, 0.660286910f, -0.264866660f, -0.907203793f, 0.339617010f, -0.322529173f, + 0.714601048f, 0.581729832f, -0.609115490f, -0.369882312f, -0.462432785f, -0.554824440f, + -0.833489997f, -0.899945507f, -0.088337136f, -0.253637339f, -0.443307744f, -0.677004897f, }; std::vector inputC = { - -0.540039918, - -0.235745675, - -0.337291175, - -0.702340580, - 0.532629731, - -0.794515569, - -0.532012999, - 0.372558416, - 0.582367524, - -0.483044018, - 0.656635884, - -0.655929499, + -0.540039918f, + -0.235745675f, + -0.337291175f, + -0.702340580f, + 0.532629731f, + -0.794515569f, + -0.532012999f, + 0.372558416f, + 0.582367524f, + -0.483044018f, + 0.656635884f, + -0.655929499f, }; std::vector output = { - -8.75421E-05, - -9.65321E-05, - 0.239491309, - -2.70329E-05, - 0.151090653, - -5.53371E-05, - -1.22197E-05, - 0.413963711, - 0.618195780, - 0.011654445, - 0.815541101, - -2.46706E-05, + -8.75421E-05f, + -9.65321E-05f, + 0.239491309f, + -2.70329E-05f, + 0.151090653f, + -5.53371E-05f, + -1.22197E-05f, + 0.413963711f, + 0.618195780f, + 0.011654445f, + 0.815541101f, + -2.46706E-05f, }; test_case.add_input(Shape{3, 6}, inputA); test_case.add_input(Shape{6, 4}, inputB); test_case.add_input(Shape{3, 4}, inputC); test_case.add_expected_output(Shape{3, 4}, output); - test_case.run_with_tolerance_as_fp(1e-6); + test_case.run_with_tolerance_as_fp(1e-6f); } NGRAPH_TEST(${BACKEND_NAME}, onnx_com_microsoft_fused_conv_hard_sigmoid) { diff --git a/src/frontends/onnx/tests/onnx_import_const_folding.in.cpp b/src/frontends/onnx/tests/onnx_import_const_folding.in.cpp index 1c9a1054bf674d..a6c151d0ee526d 100644 --- a/src/frontends/onnx/tests/onnx_import_const_folding.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_const_folding.in.cpp @@ -56,7 +56,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_const_folding_model_scatter_elements) { SERIALIZED_ZOO, "onnx/scatter_elements_opset11.onnx")); - test_constant_folding(fn, {1.0, 1.1, 3.0, 2.1, 5.0}, Shape{1, 5}); + test_constant_folding(fn, {1.0f, 1.1f, 3.0f, 2.1f, 5.0f}, Shape{1, 5}); } NGRAPH_TEST(${BACKEND_NAME}, onnx_const_folding_model_non_zero_scalar) { diff --git a/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp b/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp index 0c1d0648d8abee..3cf38e223d07ed 100644 --- a/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp @@ -537,7 +537,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_branches_with_same_inputs) { auto test_case = test::TestCase(function, s_device); std::vector x(40, 2); std::vector y(40); - std::iota(y.begin(), y.end(), -20); + std::iota(y.begin(), y.end(), -20.f); // condition test_case.add_input({true}); @@ -577,7 +577,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_branches_with_different_inputs) { auto test_case = test::TestCase(function, s_device); std::vector x(40, 2); std::vector y(40); - std::iota(y.begin(), y.end(), -20); + std::iota(y.begin(), y.end(), -20.f); // condition test_case.add_input({true}); @@ -649,7 +649,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_inside_if) { // expected value == x * y std::vector x(40, 2); std::vector y(40); - std::iota(y.begin(), y.end(), -20); + std::iota(y.begin(), y.end(), -20.f); std::vector expected; std::transform(x.begin(), x.end(), y.begin(), std::back_inserter(expected), [](float i, float j) -> float { return i * j; @@ -663,7 +663,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_inside_if) { // case when condition == true and all(x < y) // expected value == x + y std::iota(x.begin(), x.end(), -static_cast(x.size())); - std::iota(y.begin(), y.end(), 1); + std::iota(y.begin(), y.end(), 1.f); std::transform(x.begin(), x.end(), y.begin(), expected.begin(), [](float i, float j) -> float { return i + j; }); @@ -703,13 +703,13 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_branches_with_multiple_outputs) { // case when condition == true so split is along axis 0 std::vector x(36); - std::iota(x.begin(), x.end(), 0); + std::iota(x.begin(), x.end(), 0.f); std::vector expected1(12); - std::iota(expected1.begin(), expected1.end(), 0); + std::iota(expected1.begin(), expected1.end(), 0.f); std::vector expected2(12); - std::iota(expected2.begin(), expected2.end(), 12); + std::iota(expected2.begin(), expected2.end(), 12.f); std::vector expected3(12); - std::iota(expected3.begin(), expected3.end(), 24); + std::iota(expected3.begin(), expected3.end(), 24.f); test_case.add_input({true}); // condition test_case.add_input(x); test_case.add_expected_output(expected1); @@ -768,9 +768,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_with_only_indentity_in_else_branch) { auto test_case = test::TestCase(function, s_device); std::vector x(shape_size(Shape{1, 5, 2, 2})); - std::iota(x.begin(), x.end(), 0); - std::vector expected{1.333333, 3, 4.666666, 6.333333, 8, 10, 12, 14, 16, 18, - 20, 22, 24, 26, 28, 30, 25.33333, 27, 28.666667, 30.33333}; + std::iota(x.begin(), x.end(), 0.f); + std::vector expected{1.333333f, 3.f, 4.666666f, 6.333333f, 8.f, 10.f, 12.f, + 14.f, 16.f, 18.f, 20.f, 22.f, 24.f, 26.f, + 28.f, 30.f, 25.33333f, 27.f, 28.666667f, 30.33333f}; test_case.add_input(x); test_case.add_expected_output(expected); test_case.run(); @@ -820,7 +821,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_if_dynamic_inputs) { auto test_case = test::TestCase(function, s_device); std::vector x(40, 2); std::vector y(40); - std::iota(y.begin(), y.end(), -20); + std::iota(y.begin(), y.end(), -20.f); std::vector expected; std::transform(x.begin(), x.end(), y.begin(), std::back_inserter(expected), [](float i, float j) -> float { return i + j; diff --git a/src/frontends/onnx/tests/onnx_import_dyn_shapes.in.cpp b/src/frontends/onnx/tests/onnx_import_dyn_shapes.in.cpp index 2afbdef3999159..dfa888bb943ce9 100644 --- a/src/frontends/onnx/tests/onnx_import_dyn_shapes.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_dyn_shapes.in.cpp @@ -220,7 +220,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_model_asinh_3_2) { auto test_case = test::TestCase(function, s_device); test_case.add_input(Shape{3, 2}, {-1.5f, 0.0f, 1.5f, -1.5f, 0.0f, 1.5f}); - test_case.add_expected_output(Shape{3, 2}, {-1.1947632f, 0.0f, 1.1947632f, -1.1947632, 0.0f, 1.1947632f}); + test_case.add_expected_output(Shape{3, 2}, {-1.1947632f, 0.0f, 1.1947632f, -1.1947632f, 0.0f, 1.1947632f}); test_case.run(); } @@ -713,7 +713,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_transpose) { const auto elems_in_tensor = shape_size(shape); std::vector input_values(elems_in_tensor); - std::iota(std::begin(input_values), std::end(input_values), 1); + std::iota(std::begin(input_values), std::end(input_values), 1.f); test_case.add_input(shape, input_values); @@ -729,9 +729,12 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_transpose) { namespace { Shape get_flattened_shape(const Shape& in_shape, size_t axis) { - size_t first_dim_size = - std::accumulate(begin(in_shape), next(begin(in_shape), axis), 1UL, std::multiplies()); - size_t last_dim_size = std::accumulate(next(begin(in_shape), axis), end(in_shape), 1UL, std::multiplies()); + size_t first_dim_size = std::accumulate(begin(in_shape), + next(begin(in_shape), axis), + static_cast(1), + std::multiplies()); + size_t last_dim_size = + std::accumulate(next(begin(in_shape), axis), end(in_shape), static_cast(1), std::multiplies()); return Shape{first_dim_size, last_dim_size}; } } // namespace @@ -751,7 +754,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_flatten_axis_0) { const auto elems_in_tensor = shape_size(shape); std::vector input_values(elems_in_tensor); - std::iota(input_values.begin(), input_values.end(), 1); + std::iota(input_values.begin(), input_values.end(), 1.f); test_case.add_input(shape, input_values); @@ -778,7 +781,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_flatten_axis) { const auto elems_in_tensor = shape_size(shape); std::vector input_values(elems_in_tensor); - std::iota(input_values.begin(), input_values.end(), 1); + std::iota(input_values.begin(), input_values.end(), 1.f); test_case.add_input(shape, input_values); @@ -805,7 +808,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_flatten_neg_axis) { const auto elems_in_tensor = shape_size(shape); std::vector input_values(elems_in_tensor); - std::iota(input_values.begin(), input_values.end(), 1); + std::iota(input_values.begin(), input_values.end(), 1.f); test_case.add_input(shape, input_values); @@ -907,7 +910,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_3d_input) { const Shape input_shape{3, 4, 1}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({0, 0}); test_case.add_input({2, 3}); @@ -926,7 +929,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_3d_input_neg_axes) { const Shape input_shape{3, 4, 1}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({0, 0}); test_case.add_input({2, 3}); @@ -945,7 +948,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_3d_input_12_axes) { const Shape input_shape{4, 3, 2}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({0, 0}); test_case.add_input({2, 1}); @@ -963,7 +966,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_3d_input_20_axes) { const Shape input_shape{4, 3, 2}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_shape, input_values); test_case.add_input({0, 1}); test_case.add_input({1, 3}); @@ -982,7 +985,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_4d_input_23_axes) { const Shape input_shape{2, 2, 2, 2}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({0, 0}); test_case.add_input({1, 1}); @@ -1000,7 +1003,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_4d_input_0231_axes_ends_ma const Shape input_shape{2, 2, 2, 2}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({0, 1, 1, 0}); test_case.add_input({std::numeric_limits::max(), @@ -1021,7 +1024,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_4d_input_2103_axes_ends_ma const Shape input_shape{2, 2, 2, 5}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({1, 0, 0, 1}); test_case.add_input({2, @@ -1043,7 +1046,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_4d_input_23_axes_21_steps) const Shape input_shape{2, 2, 6, 2}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({0, 1}); test_case.add_input({5, 2}); @@ -1060,7 +1063,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_10_default_axes) { const Shape input_shape{4, 3, 2}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_values); test_case.add_input({1, 1, 1}); test_case.add_input({2, 2, 2}); @@ -1116,34 +1119,34 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_model_softmax_axis_2) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/softmax_axis_2.onnx")); const std::vector input = { - 2.75793882, -0.50841322, 0.82013929, -0.62409912, -0.96136118, 0.21004745, 1.38337255, - 1.19030397, 2.0940445, -0.03551657, -0.78686039, 1.992782, 0.04300319, -0.29230777, - -0.56797112, -1.26732165, -0.61935399, 0.57670432, 0.92844898, 2.82469233, + 2.75793882f, -0.50841322f, 0.82013929f, -0.62409912f, -0.96136118f, 0.21004745f, 1.38337255f, + 1.19030397f, 2.0940445f, -0.03551657f, -0.78686039f, 1.992782f, 0.04300319f, -0.29230777f, + -0.56797112f, -1.26732165f, -0.61935399f, 0.57670432f, 0.92844898f, 2.82469233f, - 0.98721677, -0.05100663, -1.21178917, -0.17530157, 1.40051805, -0.13259761, -1.14313018, - 0.2673723, -0.87996154, 1.29053106, 1.55, 0.8396538, 1.20729817, 0.23727845, - -0.89113606, -1.70909842, 0.26460363, -0.70566808, 2.383518, 1.07024615, + 0.98721677f, -0.05100663f, -1.21178917f, -0.17530157f, 1.40051805f, -0.13259761f, -1.14313018f, + 0.2673723f, -0.87996154f, 1.29053106f, 1.55f, 0.8396538f, 1.20729817f, 0.23727845f, + -0.89113606f, -1.70909842f, 0.26460363f, -0.70566808f, 2.383518f, 1.07024615f, - -1.21722605, 0.82919357, 0.55765697, 0.12657686, 0.63432172, 0.75425957, -2.43721014, - -1.24478184, 2.65316853, 1.19509542, -0.95523998, 0.5149006, -0.01151649, 0.68327026, - -0.4589638, -0.46554745, 0.21055324, 0.39266729, 2.05098086, 1.83207919}; + -1.21722605f, 0.82919357f, 0.55765697f, 0.12657686f, 0.63432172f, 0.75425957f, -2.43721014f, + -1.24478184f, 2.65316853f, 1.19509542f, -0.95523998f, 0.5149006f, -0.01151649f, 0.68327026f, + -0.4589638f, -0.46554745f, 0.21055324f, 0.39266729f, 2.05098086f, 1.83207919f}; auto test_case = test::TestCase(function, s_device); test_case.add_input(input); test_case.add_expected_output( Shape{3, 4, 5}, - {0.80619486, 0.03075257, 0.1161086, 0.027393, 0.01955098, 0.07012682, 0.22670066, - 0.18689779, 0.4614171, 0.05485763, 0.04486172, 0.72286838, 0.10286818, 0.07356265, - 0.05583908, 0.01280724, 0.02448298, 0.08096658, 0.11509768, 0.76664552, + {0.80619486f, 0.03075257f, 0.1161086f, 0.027393f, 0.01955098f, 0.07012682f, 0.22670066f, + 0.18689779f, 0.4614171f, 0.05485763f, 0.04486172f, 0.72286838f, 0.10286818f, 0.07356265f, + 0.05583908f, 0.01280724f, 0.02448298f, 0.08096658f, 0.11509768f, 0.76664552f, - 0.30399806, 0.1076406, 0.03371745, 0.0950595, 0.4595844, 0.13369873, 0.04866969, - 0.19944906, 0.06332151, 0.55486101, 0.39101105, 0.19217177, 0.27755913, 0.10521588, - 0.03404216, 0.01150354, 0.08279411, 0.03137732, 0.68902071, 0.18530432, + 0.30399806f, 0.1076406f, 0.03371745f, 0.0950595f, 0.4595844f, 0.13369873f, 0.04866969f, + 0.19944906f, 0.06332151f, 0.55486101f, 0.39101105f, 0.19217177f, 0.27755913f, 0.10521588f, + 0.03404216f, 0.01150354f, 0.08279411f, 0.03137732f, 0.68902071f, 0.18530432f, - 0.0402528, 0.31156222, 0.23747503, 0.1543129, 0.25639705, 0.10627912, 0.00436928, - 0.01439711, 0.70979614, 0.16515835, 0.06798343, 0.2957175, 0.17468555, 0.34994439, - 0.11166912, 0.03615172, 0.07108136, 0.08527994, 0.44775794, 0.35972905}); + 0.0402528f, 0.31156222f, 0.23747503f, 0.1543129f, 0.25639705f, 0.10627912f, 0.00436928f, + 0.01439711f, 0.70979614f, 0.16515835f, 0.06798343f, 0.2957175f, 0.17468555f, 0.34994439f, + 0.11166912f, 0.03615172f, 0.07108136f, 0.08527994f, 0.44775794f, 0.35972905f}); test_case.run(3); } @@ -1184,7 +1187,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_instance_normalization_dyn_shape) { Shape data_shape{1, 2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); @@ -1207,7 +1210,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_instance_normalization_dyn_shape2) { Shape data_shape{1, 2, 3, 4}; std::vector data(shape_size(data_shape)); - std::iota(std::begin(data), std::end(data), 1); + std::iota(std::begin(data), std::end(data), 1.f); auto test_case = test::TestCase(function, s_device); @@ -1275,7 +1278,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_dyn_shapes_slice_1_3d_input_21_axes_ends_max) const Shape input_shape{1, 2, 3, 4}; std::vector input_values(shape_size(input_shape)); - std::iota(input_values.begin(), input_values.end(), 0); + std::iota(input_values.begin(), input_values.end(), 0.f); test_case.add_input(input_shape, input_values); test_case.add_expected_output(Shape{1, 1, 3, 3}, {13, 14, 15, 17, 18, 19, 21, 22, 23}); test_case.run(); @@ -1315,7 +1318,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_max_pool_dyn_rank_without_default_attrs) Shape input_shape{1, 1, 4, 4}; std::vector input(shape_size(input_shape)); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); test_case.add_input(input_shape, input); test_case.add_expected_output(Shape{1, 1, 3, 3}, {5, 6, 7, 9, 10, 11, 13, 14, 15}); test_case.run(); @@ -1327,7 +1330,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_depth_to_space_dynamic_input) { "onnx/dynamic_shapes/depth_to_space.onnx")); std::vector input(32); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); std::vector expected_output{0.f, 8.f, 1.f, 9.f, 16.f, 24.f, 17.f, 25.f, 2.f, 10.f, 3.f, 11.f, 18.f, 26.f, 19.f, 27.f, 4.f, 12.f, 5.f, 13.f, 20.f, 28.f, @@ -1345,7 +1348,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_space_to_depth_dynamic_input) { "onnx/dynamic_shapes/space_to_depth.onnx")); std::vector input(32); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); std::vector expected_output{ 0.f, 2.f, 8.f, 10.f, 16.f, 18.f, 24.f, 26.f, 1.f, 3.f, 9.f, 11.f, 17.f, 19.f, 25.f, 27.f, diff --git a/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp b/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp index fe85ef213db8ae..909eb4bb7bf2fb 100644 --- a/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_org_openvino.in.cpp @@ -57,14 +57,14 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_prior_box) { std::vector A(3 * 2 * 2); std::vector B(3 * 6 * 6); std::vector output = { - -2.3200002, -2.3200002, 3.6533334, 3.6533334, -3.7053659, -3.7053659, 5.0386992, 5.0386992, - -0.98666668, -2.3200002, 4.9866667, 3.6533334, -2.3720326, -3.7053659, 6.3720322, 5.0386992, - -2.3200002, -0.98666668, 3.6533334, 4.9866667, -3.7053659, -2.3720326, 5.0386992, 6.3720322, - -0.98666668, -0.98666668, 4.9866667, 4.9866667, -2.3720326, -2.3720326, 6.3720322, 6.3720322, - 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, - 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, - 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, - 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, + -2.3200002f, -2.3200002f, 3.6533334f, 3.6533334f, -3.7053659f, -3.7053659f, 5.0386992f, 5.0386992f, + -0.98666668f, -2.3200002f, 4.9866667f, 3.6533334f, -2.3720326f, -3.7053659f, 6.3720322f, 5.0386992f, + -2.3200002f, -0.98666668f, 3.6533334f, 4.9866667f, -3.7053659f, -2.3720326f, 5.0386992f, 6.3720322f, + -0.98666668f, -0.98666668f, 4.9866667f, 4.9866667f, -2.3720326f, -2.3720326f, 6.3720322f, 6.3720322f, + 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, + 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, + 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, + 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, }; test_case.add_input(A); test_case.add_input(B); @@ -78,11 +78,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_priorbox_clustered) { "onnx/priorbox_clustered.onnx")); auto test_case = test::TestCase(function, s_device); - std::vector A{15.0}; - std::vector B{10.0}; + std::vector A{15.0f}; + std::vector B{10.0f}; std::vector output = { - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.2, 0.2, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, 0.1f, 0.1f, 0.2f, 0.2f, }; test_case.add_input(A); test_case.add_input(B); @@ -101,22 +101,22 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_priorbox_clustered_most_attrs_default) { std::iota(std::begin(A), std::end(A), 0.0f); std::vector B(1 * 1 * 3 * 3); std::iota(std::begin(B), std::end(B), 0.0f); - std::vector output = {-0.1666666716337203979, - -0.1666666716337203979, - 0.1666666716337203979, - 0.1666666716337203979, - -0.1666666716337203979, - 0.3333333432674407959, - 0.1666666716337203979, - 0.6666666865348815918, - 0.1, - 0.1, - 0.2, - 0.2, - 0.1, - 0.1, - 0.2, - 0.2}; + std::vector output = {-0.1666666716337203979f, + -0.1666666716337203979f, + 0.1666666716337203979f, + 0.1666666716337203979f, + -0.1666666716337203979f, + 0.3333333432674407959f, + 0.1666666716337203979f, + 0.6666666865348815918f, + 0.1f, + 0.1f, + 0.2f, + 0.2f, + 0.1f, + 0.1f, + 0.2f, + 0.2f}; test_case.add_input(A); test_case.add_input(B); test_case.add_expected_output(Shape{1, 2, 8}, output); @@ -170,11 +170,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_detection_output) { std::vector logits = gen_vector(12, -2, 2); std::vector class_preds = gen_vector(9, 0, 1); std::vector proposals = gen_vector(12 * 2, 0, 1); - std::vector output = {0, 1, 0.777778, 0.279849, 0.283779, 0.562743, 0.695387, - 0, 1, 0.444444, 0.12963, 0.176075, 0.212963, 0.284573, - 0, 2, 0.888889, 0.279849, 0.283779, 0.562743, 0.695387, - 0, 2, 0.555556, 0.12963, 0.176075, 0.212963, 0.284573, - 0, 2, 0.222222, -0.0608094, -0.0142007, -0.0225239, 0.0304044}; + std::vector output = {0, 1, 0.777778f, 0.279849f, 0.283779f, 0.562743f, 0.695387f, + 0, 1, 0.444444f, 0.12963f, 0.176075f, 0.212963f, 0.284573f, + 0, 2, 0.888889f, 0.279849f, 0.283779f, 0.562743f, 0.695387f, + 0, 2, 0.555556f, 0.12963f, 0.176075f, 0.212963f, 0.284573f, + 0, 2, 0.222222f, -0.0608094f, -0.0142007f, -0.0225239f, 0.0304044f}; test_case.add_input(logits); test_case.add_input(class_preds); test_case.add_input(proposals); @@ -188,18 +188,18 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_group_norm) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/group_norm.onnx")); auto test_case = test::TestCase(function, s_device); Shape shape{2, 8, 2, 2}; - int size = shape_size(shape); + const auto size = shape_size(shape); std::vector data(size); - std::iota(data.begin(), data.end(), 0); + std::iota(data.begin(), data.end(), 0.f); std::vector output = { - -0.52752507, -0.09108937, 0.3453464, 0.78178215, 2.4364357, 3.309307, 4.1821785, 5.05505, - -1.5825753, -0.27326822, 1.0360391, 2.3453465, 4.8728714, 6.618614, 8.364357, 10.1101, - -2.6376252, -0.45544672, 1.726732, 3.9089108, 7.309307, 9.927921, 12.546536, 15.165151, - -3.6926756, -0.6376257, 2.4174247, 5.472475, 9.745743, 13.237228, 16.728714, 20.2202, - -0.52752507, -0.09108937, 0.3453464, 0.78178215, 2.4364357, 3.309307, 4.1821785, 5.05505, - -1.5825753, -0.27326822, 1.0360391, 2.3453465, 4.8728714, 6.618614, 8.364357, 10.1101, - -2.6376252, -0.45544672, 1.726732, 3.9089108, 7.309307, 9.927921, 12.546536, 15.165151, - -3.6926756, -0.6376257, 2.4174247, 5.472475, 9.745743, 13.237228, 16.728714, 20.2202, + -0.52752507f, -0.09108937f, 0.3453464f, 0.78178215f, 2.4364357f, 3.309307f, 4.1821785f, 5.05505f, + -1.5825753f, -0.27326822f, 1.0360391f, 2.3453465f, 4.8728714f, 6.618614f, 8.364357f, 10.1101f, + -2.6376252f, -0.45544672f, 1.726732f, 3.9089108f, 7.309307f, 9.927921f, 12.546536f, 15.165151f, + -3.6926756f, -0.6376257f, 2.4174247f, 5.472475f, 9.745743f, 13.237228f, 16.728714f, 20.2202f, + -0.52752507f, -0.09108937f, 0.3453464f, 0.78178215f, 2.4364357f, 3.309307f, 4.1821785f, 5.05505f, + -1.5825753f, -0.27326822f, 1.0360391f, 2.3453465f, 4.8728714f, 6.618614f, 8.364357f, 10.1101f, + -2.6376252f, -0.45544672f, 1.726732f, 3.9089108f, 7.309307f, 9.927921f, 12.546536f, 15.165151f, + -3.6926756f, -0.6376257f, 2.4174247f, 5.472475f, 9.745743f, 13.237228f, 16.728714f, 20.2202f, }; test_case.add_input(data); @@ -212,15 +212,16 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_group_norm_5d) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/group_norm_5d.onnx")); auto test_case = test::TestCase(function, s_device); Shape shape{2, 8, 1, 2, 1}; - int size = shape_size(shape); + const auto size = shape_size(shape); std::vector data(size); - std::iota(data.begin(), data.end(), 0); - std::vector output = { - -0.34163546562, 0.55278813838, 2.89442372322, 4.68327093124, -1.02490639686, 1.65836453437, 5.78884744644, - 9.36654186248, -1.70817732810, 2.76394081115, 8.68327140808, 14.04981231689, -2.39144825935, 3.86951708793, - 11.57769489288, 18.73308372497, -0.34163546562, 0.55278813838, 2.89442372322, 4.68327093124, -1.02490639686, - 1.65836453437, 5.78884744644, 9.36654186248, -1.70817732810, 2.76394081115, 8.68327140808, 14.04981231689, - -2.39144825935, 3.86951708793, 11.57769489288, 18.73308372497}; + std::iota(data.begin(), data.end(), 0.f); + std::vector output = {-0.34163546562f, 0.55278813838f, 2.89442372322f, 4.68327093124f, -1.02490639686f, + 1.65836453437f, 5.78884744644f, 9.36654186248f, -1.70817732810f, 2.76394081115f, + 8.68327140808f, 14.04981231689f, -2.39144825935f, 3.86951708793f, 11.57769489288f, + 18.73308372497f, -0.34163546562f, 0.55278813838f, 2.89442372322f, 4.68327093124f, + -1.02490639686f, 1.65836453437f, 5.78884744644f, 9.36654186248f, -1.70817732810f, + 2.76394081115f, 8.68327140808f, 14.04981231689f, -2.39144825935f, 3.86951708793f, + 11.57769489288f, 18.73308372497f}; test_case.add_input(data); test_case.add_expected_output(shape, output); @@ -232,20 +233,20 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_normalize) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/normalize.onnx")); auto test_case = test::TestCase(function, s_device); std::vector data(12); - std::iota(data.begin(), data.end(), 1); + std::iota(data.begin(), data.end(), 1.f); std::vector output = { - 0.19334731, - 0.33806169, - 0.44846106, - 0.53452247, - 1.4501048, - 1.5212777, - 1.5696137, - 1.6035674, - 3.4802516, - 3.3806169, - 3.2887144, - 3.2071347, + 0.19334731f, + 0.33806169f, + 0.44846106f, + 0.53452247f, + 1.4501048f, + 1.5212777f, + 1.5696137f, + 1.6035674f, + 3.4802516f, + 3.3806169f, + 3.2887144f, + 3.2071347f, }; test_case.add_input(data); test_case.add_expected_output(Shape{1, 3, 2, 2}, output); @@ -260,7 +261,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_swish_with_beta) { auto test_case = test::TestCase(function, s_device); std::vector input_data{-0.5f, 0, 0.5f}; test_case.add_input(input_data); - test_case.add_expected_output(expected_output_shape, {-0.2036667, 0.0, 0.2963333}); + test_case.add_expected_output(expected_output_shape, {-0.2036667f, 0.0f, 0.2963333f}); test_case.run_with_tolerance_as_fp(2.0e-5f); } @@ -274,7 +275,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_swish_without_beta) { auto test_case = test::TestCase(function, s_device); std::vector input_data{-0.5f, 0, 0.5f}; test_case.add_input(input_data); - test_case.add_expected_output(expected_output_shape, {-0.18877034, 0.0, 0.31122968}); + test_case.add_expected_output(expected_output_shape, {-0.18877034f, 0.0f, 0.31122968f}); test_case.run_with_tolerance_as_fp(2.0e-5f); } @@ -313,9 +314,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_experimental_detectron_detection_output) test_case.add_expected_output(Shape{5, 4}, { 0.8929862f, - 0.892986297607421875, - 12.10701370239257812, - 12.10701370239257812, + 0.892986297607421875f, + 12.10701370239257812f, + 12.10701370239257812f, 0, 0.0f, 0.0f, @@ -440,18 +441,18 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_experimental_detectron_group_norm) { auto test_case = test::TestCase(function, s_device); Shape shape{2, 8, 2, 2}; - int size = shape_size(shape); + const auto size = shape_size(shape); std::vector data(size); - std::iota(data.begin(), data.end(), 0); + std::iota(data.begin(), data.end(), 0.f); std::vector output = { - -0.52752507, -0.09108937, 0.3453464, 0.78178215, 2.4364357, 3.309307, 4.1821785, 5.05505, - -1.5825753, -0.27326822, 1.0360391, 2.3453465, 4.8728714, 6.618614, 8.364357, 10.1101, - -2.6376252, -0.45544672, 1.726732, 3.9089108, 7.309307, 9.927921, 12.546536, 15.165151, - -3.6926756, -0.6376257, 2.4174247, 5.472475, 9.745743, 13.237228, 16.728714, 20.2202, - -0.52752507, -0.09108937, 0.3453464, 0.78178215, 2.4364357, 3.309307, 4.1821785, 5.05505, - -1.5825753, -0.27326822, 1.0360391, 2.3453465, 4.8728714, 6.618614, 8.364357, 10.1101, - -2.6376252, -0.45544672, 1.726732, 3.9089108, 7.309307, 9.927921, 12.546536, 15.165151, - -3.6926756, -0.6376257, 2.4174247, 5.472475, 9.745743, 13.237228, 16.728714, 20.2202, + -0.52752507f, -0.09108937f, 0.3453464f, 0.78178215f, 2.4364357f, 3.309307f, 4.1821785f, 5.05505f, + -1.5825753f, -0.27326822f, 1.0360391f, 2.3453465f, 4.8728714f, 6.618614f, 8.364357f, 10.1101f, + -2.6376252f, -0.45544672f, 1.726732f, 3.9089108f, 7.309307f, 9.927921f, 12.546536f, 15.165151f, + -3.6926756f, -0.6376257f, 2.4174247f, 5.472475f, 9.745743f, 13.237228f, 16.728714f, 20.2202f, + -0.52752507f, -0.09108937f, 0.3453464f, 0.78178215f, 2.4364357f, 3.309307f, 4.1821785f, 5.05505f, + -1.5825753f, -0.27326822f, 1.0360391f, 2.3453465f, 4.8728714f, 6.618614f, 8.364357f, 10.1101f, + -2.6376252f, -0.45544672f, 1.726732f, 3.9089108f, 7.309307f, 9.927921f, 12.546536f, 15.165151f, + -3.6926756f, -0.6376257f, 2.4174247f, 5.472475f, 9.745743f, 13.237228f, 16.728714f, 20.2202f, }; test_case.add_input(data); @@ -468,13 +469,13 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_experimental_detectron_prior_grid_genera auto test_case = test::TestCase(function, s_device); std::vector priors(shape_size(Shape{3, 4})); - std::iota(priors.begin(), priors.end(), 0); + std::iota(priors.begin(), priors.end(), 0.f); std::vector feature_map(shape_size(Shape{1, 1, 1, 3})); - std::iota(feature_map.begin(), feature_map.end(), 0); + std::iota(feature_map.begin(), feature_map.end(), 0.f); std::vector im_data(shape_size(Shape{1, 3, 4, 7})); - std::iota(im_data.begin(), im_data.end(), 0); + std::iota(im_data.begin(), im_data.end(), 0.f); test_case.add_input(priors); test_case.add_input(feature_map); @@ -495,51 +496,51 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_experimental_detectron_roi_feature_extra auto test_case = test::TestCase(function, s_device); std::vector rois(shape_size(Shape{2, 4})); - std::iota(rois.begin(), rois.end(), 0); + std::iota(rois.begin(), rois.end(), 0.f); std::vector pyramid_layer_0(shape_size(Shape{1, 2, 2, 3})); - std::iota(pyramid_layer_0.begin(), pyramid_layer_0.end(), 0); + std::iota(pyramid_layer_0.begin(), pyramid_layer_0.end(), 0.f); test_case.add_input(rois); test_case.add_input(pyramid_layer_0); test_case.add_expected_output(Shape{2, 2, 3, 3}, - {1.416666746139526367, - 1.750000119209289551, - 2.083333492279052734, - 2.416666746139526367, - 2.75, - 3.083333492279052734, - 3.166666507720947266, - 3.5, - 3.833333492279052734, - 7.416666507720947266, - 7.75, - 8.083333015441894531, - 8.416666984558105469, - 8.75, - 9.083333969116210938, - 9.166666030883789062, - 9.5, - 9.833333969116210938, - 4.166666984558105469, - 4.5, - 4.833333492279052734, - 4.166666984558105469, - 4.5, - 4.833333492279052734, - 2.083333492279052734, - 2.25, - 2.416666746139526367, - 10.16666603088378906, - 10.5, - 10.83333206176757812, - 10.16666603088378906, - 10.5, - 10.83333206176757812, - 5.083333015441894531, - 5.25, - 5.416666507720947266}); + {1.416666746139526367f, + 1.750000119209289551f, + 2.083333492279052734f, + 2.416666746139526367f, + 2.75f, + 3.083333492279052734f, + 3.166666507720947266f, + 3.5f, + 3.833333492279052734f, + 7.416666507720947266f, + 7.75f, + 8.083333015441894531f, + 8.416666984558105469f, + 8.75f, + 9.083333969116210938f, + 9.166666030883789062f, + 9.5f, + 9.833333969116210938f, + 4.166666984558105469f, + 4.5f, + 4.833333492279052734f, + 4.166666984558105469f, + 4.5f, + 4.833333492279052734f, + 2.083333492279052734f, + 2.25f, + 2.416666746139526367f, + 10.16666603088378906f, + 10.5f, + 10.83333206176757812f, + 10.16666603088378906f, + 10.5f, + 10.83333206176757812f, + 5.083333015441894531f, + 5.25f, + 5.416666507720947266f}); test_case.add_expected_output(Shape{2, 4}, {0, 1, 2, 3, 4, 5, 6, 7}); test_case.run(); @@ -593,29 +594,32 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_generate_proposals) { // scores test_case.add_input( Shape{1, 3, 2, 6}, - {0.56637216, 0.90457034, 0.69827306, 0.4353543, 0.47985056, 0.42658508, 0.14516132, 0.08081771, 0.1799732, - 0.9229515, 0.42420176, 0.50857586, 0.82664067, 0.4972319, 0.3752427, 0.56731623, 0.18241242, 0.33252355, - 0.30608943, 0.6572437, 0.69185436, 0.88646156, 0.36985755, 0.5590753, 0.5256446, 0.03342898, 0.1344396, - 0.68642473, 0.37953874, 0.32575172, 0.21108444, 0.5661886, 0.45378175, 0.62126315, 0.26799858, 0.37272978}); + {0.56637216f, 0.90457034f, 0.69827306f, 0.4353543f, 0.47985056f, 0.42658508f, 0.14516132f, 0.08081771f, + 0.1799732f, 0.9229515f, 0.42420176f, 0.50857586f, 0.82664067f, 0.4972319f, 0.3752427f, 0.56731623f, + 0.18241242f, 0.33252355f, 0.30608943f, 0.6572437f, 0.69185436f, 0.88646156f, 0.36985755f, 0.5590753f, + 0.5256446f, 0.03342898f, 0.1344396f, 0.68642473f, 0.37953874f, 0.32575172f, 0.21108444f, 0.5661886f, + 0.45378175f, 0.62126315f, 0.26799858f, 0.37272978f}); // deltas test_case.add_input( Shape{1, 12, 2, 6}, - {0.5337073, 0.86607957, 0.55151343, 0.21626699, 0.4462629, 0.03985678, 0.5157072, 0.9932138, 0.7565954, - 0.43803605, 0.802818, 0.14834064, 0.53932905, 0.14314, 0.3817048, 0.95075196, 0.05516243, 0.2567484, - 0.25508744, 0.77438325, 0.43561, 0.2094628, 0.8299043, 0.44982538, 0.95615596, 0.5651084, 0.11801951, - 0.05352486, 0.9774733, 0.14439464, 0.62644225, 0.14370479, 0.54161614, 0.557915, 0.53102225, 0.0840179, - 0.7249888, 0.9843559, 0.5490522, 0.53788143, 0.822474, 0.3278008, 0.39688024, 0.3286012, 0.5117038, - 0.04743988, 0.9408995, 0.29885054, 0.81039643, 0.85277915, 0.06807619, 0.86430097, 0.36225632, 0.16606331, - 0.5401001, 0.7541649, 0.11998601, 0.5131829, 0.40606487, 0.327888, 0.27721855, 0.6378373, 0.22795396, - 0.4961256, 0.3215895, 0.15607187, 0.14782153, 0.8908137, 0.8835288, 0.834191, 0.29907143, 0.7983525, - 0.755875, 0.30837986, 0.0839176, 0.26624718, 0.04371626, 0.09472824, 0.20689541, 0.37622106, 0.1083321, - 0.1342548, 0.05815459, 0.7676379, 0.8105144, 0.92348766, 0.26761323, 0.7183306, 0.8947588, 0.19020908, - 0.42731014, 0.7473663, 0.85775334, 0.9340091, 0.3278848, 0.755993, 0.05307213, 0.39705503, 0.21003333, - 0.5625373, 0.66188884, 0.80521655, 0.6125863, 0.44678232, 0.97802377, 0.0204936, 0.02686367, 0.7390654, - 0.74631, 0.58399844, 0.5988792, 0.37413648, 0.5946692, 0.6955776, 0.36377597, 0.7891322, 0.40900692, - 0.99139464, 0.50169915, 0.41435778, 0.17142445, 0.26761186, 0.31591868, 0.14249913, 0.12919712, 0.5418711, - 0.6523203, 0.50259084, 0.7379765, 0.01171071, 0.94423133, 0.00841132, 0.97486794, 0.2921785, 0.7633071, - 0.88477814, 0.03563205, 0.50833166, 0.01354555, 0.535081, 0.41366324, 0.0694767, 0.9944055, 0.9981207}); + {0.5337073f, 0.86607957f, 0.55151343f, 0.21626699f, 0.4462629f, 0.03985678f, 0.5157072f, 0.9932138f, + 0.7565954f, 0.43803605f, 0.802818f, 0.14834064f, 0.53932905f, 0.14314f, 0.3817048f, 0.95075196f, + 0.05516243f, 0.2567484f, 0.25508744f, 0.77438325f, 0.43561f, 0.2094628f, 0.8299043f, 0.44982538f, + 0.95615596f, 0.5651084f, 0.11801951f, 0.05352486f, 0.9774733f, 0.14439464f, 0.62644225f, 0.14370479f, + 0.54161614f, 0.557915f, 0.53102225f, 0.0840179f, 0.7249888f, 0.9843559f, 0.5490522f, 0.53788143f, + 0.822474f, 0.3278008f, 0.39688024f, 0.3286012f, 0.5117038f, 0.04743988f, 0.9408995f, 0.29885054f, + 0.81039643f, 0.85277915f, 0.06807619f, 0.86430097f, 0.36225632f, 0.16606331f, 0.5401001f, 0.7541649f, + 0.11998601f, 0.5131829f, 0.40606487f, 0.327888f, 0.27721855f, 0.6378373f, 0.22795396f, 0.4961256f, + 0.3215895f, 0.15607187f, 0.14782153f, 0.8908137f, 0.8835288f, 0.834191f, 0.29907143f, 0.7983525f, + 0.755875f, 0.30837986f, 0.0839176f, 0.26624718f, 0.04371626f, 0.09472824f, 0.20689541f, 0.37622106f, + 0.1083321f, 0.1342548f, 0.05815459f, 0.7676379f, 0.8105144f, 0.92348766f, 0.26761323f, 0.7183306f, + 0.8947588f, 0.19020908f, 0.42731014f, 0.7473663f, 0.85775334f, 0.9340091f, 0.3278848f, 0.755993f, + 0.05307213f, 0.39705503f, 0.21003333f, 0.5625373f, 0.66188884f, 0.80521655f, 0.6125863f, 0.44678232f, + 0.97802377f, 0.0204936f, 0.02686367f, 0.7390654f, 0.74631f, 0.58399844f, 0.5988792f, 0.37413648f, + 0.5946692f, 0.6955776f, 0.36377597f, 0.7891322f, 0.40900692f, 0.99139464f, 0.50169915f, 0.41435778f, + 0.17142445f, 0.26761186f, 0.31591868f, 0.14249913f, 0.12919712f, 0.5418711f, 0.6523203f, 0.50259084f, + 0.7379765f, 0.01171071f, 0.94423133f, 0.00841132f, 0.97486794f, 0.2921785f, 0.7633071f, 0.88477814f, + 0.03563205f, 0.50833166f, 0.01354555f, 0.535081f, 0.41366324f, 0.0694767f, 0.9944055f, 0.9981207f}); // im_info test_case.add_input(Shape{1, 3}, {200, 200, 0}); // anchors @@ -623,11 +627,11 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_generate_proposals) { test_case.add_expected_output( Shape{6, 4}, - {0.12904608, 1.3703424, 3.6230984, 3.4675088, 0.9725206, 0., 4.4917974, 4.9623675, - 4.882682, 5.1236916, 7.1700497, 10.213073, 4.4913187, 4.305372, 8.750267, 8.803502, - 0.9777608, 1.0317986, 3.228293, 4.495021, 4.125554, 5.4091997, 6.35439, 10.124915}); + {0.12904608f, 1.3703424f, 3.6230984f, 3.4675088f, 0.9725206f, 0., 4.4917974f, 4.9623675f, + 4.882682f, 5.1236916f, 7.1700497f, 10.213073f, 4.4913187f, 4.305372f, 8.750267f, 8.803502f, + 0.9777608f, 1.0317986f, 3.228293f, 4.495021f, 4.125554f, 5.4091997f, 6.35439f, 10.124915f}); test_case.add_expected_output(Shape{6}, - {0.9229515, 0.90457034, 0.88646156, 0.82664067, 0.69827306, 0.69185436}); + {0.9229515f, 0.90457034f, 0.88646156f, 0.82664067f, 0.69827306f, 0.69185436f}); test_case.add_expected_output(Shape{1}, {6}); test_case.run(); } diff --git a/src/frontends/onnx/tests/onnx_import_org_pytorch.in.cpp b/src/frontends/onnx/tests/onnx_import_org_pytorch.in.cpp index 110fbe656614de..d57b9c6ad64b32 100644 --- a/src/frontends/onnx/tests/onnx_import_org_pytorch.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_org_pytorch.in.cpp @@ -34,23 +34,23 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_adaptive_avg_pooling2d_nchw) { "onnx/org.pytorch/adaptive_avg_pooling2d_nchw.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.9945, - 0.3466, - 0.2894, - 0.9318, - 0.0115, - 0.4867, - 0.7608, - 0.1550, - 0.8485, - 0.4971, - 0.8833, - 0.4579, - 0.3673, - 0.5410, - 0.2004, - 0.1519}); - test_case.add_expected_output(Shape{1, 1, 2, 2}, {0.4598249, 0.5342500, 0.5634750, 0.4233750}); + test_case.add_input({0.9945f, + 0.3466f, + 0.2894f, + 0.9318f, + 0.0115f, + 0.4867f, + 0.7608f, + 0.1550f, + 0.8485f, + 0.4971f, + 0.8833f, + 0.4579f, + 0.3673f, + 0.5410f, + 0.2004f, + 0.1519f}); + test_case.add_expected_output(Shape{1, 1, 2, 2}, {0.4598249f, 0.5342500f, 0.5634750f, 0.4233750f}); test_case.run(); } @@ -61,8 +61,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_adaptive_avg_pooling2d_chw) { "onnx/org.pytorch/adaptive_avg_pooling2d_chw.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({12.0, -1.0, -56.0, 20.0, 1.0, -8.0, 7.0, 9.0}); + test_case.add_input({12.0f, -1.0f, -56.0f, 20.0f, 1.0f, -8.0f, 7.0f, 9.0f}); - test_case.add_expected_output(Shape{1, 2, 2}, {5.5, -18.0, -3.5, 8.0}); + test_case.add_expected_output(Shape{1, 2, 2}, {5.5f, -18.0f, -3.5f, 8.0f}); test_case.run(); } diff --git a/src/frontends/onnx/tests/onnx_import_quant.in.cpp b/src/frontends/onnx/tests/onnx_import_quant.in.cpp index 784bb8a0ae2ad3..d49b96c22b7f91 100644 --- a/src/frontends/onnx/tests/onnx_import_quant.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_quant.in.cpp @@ -1061,7 +1061,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_fake_quantize_const_inputs_infer) { const Shape data_shape{1, 2, 3, 4}; const auto n_elements = shape_size(data_shape); std::vector input_data(n_elements); - std::iota(std::begin(input_data), std::end(input_data), 0); + std::iota(std::begin(input_data), std::end(input_data), 0.f); auto test_case = test::TestCase(function, s_device); test_case.add_input(input_data); @@ -1081,7 +1081,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_fake_quantize_nonconst_inputs_infer) { const Shape data_shape{1, 2, 3, 4}; const size_t n_elements = shape_size(data_shape); std::vector input_data(n_elements); - std::iota(std::begin(input_data), std::end(input_data), 0); + std::iota(std::begin(input_data), std::end(input_data), 0.f); auto test_case = test::TestCase(function, s_device); test_case.add_input(input_data); diff --git a/src/frontends/onnx/tests/onnx_import_reshape.in.cpp b/src/frontends/onnx/tests/onnx_import_reshape.in.cpp index 771000f8091db8..645ccd8cc4cea7 100644 --- a/src/frontends/onnx/tests/onnx_import_reshape.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_reshape.in.cpp @@ -129,29 +129,29 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_reshape_negative_dim) { "onnx/reshape_negative_dim.onnx")); // 2x3x4 - auto input = test::NDArray({{{0.5488135, 0.71518934, 0.60276335, 0.5448832}, - {0.4236548, 0.6458941, 0.4375872, 0.891773}, - {0.96366274, 0.3834415, 0.79172504, 0.5288949}}, + auto input = test::NDArray({{{0.5488135f, 0.71518934f, 0.60276335f, 0.5448832f}, + {0.4236548f, 0.6458941f, 0.4375872f, 0.891773f}, + {0.96366274f, 0.3834415f, 0.79172504f, 0.5288949f}}, - {{0.56804454, 0.92559665, 0.07103606, 0.0871293}, - {0.0202184, 0.83261985, 0.77815676, 0.87001216}, - {0.9786183, 0.7991586, 0.46147937, 0.7805292}}}) + {{0.56804454f, 0.92559665f, 0.07103606f, 0.0871293f}, + {0.0202184f, 0.83261985f, 0.77815676f, 0.87001216f}, + {0.9786183f, 0.7991586f, 0.46147937f, 0.7805292f}}}) .get_vector(); // 2x6x2 - auto expected_output = test::NDArray({{{0.5488135, 0.71518934}, - {0.60276335, 0.5448832}, - {0.4236548, 0.6458941}, - {0.4375872, 0.891773}, - {0.96366274, 0.3834415}, - {0.79172504, 0.5288949}}, - - {{0.56804454, 0.92559665}, - {0.07103606, 0.0871293}, - {0.0202184, 0.83261985}, - {0.77815676, 0.87001216}, - {0.9786183, 0.7991586}, - {0.46147937, 0.7805292}}}) + auto expected_output = test::NDArray({{{0.5488135f, 0.71518934f}, + {0.60276335f, 0.5448832f}, + {0.4236548f, 0.6458941f}, + {0.4375872f, 0.891773f}, + {0.96366274f, 0.3834415f}, + {0.79172504f, 0.5288949f}}, + + {{0.56804454f, 0.92559665f}, + {0.07103606f, 0.0871293f}, + {0.0202184f, 0.83261985f}, + {0.77815676f, 0.87001216f}, + {0.9786183f, 0.7991586f}, + {0.46147937f, 0.7805292f}}}) .get_vector(); auto test_case = test::TestCase(function, s_device); @@ -207,7 +207,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_depth_to_space) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/depth_to_space.onnx")); std::vector input(32); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); std::vector expected_output{0.f, 8.f, 1.f, 9.f, 16.f, 24.f, 17.f, 25.f, 2.f, 10.f, 3.f, 11.f, 18.f, 26.f, 19.f, 27.f, 4.f, 12.f, 5.f, 13.f, 20.f, 28.f, @@ -224,7 +224,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_depth_to_space_v1) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/depth_to_space_v1.onnx")); std::vector input(32); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); std::vector expected_output{0.f, 8.f, 1.f, 9.f, 16.f, 24.f, 17.f, 25.f, 2.f, 10.f, 3.f, 11.f, 18.f, 26.f, 19.f, 27.f, 4.f, 12.f, 5.f, 13.f, 20.f, 28.f, @@ -242,7 +242,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_depth_to_space_crd) { "onnx/depth_to_space_crd.onnx")); std::vector input(32); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); std::vector expected_output{0.f, 4.f, 1.f, 5.f, 8.f, 12.f, 9.f, 13.f, 2.f, 6.f, 3.f, 7.f, 10.f, 14.f, 11.f, 15.f, 16.f, 20.f, 17.f, 21.f, 24.f, 28.f, @@ -304,7 +304,7 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_space_to_depth) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/space_to_depth.onnx")); std::vector input(32); - std::iota(input.begin(), input.end(), 0); + std::iota(input.begin(), input.end(), 0.f); std::vector expected_output{ 0.f, 2.f, 8.f, 10.f, 16.f, 18.f, 24.f, 26.f, 1.f, 3.f, 9.f, 11.f, 17.f, 19.f, 25.f, 27.f, diff --git a/src/frontends/onnx/tests/onnx_import_rnn.in.cpp b/src/frontends/onnx/tests/onnx_import_rnn.in.cpp index fbe25a2b0b0227..06b28aa76a88e6 100644 --- a/src/frontends/onnx/tests/onnx_import_rnn.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_rnn.in.cpp @@ -39,12 +39,12 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_default_const) { "onnx/lstm_fwd_default_const.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.68172926, 1.1405563, -0.03931177, -0.03759607}); // X + test_case.add_input({0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f}); // X test_case.add_expected_output(Shape{2, 1, 1, 2}, - {-0.063373, -0.20347191, -0.07230289, -0.13298286}); // Y_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.07230289, -0.13298286}); // Y_h_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.1557954, -0.24502525}); // Y_c_data + {-0.063373f, -0.20347191f, -0.07230289f, -0.13298286f}); // Y_data + test_case.add_expected_output(Shape{1, 1, 2}, {-0.07230289f, -0.13298286f}); // Y_h_data + test_case.add_expected_output(Shape{1, 1, 2}, {-0.1557954f, -0.24502525f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -55,12 +55,12 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_reverse_const) { "onnx/lstm_reverse_const.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.68172926, 1.1405563, -0.03931177, -0.03759607}); // X + test_case.add_input({0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f}); // X test_case.add_expected_output(Shape{2, 1, 1, 2}, - {-0.06082131, -0.19985214, 0.00860566, 0.00920492}); // Y_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.06082131, -0.19985214}); // Y_h_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.25917438, -0.3832652}); // Y_c_data + {-0.06082131f, -0.19985214f, 0.00860566f, 0.00920492f}); // Y_data + test_case.add_expected_output(Shape{1, 1, 2}, {-0.06082131f, -0.19985214f}); // Y_h_data + test_case.add_expected_output(Shape{1, 1, 2}, {-0.25917438f, -0.3832652f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -70,21 +70,21 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_bidir_const) { file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/lstm_bidir_const.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.68172926, 1.1405563, -0.03931177, -0.03759607}); // X + test_case.add_input({0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f}); // X test_case.add_expected_output(Shape{2, 2, 1, 2}, - {-0.063373, - -0.20347191, - -0.06082131, - -0.19985214, - -0.07230289, - -0.13298286, - 0.00860566, - 0.00920492}); // Y_data + {-0.063373f, + -0.20347191f, + -0.06082131f, + -0.19985214f, + -0.07230289f, + -0.13298286f, + 0.00860566f, + 0.00920492f}); // Y_data test_case.add_expected_output(Shape{2, 1, 2}, - {-0.07230289, -0.13298286, -0.06082131, -0.19985214}); // Y_h_data + {-0.07230289f, -0.13298286f, -0.06082131f, -0.19985214f}); // Y_h_data test_case.add_expected_output(Shape{2, 1, 2}, - {-0.1557954, -0.24502525, -0.25917438, -0.3832652}); // Y_c_data + {-0.1557954f, -0.24502525f, -0.25917438f, -0.3832652f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -95,12 +95,12 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_with_clip_const) { "onnx/lstm_fwd_clip_const.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.68172926, 1.1405563, -0.03931177, -0.03759607}); // X + test_case.add_input({0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f}); // X test_case.add_expected_output(Shape{2, 1, 1, 2}, - {-0.02391884, -0.02744377, -0.01024176, -0.01188637}); // Y_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.01024176, -0.01188637}); // Y_h_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.02039271, -0.02353566}); // Y_c_data + {-0.02391884f, -0.02744377f, -0.01024176f, -0.01188637f}); // Y_data + test_case.add_expected_output(Shape{1, 1, 2}, {-0.01024176f, -0.01188637f}); // Y_h_data + test_case.add_expected_output(Shape{1, 1, 2}, {-0.02039271f, -0.02353566f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -111,27 +111,27 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_mixed_seq_const) { "onnx/lstm_fwd_mixed_seq_const.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.68172926, 1.1405563, -0.03931177, -0.03759607}); // X + test_case.add_input({0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f}); // X test_case.add_expected_output(Shape{2, 1, 2, 3}, - {0.13528088, - -0.1779867, - -0.07448981, - 0.14769037, - -0.16327181, - -0.10419653, + {0.13528088f, + -0.1779867f, + -0.07448981f, + 0.14769037f, + -0.16327181f, + -0.10419653f, 0., 0., 0., - 0.08759661, - -0.04002844, - -0.08617793}); // Y_data + 0.08759661f, + -0.04002844f, + -0.08617793f}); // Y_data test_case.add_expected_output( Shape{1, 2, 3}, - {0.13528088, -0.1779867, -0.07448981, 0.08759661, -0.04002844, -0.08617793}); // Y_h_data + {0.13528088f, -0.1779867f, -0.07448981f, 0.08759661f, -0.04002844f, -0.08617793f}); // Y_h_data test_case.add_expected_output( Shape{1, 2, 3}, - {0.367563, -0.43762812, -0.20435227, 0.17330585, -0.0732716, -0.18809439}); // Y_c_data + {0.367563f, -0.43762812f, -0.20435227f, 0.17330585f, -0.0732716f, -0.18809439f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -142,27 +142,27 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_reverse_mixed_seq_const) { "onnx/lstm_reverse_mixed_seq_const.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({0.68172926, 1.1405563, -0.03931177, -0.03759607}); // X + test_case.add_input({0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f}); // X test_case.add_expected_output(Shape{2, 1, 2, 3}, - {0.13528088, - -0.1779867, - -0.07448981, - 0.14696799, - -0.15571019, - -0.10270946, + {0.13528088f, + -0.1779867f, + -0.07448981f, + 0.14696799f, + -0.15571019f, + -0.10270946f, 0., 0., 0., - -0.01110403, - 0.0228607, - 0.00397353}); // Y_data + -0.01110403f, + 0.0228607f, + 0.00397353f}); // Y_data test_case.add_expected_output( Shape{1, 2, 3}, - {0.13528088, -0.1779867, -0.07448981, 0.14696799, -0.15571019, -0.10270946}); // Y_h_data + {0.13528088f, -0.1779867f, -0.07448981f, 0.14696799f, -0.15571019f, -0.10270946f}); // Y_h_data test_case.add_expected_output( Shape{1, 2, 3}, - {0.367563, -0.43762812, -0.20435227, 0.50598085, -0.42627674, -0.3641275}); // Y_c_data + {0.367563f, -0.43762812f, -0.20435227f, 0.50598085f, -0.42627674f, -0.3641275f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -174,43 +174,43 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_bidir_mixed_seq_const) { auto test_case = test::TestCase(function, s_device); test_case.add_input( - {0.68172926, 1.1405563, -0.03931177, -0.03759607, 1.1397027, 0.60444903, 1.3246384, -0.28191715}); // X + {0.68172926f, 1.1405563f, -0.03931177f, -0.03759607f, 1.1397027f, 0.60444903f, 1.3246384f, -0.28191715f}); // X test_case.add_expected_output(Shape{2, 2, 2, 2}, - {-0.063373, - -0.20347191, - 0.00860566, - 0.00920492, - -0.063373, - -0.20347191, - -0.12004475, - -0.12800421, + {-0.063373f, + -0.20347191f, + 0.00860566f, + 0.00920492f, + -0.063373f, + -0.20347191f, + -0.12004475f, + -0.12800421f, 0., 0., - -0.19095606, - -0.12459831, + -0.19095606f, + -0.12459831f, 0., 0., - -0.1911628, - -0.12813942}); // Y_data + -0.1911628f, + -0.12813942f}); // Y_data test_case.add_expected_output(Shape{2, 2, 2}, - {-0.063373, - -0.20347191, - -0.19095606, - -0.12459831, - -0.063373, - -0.20347191, - -0.12004475, - -0.12800421}); // Y_h_data + {-0.063373f, + -0.20347191f, + -0.19095606f, + -0.12459831f, + -0.063373f, + -0.20347191f, + -0.12004475f, + -0.12800421f}); // Y_h_data test_case.add_expected_output(Shape{2, 2, 2}, - {-0.2732999, - -0.38956356, - -0.48170844, - -0.34701264, - -0.2732999, - -0.38956356, - -0.27130172, - -0.253659}); // Y_c_data + {-0.2732999f, + -0.38956356f, + -0.48170844f, + -0.34701264f, + -0.2732999f, + -0.38956356f, + -0.27130172f, + -0.253659f}); // Y_c_data test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -221,8 +221,8 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_with_clip_peepholes) { "onnx/lstm_fwd_with_clip_peepholes.onnx")); auto test_case = test::TestCase(function, s_device); - test_case.add_input({-0.455351, -0.276391, -0.185934, -0.269585}); // X - test_case.add_input({-0.494659f, // W + test_case.add_input({-0.455351f, -0.276391f, -0.185934f, -0.269585f}); // X + test_case.add_input({-0.494659f, // W 0.0453352f, -0.487793f, 0.417264f, @@ -560,10 +560,10 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_lstm_dynamic_batch_size_and_seq_len) { test_case.add_input({1, 2, 3, 4, 5, 6}); test_case.add_expected_output(Shape{1, 1, 3, 2}, - {0.761594, 0.761594, 0.761594, 0.761594, 0.761594, 0.761594}); // Y + {0.761594f, 0.761594f, 0.761594f, 0.761594f, 0.761594f, 0.761594f}); // Y test_case.add_expected_output(Shape{1, 3, 2}, - {0.761594, 0.761594, 0.761594, 0.761594, 0.761594, 0.761594}); // Y_c - test_case.add_expected_output(Shape{1, 3, 2}, {1, 1, 1, 1, 1, 1}); // Y_h + {0.761594f, 0.761594f, 0.761594f, 0.761594f, 0.761594f, 0.761594f}); // Y_c + test_case.add_expected_output(Shape{1, 3, 2}, {1, 1, 1, 1, 1, 1}); // Y_h test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } @@ -769,32 +769,32 @@ NGRAPH_TEST_F(${BACKEND_NAME}, GRUSequenceOp, onnx_model_gru_fwd_activations_con // Y test_case.add_expected_output( Shape{4, 1, 3, 5}, - std::vector{0.30736187, 0.10271017, 0.91698503, 0.3471303, -0.0123809, 0.51264125, 0.51235366, - 0.45471948, 0.50601995, 0.49260828, 0.4781971, 0.0668709, 0.89421916, 0.33762455, - -0.19021586, 0.6881336, 0.7331965, 0.8887774, 0.34048334, 0.38408905, 0.49962956, - 0.2948451, 0.3651103, 0.33406913, 0.57418096, 0.49882296, 0.4321446, 0.97142136, - 0.20714557, 0.66270787, 0.53192705, 0.46424377, 0.9647801, 0.19583187, 0.7362316, - 0.48205143, -0.04748845, 0.27395952, 0.35897565, 0.5801568, 0.5889811, 0.36110958, - 1.3433081, 0.29702073, 0.5709667, 0.936689, 0.84129435, 1.1782551, 0.23925206, - 0.57521456, 0.43502977, -0.5664091, 0.6758457, 0.2958132, 0.70932186, 0.4411352, - -0.1717428, 1.7761463, 0.14413449, 0.73801273}); + std::vector{0.30736187f, 0.10271017f, 0.91698503f, 0.3471303f, -0.0123809f, 0.51264125f, 0.51235366f, + 0.45471948f, 0.50601995f, 0.49260828f, 0.4781971f, 0.0668709f, 0.89421916f, 0.33762455f, + -0.19021586f, 0.6881336f, 0.7331965f, 0.8887774f, 0.34048334f, 0.38408905f, 0.49962956f, + 0.2948451f, 0.3651103f, 0.33406913f, 0.57418096f, 0.49882296f, 0.4321446f, 0.97142136f, + 0.20714557f, 0.66270787f, 0.53192705f, 0.46424377f, 0.9647801f, 0.19583187f, 0.7362316f, + 0.48205143f, -0.04748845f, 0.27395952f, 0.35897565f, 0.5801568f, 0.5889811f, 0.36110958f, + 1.3433081f, 0.29702073f, 0.5709667f, 0.936689f, 0.84129435f, 1.1782551f, 0.23925206f, + 0.57521456f, 0.43502977f, -0.5664091f, 0.6758457f, 0.2958132f, 0.70932186f, 0.4411352f, + -0.1717428f, 1.7761463f, 0.14413449f, 0.73801273f}); // Y_h test_case.add_expected_output(Shape{1, 3, 5}, - std::vector{0.936689, - 0.84129435, - 1.1782551, - 0.23925206, - 0.57521456, - 0.43502977, - -0.5664091, - 0.6758457, - 0.2958132, - 0.70932186, - 0.4411352, - -0.1717428, - 1.7761463, - 0.14413449, - 0.73801273}); + std::vector{0.936689f, + 0.84129435f, + 1.1782551f, + 0.23925206f, + 0.57521456f, + 0.43502977f, + -0.5664091f, + 0.6758457f, + 0.2958132f, + 0.70932186f, + 0.4411352f, + -0.1717428f, + 1.7761463f, + 0.14413449f, + 0.73801273f}); test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 5); } @@ -908,32 +908,33 @@ NGRAPH_TEST_F(${BACKEND_NAME}, GRUSequenceOp, onnx_model_gru_fwd_mixed_seq_len_c // Y test_case.add_expected_output( Shape{4, 1, 3, 5}, - std::vector{-0.9559332, 0.4372494, 0.9967716, -0.9079381, -1.2538278, 1.9265908, -0.8437393, - -1.2057271, -0.25887525, -0.52679026, -0.3619178, 0.67928517, 0.9486744, -0.12006134, - -1.3862017, -0.98941356, 0.80389524, 0.97586197, -0.9343586, -0.74858856, 1.797039, - -0.7873732, -0.72469383, -0.5866635, -0.42103744, -0.8406298, 0.85877097, 0.6349921, - -0.55897295, -0.6168443, 0., 0., 0., 0., 0., - 1.577129, -0.6935871, -0.304804, -0.75392795, -0.20703818, -0.93796504, 0.9220495, - 0.36017662, -0.7007159, 0.06962098, 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., -0.96323603, - 0.9265786, 0.54976916, -0.8037839, 0.73501444}); + std::vector{-0.9559332f, 0.4372494f, 0.9967716f, -0.9079381f, -1.2538278f, 1.9265908f, + -0.8437393f, -1.2057271f, -0.25887525f, -0.52679026f, -0.3619178f, 0.67928517f, + 0.9486744f, -0.12006134f, -1.3862017f, -0.98941356f, 0.80389524f, 0.97586197f, + -0.9343586f, -0.74858856f, 1.797039f, -0.7873732f, -0.72469383f, -0.5866635f, + -0.42103744f, -0.8406298f, 0.85877097f, 0.6349921f, -0.55897295f, -0.6168443f, + 0., 0., 0., 0., 0., 1.577129f, + -0.6935871f, -0.304804f, -0.75392795f, -0.20703818f, -0.93796504f, 0.9220495f, + 0.36017662f, -0.7007159f, 0.06962098f, 0., 0., 0., + 0., 0., 0., 0., 0., 0., + 0., -0.96323603f, 0.9265786f, 0.54976916f, -0.8037839f, 0.73501444f}); // Y_h test_case.add_expected_output(Shape{1, 3, 5}, - std::vector{-0.98941356, - 0.80389524, - 0.97586197, - -0.9343586, - -0.74858856, - 1.577129, - -0.6935871, - -0.304804, - -0.75392795, - -0.20703818, - -0.96323603, - 0.9265786, - 0.54976916, - -0.8037839, - 0.73501444}); + std::vector{-0.98941356f, + 0.80389524f, + 0.97586197f, + -0.9343586f, + -0.74858856f, + 1.577129f, + -0.6935871f, + -0.304804f, + -0.75392795f, + -0.20703818f, + -0.96323603f, + 0.9265786f, + 0.54976916f, + -0.8037839f, + 0.73501444f}); test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 3); } @@ -949,32 +950,33 @@ NGRAPH_TEST_F(${BACKEND_NAME}, GRUSequenceOp, onnx_model_gru_reverse_mixed_seq_l // Y test_case.add_expected_output( Shape{4, 1, 3, 5}, - std::vector{-0.9917215, 0.07583051, 0.997975, -0.9315585, -0.7483002, 1.536813, -0.59922504, - -0.33637103, -0.7565539, -0.23930266, -0.7844553, 1.0393485, 0.73516595, -0.5616293, - -0.09489207, -0.9501128, 0.7905356, 0.9928266, -0.9153729, -1.1781745, 1.7955453, - -0.77754307, -0.6831806, -0.6266324, -0.39791372, -0.8030517, 1.3107346, 0.3700709, - -0.49808976, 0.52939236, 0., 0., 0., 0., 0., - 1.9345565, -0.83817405, -1.1433047, -0.35640514, -0.5191339, -0.655544, 1.3520991, - 0.42289692, -0.3171452, -0.3922639, 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., -0.24612205, - 1.6415757, 0.79883975, -0.18640287, -1.0134869}); + std::vector{-0.9917215f, 0.07583051f, 0.997975f, -0.9315585f, -0.7483002f, 1.536813f, + -0.59922504f, -0.33637103f, -0.7565539f, -0.23930266f, -0.7844553f, 1.0393485f, + 0.73516595f, -0.5616293f, -0.09489207f, -0.9501128f, 0.7905356f, 0.9928266f, + -0.9153729f, -1.1781745f, 1.7955453f, -0.77754307f, -0.6831806f, -0.6266324f, + -0.39791372f, -0.8030517f, 1.3107346f, 0.3700709f, -0.49808976f, 0.52939236f, + 0., 0., 0., 0., 0., 1.9345565f, + -0.83817405f, -1.1433047f, -0.35640514f, -0.5191339f, -0.655544f, 1.3520991f, + 0.42289692f, -0.3171452f, -0.3922639f, 0., 0., 0., + 0., 0., 0., 0., 0., 0., + 0., -0.24612205f, 1.6415757f, 0.79883975f, -0.18640287f, -1.0134869f}); // Y_h test_case.add_expected_output(Shape{1, 3, 5}, - std::vector{-0.9917215, - 0.07583051, - 0.997975, - -0.9315585, - -0.7483002, - 1.536813, - -0.59922504, - -0.33637103, - -0.7565539, - -0.23930266, - -0.7844553, - 1.0393485, - 0.73516595, - -0.5616293, - -0.09489207}); + std::vector{-0.9917215f, + 0.07583051f, + 0.997975f, + -0.9315585f, + -0.7483002f, + 1.536813f, + -0.59922504f, + -0.33637103f, + -0.7565539f, + -0.23930266f, + -0.7844553f, + 1.0393485f, + 0.73516595f, + -0.5616293f, + -0.09489207f}); test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 3); } @@ -991,29 +993,32 @@ NGRAPH_TEST_F(${BACKEND_NAME}, GRUSequenceOp, onnx_model_gru_bidir_mixed_seq_len test_case.add_expected_output( Shape{4, 2, 3, 5}, std::vector{ - -0.3224981, -0.44282594, 0.7499796, -0.12240417, 0.12079421, 0.02534254, 0.02504561, -0.0463777, - 0.01204535, -0.01497037, -0.04651929, -0.6264307, 0.7236632, 0.06250653, 0.02594197, 0.0595789, - 0.40258542, -0.40646964, 0.70320284, -0.02962421, 0.10372428, -0.38378227, -0.4331268, -0.15696645, - -0.3451503, 0.20918667, -0.59024405, -0.845524, 0.60705113, -0.6336088, -0.0833023, -0.40062034, - 0.7579466, -0.12340625, 0.04415433, -0.24662054, 0.27420586, -0.09122991, -0.22768986, 0.19980887, - -0.218649, -0.5560231, 0.56177044, -0.25098884, 0.15462328, 0.0409361, 0.17866893, -0.2782218, - 0.27396634, -0.04992082, 0.15353821, -0.4497267, -0.44631857, -0.478926, -0.23017275, 0.25369287, - -0.7369056, -0.73285, -0.5750758, -0.533177, 0., 0., 0., 0., - 0., -0.45753813, 0.5987347, -0.07046632, -0.35819566, 0.3916747, -0.18096107, -0.24415034, - 0.38435352, -0.29881003, 0.07738188, 0., 0., 0., 0., 0., - 0.10390212, -0.29646862, -0.20532897, -0.31521815, 0.01049522, 0.19370168, -0.6386781, -0.42919028, - -0.47081998, -0.2954276, 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., -0.50112087, -0.11085765, 0.5155622, -0.5635352, - 0.54762024, 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0.17058733, -0.6941011, -0.27862304, -0.27050856, -0.03864266}); + -0.3224981f, -0.44282594f, 0.7499796f, -0.12240417f, 0.12079421f, 0.02534254f, 0.02504561f, + -0.0463777f, 0.01204535f, -0.01497037f, -0.04651929f, -0.6264307f, 0.7236632f, 0.06250653f, + 0.02594197f, 0.0595789f, 0.40258542f, -0.40646964f, 0.70320284f, -0.02962421f, 0.10372428f, + -0.38378227f, -0.4331268f, -0.15696645f, -0.3451503f, 0.20918667f, -0.59024405f, -0.845524f, + 0.60705113f, -0.6336088f, -0.0833023f, -0.40062034f, 0.7579466f, -0.12340625f, 0.04415433f, + -0.24662054f, 0.27420586f, -0.09122991f, -0.22768986f, 0.19980887f, -0.218649f, -0.5560231f, + 0.56177044f, -0.25098884f, 0.15462328f, 0.0409361f, 0.17866893f, -0.2782218f, 0.27396634f, + -0.04992082f, 0.15353821f, -0.4497267f, -0.44631857f, -0.478926f, -0.23017275f, 0.25369287f, + -0.7369056f, -0.73285f, -0.5750758f, -0.533177f, 0., 0., 0., + 0., 0., -0.45753813f, 0.5987347f, -0.07046632f, -0.35819566f, 0.3916747f, + -0.18096107f, -0.24415034f, 0.38435352f, -0.29881003f, 0.07738188f, 0., 0., + 0., 0., 0., 0.10390212f, -0.29646862f, -0.20532897f, -0.31521815f, + 0.01049522f, 0.19370168f, -0.6386781f, -0.42919028f, -0.47081998f, -0.2954276f, 0., + 0., 0., 0., 0., 0., 0., 0., + 0., 0., -0.50112087f, -0.11085765f, 0.5155622f, -0.5635352f, 0.54762024f, + 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0.17058733f, -0.6941011f, -0.27862304f, -0.27050856f, + -0.03864266f}); // Y_h test_case.add_expected_output( Shape{2, 3, 5}, - std::vector{-0.0833023, -0.40062034, 0.7579466, -0.12340625, 0.04415433, -0.45753813, - 0.5987347, -0.07046632, -0.35819566, 0.3916747, -0.50112087, -0.11085765, - 0.5155622, -0.5635352, 0.54762024, 0.0595789, 0.40258542, -0.40646964, - 0.70320284, -0.02962421, 0.10372428, -0.38378227, -0.4331268, -0.15696645, - -0.3451503, 0.20918667, -0.59024405, -0.845524, 0.60705113, -0.6336088}); + std::vector{-0.0833023f, -0.40062034f, 0.7579466f, -0.12340625f, 0.04415433f, -0.45753813f, + 0.5987347f, -0.07046632f, -0.35819566f, 0.3916747f, -0.50112087f, -0.11085765f, + 0.5155622f, -0.5635352f, 0.54762024f, 0.0595789f, 0.40258542f, -0.40646964f, + 0.70320284f, -0.02962421f, 0.10372428f, -0.38378227f, -0.4331268f, -0.15696645f, + -0.3451503f, 0.20918667f, -0.59024405f, -0.845524f, 0.60705113f, -0.6336088f}); test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 4); } @@ -1901,32 +1906,32 @@ NGRAPH_TEST_F(${BACKEND_NAME}, RNNSequenceOp, onnx_model_rnn_reverse_mixed_seq_l // Y test_case.add_expected_output( Shape{4, 1, 3, 5}, - std::vector{-0.27398264, 0.96948624, 0.26404798, 0.8068119, 0.99935544, 0.73694086, 0.44305325, - -0.9964632, 0.7063714, 0.9999049, -0.7241098, 0.08538079, -0.785874, 0.60833323, - 0.99999666, 0.53703666, 0.0267657, 0.37151086, -0.68740594, 0.9992448, 0.3254757, - 0.7716811, -0.9996745, 0.9957807, 0.9995338, 0.9997339, 0.9888724, -0.8992324, - -0.797282, 0.98666525, 0., 0., 0., 0., 0., - 0.95711637, -0.8986079, -0.99998885, 0.96265936, 0.9380511, -0.86523867, 0.3528558, - -0.99675506, 0.946875, 0.79539406, 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0.99903, - 0.9998094, 0.9499353, 0.6077225, -0.9921822}); + std::vector{ + -0.27398264f, 0.96948624f, 0.26404798f, 0.8068119f, 0.99935544f, 0.73694086f, 0.44305325f, -0.9964632f, + 0.7063714f, 0.9999049f, -0.7241098f, 0.08538079f, -0.785874f, 0.60833323f, 0.99999666f, 0.53703666f, + 0.0267657f, 0.37151086f, -0.68740594f, 0.9992448f, 0.3254757f, 0.7716811f, -0.9996745f, 0.9957807f, + 0.9995338f, 0.9997339f, 0.9888724f, -0.8992324f, -0.797282f, 0.98666525f, 0., 0., + 0., 0., 0., 0.95711637f, -0.8986079f, -0.99998885f, 0.96265936f, 0.9380511f, + -0.86523867f, 0.3528558f, -0.99675506f, 0.946875f, 0.79539406f, 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.99903f, + 0.9998094f, 0.9499353f, 0.6077225f, -0.9921822f}); // Y_h test_case.add_expected_output(Shape{1, 3, 5}, - std::vector{-0.27398264, - 0.96948624, - 0.26404798, - 0.8068119, - 0.99935544, - 0.73694086, - 0.44305325, - -0.9964632, - 0.7063714, - 0.9999049, - -0.7241098, - 0.08538079, - -0.785874, - 0.60833323, - 0.99999666}); + std::vector{-0.27398264f, + 0.96948624f, + 0.26404798f, + 0.8068119f, + 0.99935544f, + 0.73694086f, + 0.44305325f, + -0.9964632f, + 0.7063714f, + 0.9999049f, + -0.7241098f, + 0.08538079f, + -0.785874f, + 0.60833323f, + 0.99999666f}); test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 4); } @@ -1942,29 +1947,32 @@ NGRAPH_TEST_F(${BACKEND_NAME}, RNNSequenceOp, onnx_model_rnn_bidir_mixed_seq_len test_case.add_expected_output( Shape{4, 2, 3, 5}, std::vector{ - 0.02254748, 0.15776646, -0.8229023, 0.19205809, 0.76984656, -0.00603169, -0.0286147, 0.04512155, - -0.0011912, -0.02572936, -0.13703543, -0.49651444, -0.78868157, 0.3566854, 0.8758509, -0.99602485, - -0.8151508, -0.5803147, 0.4985683, 0.30210292, 0.11550081, -0.30236644, 0.99622667, -0.8732492, - -0.43772405, -0.9284624, -0.5595875, 0.9986867, -0.18373811, 0.8451735, -0.43823165, -0.1904698, - 0.8320786, 0.9830735, 0.61861455, 0.19109797, 0.64407, 0.00962067, -0.32752877, -0.5050589, - -0.23455954, 0.9517933, 0.9050665, 0.91091585, -0.77941567, -0.71390504, -0.24422187, -0.38115412, - 0.3462553, 0.44084883, -0.81455964, -0.23556596, 0.85043025, -0.7840209, -0.82087713, -0.8349008, - -0.7880142, 0.99017143, -0.9816452, -0.93827677, 0., 0., 0., 0., - 0., 0.28117967, 0.20685148, 0.01166701, -0.5441828, -0.5463747, -0.85301256, 0.52109087, - -0.8317892, -0.9676957, -0.30258918, 0., 0., 0., 0., 0., - -0.7010546, -0.3106169, -0.04788882, -0.21822351, -0.33518708, -0.9073148, 0.16276085, 0.9518349, - -0.8635942, -0.92539954, 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0.9948462, -0.6242633, -0.19065344, -0.36072153, - -0.99407107, 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., -0.9957684, -0.7924, -0.40261805, -0.34061068, -0.55580306}); + 0.02254748f, 0.15776646f, -0.8229023f, 0.19205809f, 0.76984656f, -0.00603169f, -0.0286147f, + 0.04512155f, -0.0011912f, -0.02572936f, -0.13703543f, -0.49651444f, -0.78868157f, 0.3566854f, + 0.8758509f, -0.99602485f, -0.8151508f, -0.5803147f, 0.4985683f, 0.30210292f, 0.11550081f, + -0.30236644f, 0.99622667f, -0.8732492f, -0.43772405f, -0.9284624f, -0.5595875f, 0.9986867f, + -0.18373811f, 0.8451735f, -0.43823165f, -0.1904698f, 0.8320786f, 0.9830735f, 0.61861455f, + 0.19109797f, 0.64407f, 0.00962067f, -0.32752877f, -0.5050589f, -0.23455954f, 0.9517933f, + 0.9050665f, 0.91091585f, -0.77941567f, -0.71390504f, -0.24422187f, -0.38115412f, 0.3462553f, + 0.44084883f, -0.81455964f, -0.23556596f, 0.85043025f, -0.7840209f, -0.82087713f, -0.8349008f, + -0.7880142f, 0.99017143f, -0.9816452f, -0.93827677f, 0., 0., 0., + 0., 0., 0.28117967f, 0.20685148f, 0.01166701f, -0.5441828f, -0.5463747f, + -0.85301256f, 0.52109087f, -0.8317892f, -0.9676957f, -0.30258918f, 0., 0., + 0., 0., 0., -0.7010546f, -0.3106169f, -0.04788882f, -0.21822351f, + -0.33518708f, -0.9073148f, 0.16276085f, 0.9518349f, -0.8635942f, -0.92539954f, 0., + 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0.9948462f, -0.6242633f, -0.19065344f, -0.36072153f, -0.99407107f, + 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., -0.9957684f, -0.7924f, -0.40261805f, -0.34061068f, + -0.55580306f}); // Y_h test_case.add_expected_output( Shape{2, 3, 5}, - std::vector{-0.43823165, -0.1904698, 0.8320786, 0.9830735, 0.61861455, 0.28117967, - 0.20685148, 0.01166701, -0.5441828, -0.5463747, 0.9948462, -0.6242633, - -0.19065344, -0.36072153, -0.99407107, -0.99602485, -0.8151508, -0.5803147, - 0.4985683, 0.30210292, 0.11550081, -0.30236644, 0.99622667, -0.8732492, - -0.43772405, -0.9284624, -0.5595875, 0.9986867, -0.18373811, 0.8451735}); + std::vector{-0.43823165f, -0.1904698f, 0.8320786f, 0.9830735f, 0.61861455f, 0.28117967f, + 0.20685148f, 0.01166701f, -0.5441828f, -0.5463747f, 0.9948462f, -0.6242633f, + -0.19065344f, -0.36072153f, -0.99407107f, -0.99602485f, -0.8151508f, -0.5803147f, + 0.4985683f, 0.30210292f, 0.11550081f, -0.30236644f, 0.99622667f, -0.8732492f, + -0.43772405f, -0.9284624f, -0.5595875f, 0.9986867f, -0.18373811f, 0.8451735f}); // loosest match @ mantissa bit: // 16 or next bit (0.01166688557714223862 vs 0.01166701037436723709) diff --git a/src/frontends/onnx/tests/onnx_import_signal.in.cpp b/src/frontends/onnx/tests/onnx_import_signal.in.cpp new file mode 100644 index 00000000000000..af186a83f80894 --- /dev/null +++ b/src/frontends/onnx/tests/onnx_import_signal.in.cpp @@ -0,0 +1,1578 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +// clang-format off +#ifdef ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#define DEFAULT_FLOAT_TOLERANCE_BITS ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#endif +#ifdef ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#define DEFAULT_DOUBLE_TOLERANCE_BITS ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#endif +// clang-format on + +#include "common_test_utils/file_utils.hpp" +#include "default_opset.hpp" +#include "engines_util/test_case.hpp" +#include "engines_util/test_engines.hpp" +#include "onnx_import/onnx.hpp" +#include "util/test_control.hpp" + +NGRAPH_SUPPRESS_DEPRECATED_START + +using namespace ngraph; + +static std::string s_manifest = "${MANIFEST}"; +static std::string s_device = test::backend_name_to_device("${BACKEND_NAME}"); + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, + 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, + 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, + 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, + 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); + test_case.add_expected_output( + Shape{3, 5, 2}, + {10.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, -2.500000f, -0.812299f, + -2.500000f, -3.440955f, 35.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, + -2.500000f, -0.812299f, -2.500000f, -3.440955f, 60.000000f, 0.000000f, -2.500000f, 3.440955f, + -2.500000f, 0.812299f, -2.500000f, -0.812299f, -2.500000f, -3.440955f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_only_real) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft_only_real.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 5, 1}, + { + 0.000000f, + 1.000000f, + 2.000000f, + 3.000000f, + 4.000000f, + 5.000000f, + 6.000000f, + 7.000000f, + 8.000000f, + 9.000000f, + 10.000000f, + 11.000000f, + 12.000000f, + 13.000000f, + 14.000000f, + }); + test_case.add_expected_output( + Shape{3, 5, 2}, + {10.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, -2.500000f, -0.812299f, + -2.500000f, -3.440955f, 35.000000f, 0.000000f, -2.500000f, 3.440955f, -2.500000f, 0.812299f, + -2.500000f, -0.812299f, -2.500000f, -3.440955f, 60.000000f, 0.000000f, -2.500000f, 3.440955f, + -2.500000f, 0.812299f, -2.500000f, -0.812299f, -2.500000f, -3.440955f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_onesided) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft_onesided.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input( + Shape{2, 4}, + {0.000000f, 1.000000f, 2.000000f, 3.000000f, 4.000000f, 5.000000f, 6.000000f, 7.000000f}); + test_case.add_expected_output(Shape{2, 3, 2}, + {6.000000f, + 0.000000f, + -2.000000f, + 2.000000f, + -2.000000f, + 0.000000f, + 22.000000f, + 0.000000f, + -2.000000f, + 2.000000f, + -2.000000f, + 0.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_onesided_skip_convert_to_complex) { + auto function = + onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_onesided_skip_convert_to_complex.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input( + Shape{2, 4, 1}, + {0.000000f, 1.000000f, 2.000000f, 3.000000f, 4.000000f, 5.000000f, 6.000000f, 7.000000f}); + test_case.add_expected_output(Shape{2, 3, 1, 2}, + {6.000000f, + 0.000000f, + -2.000000f, + 2.000000f, + -2.000000f, + 0.000000f, + 22.000000f, + 0.000000f, + -2.000000f, + 2.000000f, + -2.000000f, + 0.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_length_provided) { + auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_lenght_provided.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, + 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, + 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, + 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, + 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); + test_case.add_expected_output( + Shape{1, 5, 2}, + {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, 3.000000f, 0.000000f, 4.000000f, 0.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_length_provided_onesided) { + auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_lenght_provided_onesided.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{4, 3}, + {0.000000f, + 1.000000f, + 2.000000f, + 3.000000f, + 4.000000f, + 5.000000f, + 6.000000f, + 7.000000f, + 8.000000f, + 9.000000f, + 10.000000f, + 11.000000f}); + test_case.add_expected_output(Shape{1, 3, 2}, + {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO, "onnx/dft_inverse.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, + 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, + 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, + 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, + 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); + test_case.add_expected_output( + Shape{3, 5, 2}, + {2.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, -0.500000f, 0.162460f, + -0.500000f, 0.688191f, 7.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, + -0.500000f, 0.162460f, -0.500000f, 0.688191f, 12.000000f, 0.000000f, -0.500000f, -0.688191f, + -0.500000f, -0.162460f, -0.500000f, 0.162460f, -0.500000f, 0.688191f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_only_real) { + auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_inverse_only_real.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 5, 1}, + {0.000000f, + 1.000000f, + 2.000000f, + 3.000000f, + 4.000000f, + 5.000000f, + 6.000000f, + 7.000000f, + 8.000000f, + 9.000000f, + 10.000000f, + 11.000000f, + 12.000000f, + 13.000000f, + 14.000000f}); + test_case.add_expected_output( + Shape{3, 5, 2}, + {2.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, -0.500000f, 0.162460f, + -0.500000f, 0.688191f, 7.000000f, 0.000000f, -0.500000f, -0.688191f, -0.500000f, -0.162460f, + -0.500000f, 0.162460f, -0.500000f, 0.688191f, 12.000000f, 0.000000f, -0.500000f, -0.688191f, + -0.500000f, -0.162460f, -0.500000f, 0.162460f, -0.500000f, 0.688191f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_onesided) { + auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_inverse_onesided.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{2, 3, 2}, + {6.000000f, + 0.000000f, + -2.000000f, + 2.000000f, + -2.000000f, + 0.000000f, + 22.000000f, + 0.000000f, + -2.000000f, + 2.000000f, + -2.000000f, + 0.000000f}); + test_case.add_expected_output( + Shape{2, 4}, + {0.000000f, 1.000000f, 2.000000f, 3.000000f, 4.000000f, 5.000000f, 6.000000f, 7.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_onesided_real_input) { + auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_inverse_onesided_real_input.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{2, 3, 1}, {1.000000f, 0.000000f, -1.000000f, 0.5000000f, -0.5000000f, 0.000000f}); + test_case.add_expected_output(Shape{2, 3, 1}, + {0.750000f, -0.250000f, -0.500000f, 0.250000f, 0.250000f, -0.500000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inversed_length_provided) { + auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_inversed_lenght_provided.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 5, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, + 3.000000f, 0.000000f, 4.000000f, 0.000000f, 5.000000f, 0.000000f, + 6.000000f, 0.000000f, 7.000000f, 0.000000f, 8.000000f, 0.000000f, + 9.000000f, 0.000000f, 10.000000f, 0.000000f, 11.000000f, 0.000000f, + 12.000000f, 0.000000f, 13.000000f, 0.000000f, 14.000000f, 0.000000f}); + test_case.add_expected_output( + Shape{1, 5, 2}, + {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f, 3.000000f, 0.000000f, 4.000000f, 0.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_dft_inverse_length_provided_onesided) { + auto function = + onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/dft_inverse_lenght_provided_onesided.onnx")); + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{1, 3, 2}, {0.000000f, 0.000000f, 1.000000f, 0.000000f, 2.000000f, 0.000000f}); + test_case.add_expected_output(Shape{4, 3}, + {0.000000f, + 1.000000f, + 2.000000f, + 3.000000f, + 4.000000f, + 5.000000f, + 6.000000f, + 7.000000f, + 8.000000f, + 9.000000f, + 10.000000f, + 11.000000f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_onesided_real_input_no_window_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_onesided_real_input_no_window_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{1, 128, 1}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_expected_output( + Shape{1, 15, 9, 2}, + {120.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, + -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, 248.0f, 0.0f, + -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, + -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, 376.0f, 0.0f, -8.0f, 40.218716f, + -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, + -8.0f, 1.5912971f, -8.0f, 0.0f, 504.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, + -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, + -8.0f, 0.0f, 632.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, + -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, + 760.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, + -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, 888.0f, 0.0f, + -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, + -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, 1016.0f, 0.0f, -8.0f, 40.218716f, + -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, + -8.0f, 1.5912971f, -8.0f, 0.0f, 1144.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, + -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, + -8.0f, 0.0f, 1272.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, + -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, + 1400.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, + -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, 1528.0f, 0.0f, + -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, + -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, 1656.0f, 0.0f, -8.0f, 40.218716f, + -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, + -8.0f, 1.5912971f, -8.0f, 0.0f, 1784.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, + -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, + -8.0f, 0.0f, 1912.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, + -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_no_onesided_real_input_no_window_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_no_onesided_real_input_no_window_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{1, 128, 1}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_expected_output( + Shape{1, 15, 16, 2}, + {120.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, + -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, + -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, + -8.0f, -40.218716f, 248.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, + -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, + -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, + -8.0f, -19.31371f, -8.0f, -40.218716f, 376.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, + -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, + -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, + -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, 504.0f, 0.0f, -8.0f, 40.218716f, + -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, + -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, + -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, 632.0f, 0.0f, + -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, + -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, + -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, + 760.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, + -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, + -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, + -8.0f, -40.218716f, 888.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, + -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, + -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, + -8.0f, -19.31371f, -8.0f, -40.218716f, 1016.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, + -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, + -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, + -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, 1144.0f, 0.0f, -8.0f, 40.218716f, + -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, + -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, + -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, 1272.0f, 0.0f, + -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, + -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, + -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, + 1400.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, + -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, + -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, + -8.0f, -40.218716f, 1528.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, + -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, + -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, + -8.0f, -19.31371f, -8.0f, -40.218716f, 1656.0f, 0.0f, -8.0f, 40.218716f, -8.0f, 19.31371f, + -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, -8.0f, 1.5912971f, + -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, -8.0f, -8.0f, + -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, 1784.0f, 0.0f, -8.0f, 40.218716f, + -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, -8.0f, 3.3137083f, + -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, -8.0f, -5.3454294f, + -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f, 1912.0f, 0.0f, + -8.0f, 40.218716f, -8.0f, 19.31371f, -8.0f, 11.972846f, -8.0f, 8.0f, -8.0f, 5.3454294f, + -8.0f, 3.3137083f, -8.0f, 1.5912971f, -8.0f, 0.0f, -8.0f, -1.5912971f, -8.0f, -3.3137083f, + -8.0f, -5.3454294f, -8.0f, -8.0f, -8.0f, -11.972846f, -8.0f, -19.31371f, -8.0f, -40.218716f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_no_onesided_complex_input_no_window_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_no_onesided_complex_input_no_window_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{1, 128, 2}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_expected_output(Shape{1, 15, 16, 2}, + {240.0f, + 256.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 496.0f, + 512.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 752.0f, + 768.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 1008.0f, + 1024.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 1264.0f, + 1280.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 1520.0f, + 1536.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 1776.0f, + 1792.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 2032.0f, + 2048.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 2288.0f, + 2304.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 2544.0f, + 2560.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 2800.0f, + 2816.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 3056.0f, + 3072.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 3312.0f, + 3328.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 3568.0f, + 3584.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f, + 3824.0f, + 3840.0f, + -96.43743187401357f, + 64.43743187401357f, + -54.62741699796952f, + 22.627416997969522f, + -39.94569220264782f, + 7.945692202647823f, + -32.0f, + 0.0f, + -26.69085820670878f, + -5.3091417932912215f, + -22.627416997969522f, + -9.372583002030478f, + -19.182597878074528f, + -12.817402121925468f, + -16.0f, + -16.0f, + -12.817402121925468f, + -19.182597878074528f, + -9.372583002030478f, + -22.627416997969522f, + -5.3091417932912215f, + -26.69085820670878f, + 0.0f, + -32.0f, + 7.945692202647823f, + -39.94569220264782f, + 22.627416997969522f, + -54.62741699796952f, + 64.43743187401357f, + -96.43743187401357f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_no_onesided_complex_input_no_window_default_length_2) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_no_onesided_complex_input_no_window_default_length_2.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{2, 64, 2}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_expected_output(Shape{2, 8, 8, 2}, + {56.0f, 64.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 184.0f, 192.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 312.0f, 320.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 440.0f, 448.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 568.0f, 576.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 696.0f, 704.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 824.0f, 832.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 952.0f, 960.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1080.0f, 1088.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1208.0f, 1216.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1336.0f, 1344.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1464.0f, 1472.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1592.0f, 1600.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1720.0f, 1728.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1848.0f, 1856.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f, -27.31370849898476f, + 1976.0f, 1984.0f, -27.31370849898476f, 11.313708498984761f, + -16.0f, 0.0f, -11.313708498984761f, -4.686291501015239f, + -8.0f, -8.0f, -4.686291501015239f, -11.313708498984761f, + 0.0f, -16.0f, 11.313708498984761f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_no_onesided_complex_input_no_window_no_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_no_onesided_complex_input_no_window_no_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{4, 32, 2}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_expected_output(Shape{4, 2, 24, 2}, + {552.0f, + 576.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 936.0f, + 960.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 2088.0f, + 2112.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 2472.0f, + 2496.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 3624.0f, + 3648.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 4008.0f, + 4032.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 5160.0f, + 5184.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f, + 5544.0f, + 5568.0f, + -206.29809870540362f, + 158.29809870540362f, + -113.56921938165306f, + 65.56921938165306f, + -81.94112549695429f, + 33.941125496954285f, + -65.56921938165306f, + 17.569219381653056f, + -55.277408948188935f, + 7.277408948188935f, + -48.0f, + 0.0f, + -42.41584771149505f, + -5.584152288504953f, + -37.856406460551014f, + -10.143593539448982f, + -33.941125496954285f, + -14.058874503045715f, + -30.430780618346944f, + -17.569219381653056f, + -27.159659942097512f, + -20.840340057902488f, + -24.0f, + -24.0f, + -20.840340057902495f, + -27.159659942097505f, + -17.569219381653056f, + -30.430780618346944f, + -14.058874503045715f, + -33.941125496954285f, + -10.143593539448982f, + -37.856406460551014f, + -5.584152288504953f, + -42.41584771149505f, + 0.0f, + -48.0f, + 7.277408948188942f, + -55.27740894818894f, + 17.569219381653056f, + -65.56921938165306f, + 33.941125496954285f, + -81.94112549695429f, + 65.56921938165306f, + -113.56921938165306f, + 158.29809870540362f, + -206.29809870540362f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_no_onesided_complex_input_given_window_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_no_onesided_complex_input_given_window_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{3, 32, 2}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_input(Shape{8}, + {1.0f, 0.8535616f, 0.5000232f, 0.1464712f, 0.0f, 0.1464057f, 0.49993038f, 0.853496f}); + test_case.add_expected_output( + Shape{3, 7, 8, 2}, + {23.9982088804245f, 27.99809694290161f, -3.6551388463010377f, 21.654558190325933f, -17.655848562717438f, + 1.6573804020881653f, -11.656437880922656f, -4.342796101262486f, -7.999694228172302f, -7.999675154685974f, + -4.342819217778247f, -11.656414517382817f, 1.6573339104652405f, -17.655802190303802f, 19.65439594500194f, + -1.6553475716806292f, 55.997313380241394f, 59.997201442718506f, 12.346159116290885f, 37.6528883872892f, + -17.65547662973404f, 1.6577513813972473f, -11.65625294879657f, -4.342609192943764f, -7.999541640281677f, + -7.999522566795349f, -4.342632309459525f, -11.656229585256732f, 1.6577048897743225f, -17.655430257320404f, + 35.65272614196521f, 14.345950390911293f, 87.99641788005829f, 91.9963059425354f, 28.347457078882808f, + 53.651218584252476f, -17.65510469675064f, 1.6581223607063293f, -11.656068016670485f, -4.342422284625041f, + -7.999389052391052f, -7.999369978904724f, -4.342445401140804f, -11.65604465313065f, 1.6580758690834045f, + -17.655058324337006f, 51.651056338928484f, 30.347248353503215f, 119.99552237987518f, 123.9954104423523f, + 44.34875504147473f, 69.64954878121574f, -17.654732763767242f, 1.6584933400154114f, -11.6558830845444f, + -4.34223537630632f, -7.999236464500427f, -7.999217391014099f, -4.342258492822079f, -11.65585972100456f, + 1.6584468483924866f, -17.654686391353607f, 67.64938653589175f, 46.34854631609514f, 151.99462687969208f, + 155.9945149421692f, 60.35005300406665f, 85.64787897817901f, -17.654360830783844f, 1.6588643193244934f, + -11.65569815241831f, -4.3420484679875955f, -7.999083876609802f, -7.999064803123474f, -4.342071584503358f, + -11.655674788878471f, 1.6588178277015686f, -17.65431445837021f, 83.64771673285502f, 62.34984427868706f, + 183.99373137950897f, 187.99361944198608f, 76.35135096665857f, 101.64620917514227f, -17.653988897800446f, + 1.6592352986335754f, -11.655513220292228f, -4.341861559668871f, -7.998931288719177f, -7.998912215232849f, + -4.341884676184634f, -11.65548985675239f, 1.6591888070106506f, -17.65394252538681f, 99.64604692981828f, + 78.35114224127898f, 215.99283587932587f, 219.99272394180298f, 92.3526489292505f, 117.64453937210556f, + -17.653616964817047f, 1.6596062779426575f, -11.655328288166146f, -4.341674651350154f, -7.998778700828552f, + -7.998759627342224f, -4.3416977678659165f, -11.655304924626307f, 1.6595597863197327f, -17.653570592403412f, + 115.64437712678156f, 94.35244020387091f, 279.99104487895966f, 283.99093294143677f, 124.35524485443435f, + 149.6411997660321f, -17.65287309885025f, 1.6603482365608215f, -11.654958423913968f, -4.341300834712712f, + -7.998473525047302f, -7.998454451560974f, -4.341323951228475f, -11.654935060374129f, 1.6603017449378967f, + -17.652826726436615f, 147.6410375207081f, 126.35503612905475f, 311.99014937877655f, 315.99003744125366f, + 140.35654281702625f, 165.63952996299537f, -17.652501165866852f, 1.6607192158699036f, -11.654773491787878f, + -4.341113926393987f, -7.998320937156677f, -7.998301863670349f, -4.34113704290975f, -11.65475012824804f, + 1.6606727242469788f, -17.652454793453217f, 163.63936771767135f, 142.35633409164666f, 343.98925387859344f, + 347.98914194107056f, 156.3578407796182f, 181.63786015995862f, -17.652129232883453f, 1.6610901951789856f, + -11.654588559661818f, -4.340927018075263f, -7.998168349266052f, -7.998149275779724f, -4.340950134591026f, + -11.654565196121965f, 1.6610437035560608f, -17.652082860469818f, 179.63769791463466f, 158.3576320542386f, + 375.98835837841034f, 379.98824644088745f, 172.35913874221012f, 197.63619035692193f, -17.651757299900055f, + 1.6614611744880676f, -11.654403627535729f, -4.340740109756538f, -7.998015761375427f, -7.997996687889099f, + -4.340763226272301f, -11.65438026399589f, 1.6614146828651428f, -17.65171092748642f, 195.6360281115979f, + 174.35893001683053f, 407.98746287822723f, 411.98735094070435f, 188.36043670480203f, 213.63452055388518f, + -17.651385366916656f, 1.6618321537971497f, -11.65421869540964f, -4.340553201437814f, -7.997863173484802f, + -7.997844099998474f, -4.340576317953577f, -11.6541953318698f, 1.6617856621742249f, -17.65133899450302f, + 211.6343583085612f, 190.36022797942243f, 439.9865673780441f, 443.98645544052124f, 204.36173466739393f, + 229.63285075084843f, -17.651013433933258f, 1.6622031331062317f, -11.65403376328355f, -4.340366293119089f, + -7.997710585594177f, -7.997691512107849f, -4.340389409634852f, -11.654010399743711f, 1.6621566414833069f, + -17.650967061519623f, 227.63268850552447f, 206.36152594201434f, 471.985671877861f, 475.98555994033813f, + 220.36303262998587f, 245.6311809478117f, -17.65064150094986f, 1.6625741124153137f, -11.653848831157461f, + -4.340179384800379f, -7.997557997703552f, -7.997538924217224f, -4.340202501316128f, -11.653825467617622f, + 1.662527620792389f, -17.650595128536224f, 243.63101870248772f, 222.36282390460627f, 535.9838808774948f, + 539.9837689399719f, 252.3656285551697f, 277.62784134173825f, -17.649897634983063f, 1.6633160710334778f, + -11.653478966905283f, -4.339805568162916f, -7.997252821922302f, -7.997233748435974f, -4.339828684678679f, + -11.653455603365444f, 1.663269579410553f, -17.649851262569427f, 275.6276790964142f, 254.3654198297901f, + 567.9829853773117f, 571.9828734397888f, 268.3669265177616f, 293.62617153870156f, -17.649525701999664f, + 1.6636870503425598f, -11.653294034779236f, -4.339618659844206f, -7.997100234031677f, -7.997081160545349f, + -4.339641776359969f, -11.653270671239369f, 1.663640558719635f, -17.64947932958603f, 291.6260092933776f, + 270.366717792382f, 599.9820898771286f, 603.9819779396057f, 284.36822448035355f, 309.6245017356648f, + -17.649153769016266f, 1.6640580296516418f, -11.653109102653133f, -4.339431751525495f, -7.996947646141052f, + -7.996928572654724f, -4.33945486804123f, -11.653085739113294f, 1.664011538028717f, -17.64910739660263f, + 307.62433949034084f, 286.36801575497395f, 631.9811943769455f, 635.9810824394226f, 300.3695224429455f, + 325.62283193262806f, -17.648781836032867f, 1.6644290089607239f, -11.652924170527058f, -4.339244843206757f, + -7.996795058250427f, -7.996775984764099f, -4.33926795972252f, -11.65290080698719f, 1.664382517337799f, + -17.648735463619232f, 323.6226696873041f, 302.3693137175659f, 663.9802988767624f, 667.9801869392395f, + 316.3708204055374f, 341.6211621295913f, -17.64840990304947f, 1.664799988269806f, -11.652739238400954f, + -4.3390579348880465f, -7.996642470359802f, -7.996623396873474f, -4.339081051403809f, -11.652715874861116f, + 1.664753496646881f, -17.648363530635834f, 339.62099988426735f, 318.3706116801578f, 695.9794033765793f, + 699.9792914390564f, 332.37211836812935f, 357.61949232655456f, -17.64803797006607f, 1.665170967578888f, + -11.65255430627488f, -4.338871026569308f, -7.996489882469177f, -7.996470808982849f, -4.338894143085071f, + -11.652530942735012f, 1.6651244759559631f, -17.647991597652435f, 355.6193300812306f, 334.37190964274976f, + 727.9785078763962f, 731.9783959388733f, 348.3734163307213f, 373.6178225235179f, -17.647666037082672f, + 1.66554194688797f, -11.652369374148776f, -4.3386841182505975f, -7.996337294578552f, -7.996318221092224f, + -4.33870723476636f, -11.652346010608937f, 1.6654954552650452f, -17.647619664669037f, 371.61766027819385f, + 350.3732076053417f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_no_onesided_complex_input_given_window_no_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_no_onesided_complex_input_given_window_no_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{2, 48, 2}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_input(Shape{10}, + {1.0f, + 0.90451396f, + 0.6545261f, + 0.345518f, + 0.095513284f, + 0.0f, + 0.095458746f, + 0.34542978f, + 0.654438f, + 0.9044596f}); + test_case.add_expected_output( + Shape{2, 4, 10, 2}, + {39.99714946746826f, 44.997006952762604f, -3.8268060472046876f, 36.32573598813075f, -26.390974611367387f, + 6.3932079238754085f, -17.885399480520576f, -2.113604756384621f, -13.440595214714953f, -6.558716539769538f, + -9.999701499938965f, -9.999686658382416f, -6.558733350911233f, -13.440578420114413f, -2.1136289848897007f, + -17.88537512225368f, 6.393153709525318f, -26.390919916754058f, 33.82553601255393f, -1.327069451110034f, + 159.99372911453247f, 164.9935865998268f, 56.17799336663901f, 96.31941429440245f, -26.389673466965206f, + 6.3945205945953365f, -17.884817996398652f, -2.113020157979113f, -13.440191747314223f, -6.558313469356559f, + -9.999345302581787f, -9.999330461025238f, -6.558330280498261f, -13.440174952713676f, -2.113044386484189f, + -17.88479363813176f, 6.394466380245246f, -26.38961877235188f, 93.81921431882563f, 58.677729962733665f, + 279.9903087615967f, 284.990166246891f, 116.1827927804827f, 156.31309260067414f, -26.38837232256303f, + 6.395833265315265f, -17.88423651227673f, -2.1124355595736013f, -13.4397882799135f, -6.557910398943584f, + -9.99898910522461f, -9.99897426366806f, -6.557927210085282f, -13.439771485312946f, -2.112459788078681f, + -17.88421215400984f, 6.3957790509651735f, -26.3883176279497f, 153.81289262509733f, 118.68252937657735f, + 399.9868884086609f, 404.98674589395523f, 176.18759219432638f, 216.30677090694581f, -26.38707117816086f, + 6.397145936035198f, -17.88365502815482f, -2.1118509611680913f, -13.43938481251277f, -6.557507328530605f, + -9.998632907867432f, -9.998618066310883f, -6.557524139672296f, -13.43936801791223f, -2.1118751896731798f, + -17.883630669887918f, 6.397091721685095f, -26.387016483547516f, 213.806570931369f, 178.68732879042108f, + 519.9834680557251f, 524.9833255410194f, 236.19239160817006f, 276.3004492132175f, -26.385770033758675f, + 6.398458606755116f, -17.8830735440329f, -2.1112663627625903f, -13.43898134511204f, -6.5571042581176044f, + -9.998276710510254f, -9.998261868953705f, -6.55712106925931f, -13.438964550511457f, -2.1112905912676663f, + -17.883049185766005f, 6.398404392405028f, -26.385715339145342f, 273.8002492376407f, 238.69212820426475f, + 639.9800477027893f, 644.9799051880836f, 296.1971910220138f, 336.2941275194893f, -26.384468889356498f, + 6.399771277475056f, -17.882492059910966f, -2.1106817643570555f, -13.438577877711282f, -6.556701187704647f, + -9.997920513153076f, -9.997905671596527f, -6.556717998846352f, -13.438561083110784f, -2.1107059928621457f, + -17.88246770164406f, 6.399717063124951f, -26.384414194743158f, 333.7939275439124f, 298.6969276181085f, + 759.9766273498535f, 764.9764848351479f, 356.2019904358574f, 396.28780582576087f, -26.38316774495431f, + 6.401083948194983f, -17.88191057578905f, -2.1100971659515597f, -13.438174410310552f, -6.556298117291675f, + -9.997564315795898f, -9.99754947423935f, -6.556314928433352f, -13.438157615710026f, -2.1101213944566375f, + -17.88188621752216f, 6.401029733844893f, -26.38311305034098f, 393.78760585018404f, 358.70172703195215f, + 879.9732069969177f, 884.9730644822121f, 416.2067898497012f, 456.2814841320326f, -26.38186660055215f, + 6.402396618914892f, -17.881329091667133f, -2.109512567546055f, -13.437770942909793f, -6.555895046878646f, + -9.99720811843872f, -9.997193276882172f, -6.55591185802038f, -13.437754148309267f, -2.1095367960511293f, + -17.881304733400242f, 6.402342404564804f, -26.381811905938818f, 453.78128415645574f, 418.7065264457958f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_onesided_real_input_given_window_no_default_length) { + auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_onesided_real_input_given_window_no_default_length.onnx")); + auto test_case = test::TestCase(function, s_device); + const Shape signal_shape{2, 32, 1}; + std::vector signal(ov::shape_size(signal_shape)); + std::iota(std::begin(signal), std::end(signal), 0.f); + test_case.add_input(signal_shape, signal); + test_case.add_input(Shape{16}, + {1.0f, + 0.96194196f, + 0.8535616f, + 0.6913578f, + 0.5000232f, + 0.308685f, + 0.1464712f, + 0.038075745f, + 0.0f, + 0.03804031f, + 0.1464057f, + 0.3085994f, + 0.49993038f, + 0.6912722f, + 0.853496f, + 0.96190655f}); + test_case.add_expected_output( + Shape{2, 3, 9, 2}, + {55.996273f, 0.0f, 23.999104f, 24.93398f, -7.99869f, 22.70421f, -7.999475f, 12.814739f, + -7.999694f, 8.329526f, -7.999781f, 5.5011215f, -7.999824f, 3.3910275f, -7.9998474f, 1.6240749f, + -7.999859f, 0.0f, 119.99441f, 0.0f, 55.998657f, 24.931015f, -7.9980354f, 22.704208f, + -7.9992137f, 12.814741f, -7.999542f, 8.329527f, -7.9996705f, 5.5011196f, -7.999737f, 3.3910284f, + -7.999769f, 1.6240768f, -7.9997787f, 0.0f, 183.99254f, 0.0f, 87.998215f, 24.928047f, + -7.9973803f, 22.70421f, -7.9989514f, 12.814744f, -7.9993896f, 8.329525f, -7.999561f, 5.5011206f, + -7.999648f, 3.3910275f, -7.9996986f, 1.6240759f, -7.99971f, 0.0f, 311.98883f, 0.0f, + 151.99731f, 24.922113f, -7.9960704f, 22.704214f, -7.998431f, 12.814743f, -7.9990845f, 8.329521f, + -7.999341f, 5.5011144f, -7.999474f, 3.3910275f, -7.999542f, 1.6240788f, -7.9995728f, 0.0f, + 375.98694f, 0.0f, 183.99686f, 24.919151f, -7.9954133f, 22.704203f, -7.9981623f, 12.814751f, + -7.998932f, 8.329529f, -7.9992285f, 5.501122f, -7.9993834f, 3.3910313f, -7.9994736f, 1.6240854f, + -7.9994965f, 0.0f, 439.98508f, 0.0f, 215.9964f, 24.916183f, -7.99476f, 22.704206f, + -7.9979024f, 12.814749f, -7.9987793f, 8.329529f, -7.999122f, 5.5011153f, -7.9992967f, 3.391035f, + -7.999382f, 1.6240807f, -7.99942f, 0.0f}); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_proper_exception_if_non_const_frame_step) { + try { + onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_non_const_frame_step.onnx")); + FAIL() << "Unknown error during STFT import"; + } catch (const std::runtime_error& exc) { + std::string msg{exc.what()}; + EXPECT_TRUE(msg.find("frame_step input must be a scalar or Shape{1} constant.") != std::string::npos); + } +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_proper_exception_if_dynamic_singal_shape) { + try { + onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_dynamic_signal_shape.onnx")); + FAIL() << "Unknown error during STFT import"; + } catch (const std::runtime_error& exc) { + std::string msg{exc.what()}; + EXPECT_TRUE(msg.find("Shape of signal input must be static with the rank equal to 3.") != std::string::npos); + } +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_proper_exception_if_non_const_frame_length) { + try { + onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_non_const_frame_length.onnx")); + FAIL() << "Unknown error during STFT import"; + } catch (const std::runtime_error& exc) { + std::string msg{exc.what()}; + EXPECT_TRUE(msg.find("frame_length input must be a scalar or Shape{1} constant.") != std::string::npos); + } +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_stft_proper_exception_if_complex_signal_and_onesided) { + try { + onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/stft_onesided_complex_input.onnx")); + FAIL() << "Unknown error during STFT import"; + } catch (const std::runtime_error& exc) { + std::string msg{exc.what()}; + EXPECT_TRUE(msg.find("If attribute onesided==1, signal input can NOT be complex.") != std::string::npos); + } +} diff --git a/src/frontends/onnx/tests/onnx_transformations.cpp b/src/frontends/onnx/tests/onnx_transformations.cpp index 76420f41e314c3..ca4535c921ff76 100644 --- a/src/frontends/onnx/tests/onnx_transformations.cpp +++ b/src/frontends/onnx/tests/onnx_transformations.cpp @@ -42,7 +42,7 @@ bool after_func_expand_name_comp(std::string lhs, std::string rhs) { if (is_hex_symbol(name[i])) { ++founded_hex; if (cut_begin == -1) { - cut_begin = i; + cut_begin = static_cast(i); } if (founded_hex >= min_address) { cut_length = founded_hex; diff --git a/src/frontends/onnx/tests/requirements.txt b/src/frontends/onnx/tests/requirements.txt index 6216064a61c146..e3d6ea192ad1de 100644 --- a/src/frontends/onnx/tests/requirements.txt +++ b/src/frontends/onnx/tests/requirements.txt @@ -1,4 +1,5 @@ # ONNX - generate test models -docopt~=0.6.2 -onnx==1.12.0 -protobuf>=3.18.1,<4.0.0 \ No newline at end of file +-c ../../../bindings/python/constraints.txt +protobuf>=3.18.1,<4.0.0 +docopt +onnx diff --git a/src/frontends/onnx/tests/runtime/ie/unit_test.manifest b/src/frontends/onnx/tests/runtime/ie/unit_test.manifest index 6f2b254f64d201..5a134066a6ce0d 100644 --- a/src/frontends/onnx/tests/runtime/ie/unit_test.manifest +++ b/src/frontends/onnx/tests/runtime/ie/unit_test.manifest @@ -433,3 +433,4 @@ IE_CPU.onnx_model_top_k_repeating_unsorted # Accuracy regression - Ticket 105909 IE_CPU.onnx_model_attention_qkv_hidden_sizes + diff --git a/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest b/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest index a1da514732a1a2..938e5870d0ea9a 100644 --- a/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest +++ b/src/frontends/onnx/tests/runtime/interpreter/unit_test.manifest @@ -3,9 +3,6 @@ INTERPRETER.onnx_top_k_opset_10 # Temporarily disabled: INTERPRETER.onnx_resize11_scales_nearest_asymmetric_floor_dynamic_sizes INTERPRETER.onnx_resize11_up_sizes_cubic_half_pixel_dynamic_sizes -INTERPRETER.onnx_resize10_up_scales_const_nearest -INTERPRETER.onnx_resize10_up_scales_const_linear -INTERPRETER.onnx_resize10_down_scales_const_nearest # Failed in MacOS: INTERPRETER.onnx_resize11_sizes_nearest_asymmetric_floor diff --git a/src/frontends/paddle/CMakeLists.txt b/src/frontends/paddle/CMakeLists.txt index 7cd24e56b75342..e11be311e03574 100644 --- a/src/frontends/paddle/CMakeLists.txt +++ b/src/frontends/paddle/CMakeLists.txt @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # - add_subdirectory(src) if(ENABLE_TESTS) diff --git a/src/frontends/paddle/include/openvino/frontend/paddle/extension/op.hpp b/src/frontends/paddle/include/openvino/frontend/paddle/extension/op.hpp new file mode 100644 index 00000000000000..68cea85c19cc44 --- /dev/null +++ b/src/frontends/paddle/include/openvino/frontend/paddle/extension/op.hpp @@ -0,0 +1,18 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "openvino/frontend/extension/op.hpp" +#include "openvino/frontend/paddle/extension/conversion.hpp" + +namespace ov { +namespace frontend { +namespace paddle { + +template +using OpExtension = ov::frontend::OpExtensionBase; + +} // namespace paddle +} // namespace frontend +} // namespace ov diff --git a/src/frontends/paddle/src/CMakeLists.txt b/src/frontends/paddle/src/CMakeLists.txt index 8ce804f87ce833..ec03bc7b4c741d 100644 --- a/src/frontends/paddle/src/CMakeLists.txt +++ b/src/frontends/paddle/src/CMakeLists.txt @@ -4,6 +4,8 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") ie_add_compiler_flags(/wd4305) +elseif(OV_COMPILER_IS_CLANG) + ie_add_compiler_flags(-Wno-delete-non-abstract-non-virtual-dtor) endif() ov_add_frontend(NAME paddle diff --git a/src/frontends/paddle/src/frontend.cpp b/src/frontends/paddle/src/frontend.cpp index 831bf33f583cf2..6568c0976b9a9b 100644 --- a/src/frontends/paddle/src/frontend.cpp +++ b/src/frontends/paddle/src/frontend.cpp @@ -350,8 +350,10 @@ void FrontEnd::fuse_fakequantize_ops(const std::vector>& } bool FrontEnd::supported_impl(const std::vector& variants) const { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; // FrontEnd can only load model specified by one path, one file or two files. - if (variants.empty() || variants.size() > 2) + if (variants.empty() || variants.size() > 2 + extra_variants_num) return false; // Validating first path, it must contain a model @@ -389,7 +391,9 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { } InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { - if (variants.size() == 1) { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + if (variants.size() == 1 + extra_variants_num) { // The case when folder with __model__ and weight files is provided or .pdmodel file if (variants[0].is()) { std::string m_path = variants[0].as(); @@ -407,7 +411,7 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const auto p_model_stream = variants[0].as(); return std::make_shared(std::vector{p_model_stream}, m_telemetry); } - } else if (variants.size() == 2) { + } else if (variants.size() == 2 + extra_variants_num) { // The case when .pdmodel and .pdparams files are provided std::ifstream model_stream; std::ifstream weights_stream; diff --git a/src/frontends/paddle/tests/op_extension.cpp b/src/frontends/paddle/tests/op_extension.cpp new file mode 100644 index 00000000000000..6d0efe03d4db78 --- /dev/null +++ b/src/frontends/paddle/tests/op_extension.cpp @@ -0,0 +1,181 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op_extension.hpp" + +#include "openvino/frontend/extension/op.hpp" +#include "openvino/frontend/paddle/extension/op.hpp" +#include "openvino/frontend/paddle/frontend.hpp" +#include "openvino/runtime/core.hpp" +#include "paddle_utils.hpp" +#include "so_extension.hpp" +#include "utils.hpp" + +using namespace ov::frontend; + +class Relu1 : public Relu { +public: + OPENVINO_OP("relu"); + OPENVINO_FRAMEWORK_MAP(paddle, {"X"}, {"Out"}) +}; + +class Relu2 : public Relu { +public: + OPENVINO_FRAMEWORK_MAP(paddle, {"X"}, {"Out"}, "CustomRelu_2") +}; + +class Relu3 : public Relu { +public: + OPENVINO_FRAMEWORK_MAP(paddle, + {"X"}, + {"Out"}, + "CustomRelu_3", + {{"ov_attribute_1", "fw_attribute_1"}, {"ov_attribute_2", "fw_attribute_2"}}) +}; + +class Relu4 : public Relu { +public: + OPENVINO_FRAMEWORK_MAP(paddle, + {"X"}, + {"Out"}, + "CustomRelu_4", + {{"ov_attribute_1", "fw_attribute_1"}, {"ov_attribute_2", "fw_attribute_2"}}, + { + {"ov_attribute_str", "string"}, + {"ov_attribute_int", 4}, + {"ov_attribute_bool", true}, + {"ov_attribute_float", 4.f}, + {"ov_attribute_vec_string", std::vector{"str1", "str2", "str3"}}, + {"ov_attribute_vec_int", std::vector{1, 2, 3, 4, 5, 6, 7}}, + {"ov_attribute_vec_bool", std::vector{true, false, true}}, + {"ov_attribute_vec_float", std::vector{1., 2., 3., 4., 5., 6., 7.}}, + }) +}; + +static OpExtensionFEParam getTestDataOpExtensionViaUserClass() { + OpExtensionFEParam res; + res.m_frontEndName = PADDLE_FE; + res.m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME); + res.m_modelName = "relu/relu.pdmodel"; + // use core OpExtension + res.m_extensions = std::vector>{std::make_shared>(), + std::make_shared>(), + std::make_shared>(), + std::make_shared>()}; + return res; +} + +static OpExtensionFEParam getTestDataOpExtensionViaPaddleConstructor() { + OpExtensionFEParam res; + res.m_frontEndName = PADDLE_FE; + res.m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME); + res.m_modelName = "relu/relu.pdmodel"; + // use ov::frontend::paddle OpExtension + res.m_extensions = std::vector>{ + std::make_shared>("CustomRelu_5", + std::vector{"X"}, + std::vector{"Out"}), + std::make_shared>("ov_CustomRelu_6", + "fw_CustomRelu_6", + std::vector{"X"}, + std::vector{"Out"}), + std::make_shared>( + "ov_CustomRelu_7", + "fw_CustomRelu_7", + std::vector{"X"}, + std::vector{"Out"}, + std::map{{"ov_attribute_1", "fw_attribute_1"}, + {"ov_attribute_2", "fw_attribute_2"}}), + std::make_shared>( + "ov_CustomRelu_8", + "fw_CustomRelu_8", + std::vector{"X"}, + std::vector{"Out"}, + std::map{{"ov_attribute_1", "fw_attribute_1"}, + {"ov_attribute_2", "fw_attribute_2"}}, + std::map{ + {"ov_attribute_str", "string"}, + {"ov_attribute_int", 4}, + {"ov_attribute_bool", true}, + {"ov_attribute_float", 4.f}, + {"ov_attribute_vec_string", std::vector{"str1", "str2", "str3"}}, + {"ov_attribute_vec_int", std::vector{1, 2, 3, 4, 5, 6, 7}}, + {"ov_attribute_vec_bool", std::vector{true, false, true}}, + {"ov_attribute_vec_float", std::vector{1., 2., 3., 4., 5., 6., 7.}}, + })}; + return res; +} + +static OpExtensionFEParam getTestDataOpExtensionViaCommonConstructor() { + OpExtensionFEParam res; + res.m_frontEndName = PADDLE_FE; + res.m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME); + res.m_modelName = "relu/relu.pdmodel"; + // use ov::frontend::OpExtension + res.m_extensions = std::vector>{ + std::make_shared>("CustomRelu_9", + std::vector{"X"}, + std::vector{"Out"}), + std::make_shared>("ov_CustomRelu_10", + "fw_CustomRelu_10", + std::vector{"X"}, + std::vector{"Out"}), + std::make_shared>( + "ov_CustomRelu_11", + "fw_CustomRelu_11", + std::vector{"X"}, + std::vector{"Out"}, + std::map{{"ov_attribute_1", "fw_attribute_1"}, + {"ov_attribute_2", "fw_attribute_2"}}), + std::make_shared>( + "ov_CustomRelu_12", + "fw_CustomRelu_12", + std::vector{"X"}, + std::vector{"Out"}, + std::map{{"ov_attribute_1", "fw_attribute_1"}, + {"ov_attribute_2", "fw_attribute_2"}}, + std::map{ + {"ov_attribute_str", "string"}, + {"ov_attribute_int", 4}, + {"ov_attribute_bool", true}, + {"ov_attribute_float", 4.f}, + {"ov_attribute_vec_string", std::vector{"str1", "str2", "str3"}}, + {"ov_attribute_vec_int", std::vector{1, 2, 3, 4, 5, 6, 7}}, + {"ov_attribute_vec_bool", std::vector{true, false, true}}, + {"ov_attribute_vec_float", std::vector{1., 2., 3., 4., 5., 6., 7.}}, + })}; + return res; +} + +INSTANTIATE_TEST_SUITE_P(PaddleOpExtensionTestViaUserClass, + FrontEndOpExtensionTest, + ::testing::Values(getTestDataOpExtensionViaUserClass()), + FrontEndOpExtensionTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(PaddleOpExtensionViaPaddleConstructor, + FrontEndOpExtensionTest, + ::testing::Values(getTestDataOpExtensionViaPaddleConstructor()), + FrontEndOpExtensionTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(PaddleOpExtensionViaCommonConstructor, + FrontEndOpExtensionTest, + ::testing::Values(getTestDataOpExtensionViaCommonConstructor()), + FrontEndOpExtensionTest::getTestCaseName); + +TEST(FrontEndOpExtensionTest, paddle_opextension_relu) { + ov::Core core; + const auto extensions = std::vector>{std::make_shared>()}; + core.add_extension(extensions); + std::string m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME); + std::string m_modelName = "relu/relu.pdmodel"; + auto model = core.read_model(FrontEndTestUtils::make_model_path(m_modelsPath + m_modelName)); + bool has_relu = false; + for (const auto& op : model->get_ops()) { + std::string name = op->get_type_info().name; + std::string version = op->get_type_info().version_id; + if (name == "relu" && version == "extension") + has_relu = true; + } + EXPECT_TRUE(has_relu); +} diff --git a/src/frontends/paddle/tests/op_fuzzy.cpp b/src/frontends/paddle/tests/op_fuzzy.cpp index 598b49dc41a548..f5d6479c9dddf8 100644 --- a/src/frontends/paddle/tests/op_fuzzy.cpp +++ b/src/frontends/paddle/tests/op_fuzzy.cpp @@ -4,8 +4,6 @@ #include "op_fuzzy.hpp" -#include - #include #include "engines_util/test_engines.hpp" diff --git a/src/frontends/paddle/tests/requirements.txt b/src/frontends/paddle/tests/requirements.txt index 8ad03b94a79985..8263f791981cd8 100644 --- a/src/frontends/paddle/tests/requirements.txt +++ b/src/frontends/paddle/tests/requirements.txt @@ -1,5 +1,6 @@ # PaddlePaddle - generate test models -paddlepaddle==2.4.2 +-c ../../../../src/bindings/python/constraints.txt numpy>=1.16.6,<1.25.0 -six~=1.16.0 protobuf>=3.18.1,<4.0.0 +six +paddlepaddle diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp index 12b8b95e4e1027..f20440b9d535bf 100644 --- a/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/decoder.hpp @@ -34,6 +34,9 @@ class TorchDecoder : public IDecoder { // Return debug name of the input tensor virtual const std::string& get_input_debug_name(size_t index) const = 0; + // Return signature name of the input tensor + virtual const std::string& get_input_signature_name(size_t index) const = 0; + // Return shape if inputs has torch::Tensor type in the original model, otherwise returns the shape [] of a scalar virtual PartialShape get_input_shape(size_t index) const = 0; diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/extension/conversion.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/extension/conversion.hpp new file mode 100644 index 00000000000000..a1b102cead4a2d --- /dev/null +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/extension/conversion.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/frontend/extension/conversion.hpp" +#include "openvino/frontend/frontend.hpp" +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/frontend/pytorch/visibility.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { + +class PYTORCH_API ConversionExtension : public ConversionExtensionBase { +public: + using Ptr = std::shared_ptr; + + ConversionExtension() = delete; + + ConversionExtension(const std::string& op_type, const ov::frontend::CreatorFunction& converter) + : ConversionExtensionBase(op_type), + m_converter(converter) {} + + const ov::frontend::CreatorFunction& get_converter() const { + return m_converter; + } + + ~ConversionExtension() override = default; + +private: + ov::frontend::CreatorFunction m_converter; +}; + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/extension/op.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/extension/op.hpp new file mode 100644 index 00000000000000..ee2c1ca44de8ad --- /dev/null +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/extension/op.hpp @@ -0,0 +1,18 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "openvino/frontend/extension/op.hpp" +#include "openvino/frontend/pytorch/extension/conversion.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { + +template +using OpExtension = ov::frontend::OpExtensionBase; + +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/frontend.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/frontend.hpp index 123f24fd4a16a1..f01975a135e4c2 100644 --- a/src/frontends/pytorch/include/openvino/frontend/pytorch/frontend.hpp +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/frontend.hpp @@ -4,6 +4,8 @@ #pragma once +#include "openvino/frontend/extension/conversion.hpp" +#include "openvino/frontend/extension/telemetry.hpp" #include "openvino/frontend/frontend.hpp" #include "openvino/frontend/pytorch/node_context.hpp" #include "openvino/frontend/pytorch/visibility.hpp" @@ -60,7 +62,9 @@ class PYTORCH_API FrontEnd : public ov::frontend::FrontEnd { bool supported_impl(const std::vector& variants) const override; ov::frontend::InputModel::Ptr load_impl(const std::vector& variants) const override; - std::map m_op_translators; + std::map m_op_translators; + std::vector m_conversion_extensions; + TelemetryExtension::Ptr m_telemetry; }; } // namespace pytorch diff --git a/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp b/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp index 41205130137c09..8502101bdfb810 100644 --- a/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp +++ b/src/frontends/pytorch/include/openvino/frontend/pytorch/node_context.hpp @@ -19,20 +19,22 @@ typedef std::unordered_map> TensorMap; class NodeContext : public frontend::NodeContext { public: NodeContext(std::shared_ptr decoder, - TensorMap* tensor_map, - ParameterVector* external_parameters, const TensorMap& ext_tensor_map, + std::shared_ptr tensor_map, + std::shared_ptr external_parameters, + std::shared_ptr> mutated_tensors, TranslateSession* translate_session) : frontend::NodeContext(decoder->get_op_type()), m_decoder(decoder), - m_tensor_map(tensor_map), m_ext_tensor_map(ext_tensor_map), + m_tensor_map(tensor_map), m_external_parameters(external_parameters), + m_mutated_tensors(mutated_tensors), m_translate_session(translate_session), m_decoder_inputs(decoder->inputs()), m_decoder_outputs(decoder->outputs()) { - FRONT_END_GENERAL_CHECK(tensor_map != nullptr && external_parameters != nullptr && - translate_session != nullptr); + FRONT_END_GENERAL_CHECK(m_tensor_map != nullptr && m_external_parameters != nullptr && + m_mutated_tensors != nullptr && m_translate_session != nullptr); } // Do not search for input in tensor map; try to access it as a constant of specified type T and return its value @@ -52,6 +54,8 @@ class NodeContext : public frontend::NodeContext { return m_tensor_map->at(input); } + Any get_values_from_const_input(int index) const override; + // TODO: upstream to base class OutputVector inputs() const { OutputVector res; @@ -106,11 +110,7 @@ class NodeContext : public frontend::NodeContext { "There is no any named attributes in PyTorch node, query by attribute name is not implemented"); } - void mutate_input(size_t index, Output ov_output); - - std::set get_mutated_tensors() const { - return m_mutated_tensors; - } + void mutate_input(size_t index, Output ov_output) const; std::shared_ptr get_decoder() const { return m_decoder; @@ -120,7 +120,7 @@ class NodeContext : public frontend::NodeContext { return m_translate_session; } - void add_tensor_to_context(size_t index, Output ov_output); + void add_tensor_to_context(size_t index, Output ov_output) const; Output get_tensor_from_model(size_t index) const { if (m_tensor_map->find(index) != m_tensor_map->end()) { @@ -130,22 +130,22 @@ class NodeContext : public frontend::NodeContext { } } - Output get_tensor_from_model_or_create_input(size_t index); + Output get_tensor_from_model_or_create_input(size_t index) const; Output get_input_from_visible_context(size_t index) const; - std::shared_ptr convert_subgraph(size_t index); + std::shared_ptr convert_subgraph(size_t index) const; private: std::shared_ptr m_decoder; - std::set m_mutated_tensors; - TensorMap* m_tensor_map; const TensorMap& m_ext_tensor_map; - ParameterVector* m_external_parameters; - TranslateSession* m_translate_session; + std::shared_ptr m_tensor_map; + std::shared_ptr m_external_parameters; + std::shared_ptr> m_mutated_tensors; + TranslateSession* m_translate_session = nullptr; const std::vector m_decoder_inputs; const std::vector m_decoder_outputs; }; -using PytorchCreatorFunction = std::function; +using CreatorFunction = std::function; } // namespace pytorch } // namespace frontend diff --git a/src/frontends/pytorch/src/frontend.cpp b/src/frontends/pytorch/src/frontend.cpp index be3c925a8e2457..042f7dcf1c7af0 100644 --- a/src/frontends/pytorch/src/frontend.cpp +++ b/src/frontends/pytorch/src/frontend.cpp @@ -6,10 +6,12 @@ #include "input_model.hpp" #include "op_table.hpp" +#include "openvino/frontend/pytorch/extension/conversion.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/util/log.hpp" #include "pt_framework_node.hpp" +#include "so_extension.hpp" #include "transformations/common_optimizations/push_constant_to_subgraph.hpp" #include "transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.hpp" #include "transformations/common_optimizations/reverse_shape_and_type_infer.hpp" @@ -62,6 +64,9 @@ std::shared_ptr FrontEnd::convert(const InputModel::Ptr& model) const { std::set unconverted_ops_types = get_unconverted_types_from_model(converted_model); std::stringstream ops_str; for (auto&& op_type : unconverted_ops_types) { + if (m_telemetry) { + m_telemetry->send_event("error_cause", "pytorch_" + op_type); + } ops_str << op_type << '\n'; } FRONT_END_OP_CONVERSION_CHECK(unconverted_ops_types.size() == 0, @@ -76,7 +81,7 @@ void FrontEnd::convert(const std::shared_ptr& partiallyConverted) const { std::shared_ptr FrontEnd::convert_partially(const ov::frontend::InputModel::Ptr& model) const { FRONT_END_GENERAL_CHECK(std::dynamic_pointer_cast(model), "Invalid input model"); try { - TranslateSession translate_session(model, m_op_translators); + TranslateSession translate_session(model, m_op_translators, m_telemetry); return translate_session.get_converted_model(); } catch (const std::runtime_error& e) { std::cerr << "[ ERROR ] Unexpected error while converting pytorch model: " << e.what() << '\n'; @@ -118,7 +123,7 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { // Usually if nn.Module.forward is given as a source model for conversion, there is the first Parameter // that represents original `self` argument in forward(self, ...). `self` shouldn't play any role in model - // inference if model is completelly frozed and all methods are inlined. So we check if it doesn't have any + // inference if model is completely frozen and all methods are inlined. So we check if it doesn't have any // consumers in the finally converted model and remove this parameter. This parameter should have index 0. if (model->get_parameters().size() > 0) { auto self = model->get_parameters()[0]; @@ -134,21 +139,38 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { } void FrontEnd::add_extension(const std::shared_ptr& extension) { - // Extension loading mechanism is not implemented, any extensions will be ignored - // see CVS-98766 for tracking progress - return; + if (auto conv_ext = std::dynamic_pointer_cast(extension)) { + m_conversion_extensions.push_back(conv_ext); + m_op_translators[conv_ext->get_op_type()] = [=](const NodeContext& context) { + return conv_ext->get_converter()(context); + }; + } else if (auto conv_ext = std::dynamic_pointer_cast(extension)) { + m_conversion_extensions.push_back(conv_ext); + m_op_translators[conv_ext->get_op_type()] = [=](const NodeContext& context) { + return conv_ext->get_converter()(context); + }; + } else if (const auto& so_ext = std::dynamic_pointer_cast(extension)) { + add_extension(so_ext->extension()); + m_extensions.push_back(so_ext); + } else if (const auto& telemetry = std::dynamic_pointer_cast(extension)) { + m_telemetry = telemetry; + } } bool FrontEnd::supported_impl(const std::vector& variants) const { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; // Currently PyTorch FrontEnd only support TorchDecoder as input - if (variants.size() != 1 || !variants[0].is>()) + if (variants.size() != 1 + extra_variants_num || !variants[0].is>()) return false; auto decoder = variants[0].as>(); return decoder && std::dynamic_pointer_cast(decoder); } ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { - FRONT_END_GENERAL_CHECK(variants.size() == 1, + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + FRONT_END_GENERAL_CHECK(variants.size() == 1 + extra_variants_num, "PyTorch Frontend supports exactly one parameter in model representation, got ", std::to_string(variants.size()), " instead."); diff --git a/src/frontends/pytorch/src/input_model.cpp b/src/frontends/pytorch/src/input_model.cpp index d273b091b8dce6..c03bcdcee0adfc 100644 --- a/src/frontends/pytorch/src/input_model.cpp +++ b/src/frontends/pytorch/src/input_model.cpp @@ -11,7 +11,7 @@ namespace ov { namespace frontend { namespace pytorch { -InputModel::InputModel(std::shared_ptr model_decoder) : m_model_decoder(model_decoder) { +InputModel::InputModel(const std::shared_ptr& model_decoder) : m_model_decoder(model_decoder) { const auto& inputs = m_model_decoder->inputs(); for (size_t i = 0; i < inputs.size(); ++i) { auto in_place = std::make_shared(*this, inputs[i]); @@ -45,7 +45,9 @@ std::vector InputModel::get_inputs() const { for (const auto& input_idx : m_model_decoder->inputs()) { auto place_it = m_name_to_place.find(std::to_string(input_idx)); FRONT_END_GENERAL_CHECK(place_it != m_name_to_place.end(), "Couldn't find Place for input."); - res.push_back(place_it->second); + if (input_idx != 0) { + res.push_back(place_it->second); + } } return res; } diff --git a/src/frontends/pytorch/src/input_model.hpp b/src/frontends/pytorch/src/input_model.hpp index 14ede27c23e61b..c4517129a4bf26 100644 --- a/src/frontends/pytorch/src/input_model.hpp +++ b/src/frontends/pytorch/src/input_model.hpp @@ -31,8 +31,7 @@ class InputModel : public ov::frontend::InputModel { friend class ::ov::frontend::pytorch::Place; public: - // TODO: pass telemetry extension to this ctor - explicit InputModel(std::shared_ptr model_decoder); + explicit InputModel(const std::shared_ptr& model_decoder); std::vector get_inputs() const override; std::vector get_outputs() const override; diff --git a/src/frontends/pytorch/src/node_context.cpp b/src/frontends/pytorch/src/node_context.cpp index 10ac4fc5d005e9..1eec75c3f25cfd 100644 --- a/src/frontends/pytorch/src/node_context.cpp +++ b/src/frontends/pytorch/src/node_context.cpp @@ -42,16 +42,19 @@ std::shared_ptr NodeContext::mark_node(std::shared_ptr ov_node) cons return m_decoder->mark_node(ov_node); } -void NodeContext::mutate_input(size_t index, Output ov_output) { +void NodeContext::mutate_input(size_t index, Output ov_output) const { FRONT_END_GENERAL_CHECK(!m_decoder->input_is_none(index), "Input is none with index: ", index); auto input_id = m_decoder_inputs.at(index); FRONT_END_GENERAL_CHECK(m_tensor_map->count(input_id), "No tensor corresponding input: ", input_id, " exist."); - m_translate_session->encode_tensor_name(ov_output, input_id, m_decoder->get_input_debug_name(index)); + m_translate_session->encode_tensor_name( + ov_output, + input_id, + {m_decoder->get_input_debug_name(index), m_decoder->get_input_signature_name(index)}); (*m_tensor_map)[input_id] = ov_output; - m_mutated_tensors.insert(input_id); + m_mutated_tensors->insert(input_id); } -void NodeContext::add_tensor_to_context(size_t index, Output ov_output) { +void NodeContext::add_tensor_to_context(size_t index, Output ov_output) const { if (m_tensor_map->count(index)) { OPENVINO_DEBUG << "[ WARNING ] Current context has tensor. Rewriting.\n"; } @@ -59,7 +62,7 @@ void NodeContext::add_tensor_to_context(size_t index, Output ov_output) { (*m_tensor_map)[index] = ov_output; } -Output NodeContext::get_tensor_from_model_or_create_input(size_t index) { +Output NodeContext::get_tensor_from_model_or_create_input(size_t index) const { if (m_tensor_map->find(index) != m_tensor_map->end()) { return m_tensor_map->at(index); } else { @@ -87,7 +90,7 @@ Output NodeContext::get_input_from_visible_context(size_t index) const { return input_tensor; } -std::shared_ptr NodeContext::convert_subgraph(size_t index) { +std::shared_ptr NodeContext::convert_subgraph(size_t index) const { auto subgraph_decoder = m_decoder->get_subgraph_decoder(index); // Extend external context with internal tensors except Parameter nodes, because internal Parameters are created to @@ -180,6 +183,70 @@ std::string NodeContext::const_input(size_t index) const { return input->get_decoder()->as_string(); } +namespace { +template +Any get_constant_data(const std::shared_ptr& constant) { + const T* ptr = reinterpret_cast(constant->get_data_ptr()); + const auto& shape = constant->get_shape(); + if (is_scalar(shape)) { + return ptr[0]; + } + return std::vector(ptr, ptr + shape_size(shape)); +} +} // namespace + +Any NodeContext::get_values_from_const_input(int index) const { + FRONT_END_GENERAL_CHECK(static_cast(index) < get_input_size(), + "Input with index: ", + index, + " does not exist."); + + if (input_is_none(index)) { + return {}; + } + + auto input_node = get_input_from_visible_context(index).get_node_shared_ptr(); + if (auto constant = as_type_ptr(input_node)) { + switch (constant->get_element_type()) { + case element::f32: + return get_constant_data(constant); + case element::f64: + return get_constant_data(constant); + case element::i32: + return get_constant_data(constant); + case element::u32: + return get_constant_data(constant); + case element::i64: + return get_constant_data(constant); + case element::u64: + return get_constant_data(constant); + case element::i8: + return get_constant_data(constant); + case element::u8: + return get_constant_data(constant); + case element::i16: + return get_constant_data(constant); + case element::u16: + return get_constant_data(constant); + default: + FRONT_END_GENERAL_CHECK(false, "Input with index: ", index, " has unsupported type."); + } + } else if (auto input = std::dynamic_pointer_cast(input_node)) { + const auto& attrs = input->get_attrs(); + if (attrs.find("none_value") != attrs.end()) { + return {}; + } + auto it = attrs.find("string_value"); + if (it != attrs.end()) { + return it->second; + } + } + + FRONT_END_GENERAL_CHECK(false, "Input node with index ", index, " cannot be interpreted as constant", input_node); + + return 0; +} + } // namespace pytorch } // namespace frontend } // namespace ov diff --git a/src/frontends/pytorch/src/op/adaptive_avg_pool3d.cpp b/src/frontends/pytorch/src/op/adaptive_avg_pool3d.cpp index 75d000ce1d7da8..42aa3da1d8586b 100644 --- a/src/frontends/pytorch/src/op/adaptive_avg_pool3d.cpp +++ b/src/frontends/pytorch/src/op/adaptive_avg_pool3d.cpp @@ -19,7 +19,7 @@ namespace op { using namespace ov::op; -OutputVector translate_adaptive_avg_pool3d(NodeContext& context) { +OutputVector translate_adaptive_avg_pool3d(const NodeContext& context) { num_inputs_check(context, 2, 2); auto const_tile_params = context.mark_node(v0::Constant::create(element::i32, Shape{5}, {1, 1, 1, 1, 1})); auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); diff --git a/src/frontends/pytorch/src/op/adaptive_max_pool2d.cpp b/src/frontends/pytorch/src/op/adaptive_max_pool2d.cpp index bbd72927fc6fac..5705fd22e70f47 100644 --- a/src/frontends/pytorch/src/op/adaptive_max_pool2d.cpp +++ b/src/frontends/pytorch/src/op/adaptive_max_pool2d.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_adaptive_max_pool2d(NodeContext& context) { +OutputVector translate_adaptive_max_pool2d(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/add.cpp b/src/frontends/pytorch/src/op/add.cpp index 3c4976efe2cdef..f0a997b6f8fa3a 100644 --- a/src/frontends/pytorch/src/op/add.cpp +++ b/src/frontends/pytorch/src/op/add.cpp @@ -15,7 +15,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_add(NodeContext& context) { +OutputVector translate_add(const NodeContext& context) { num_inputs_check(context, 2, 3); auto lhs = context.get_input(0); auto rhs = context.get_input(1); @@ -26,7 +26,7 @@ OutputVector translate_add(NodeContext& context) { // Case when two lists gets concatenated FRONT_END_OP_CONVERSION_CHECK(false, "aten::add is used for concatenation of lists, not possible to convert"); } - align_eltwise_input_types(context, lhs, rhs); + align_eltwise_input_types(context, lhs, rhs, true); if (!context.input_is_none(2)) { auto converted_alpha = context.mark_node(std::make_shared(context.get_input(2), rhs)); rhs = context.mark_node(std::make_shared(converted_alpha, rhs)); diff --git a/src/frontends/pytorch/src/op/addcmul.cpp b/src/frontends/pytorch/src/op/addcmul.cpp index 50d2cec4a3d045..7933acfde1d176 100644 --- a/src/frontends/pytorch/src/op/addcmul.cpp +++ b/src/frontends/pytorch/src/op/addcmul.cpp @@ -17,7 +17,7 @@ namespace op { using namespace ov::op; -OutputVector translate_addcmul(NodeContext& context) { +OutputVector translate_addcmul(const NodeContext& context) { num_inputs_check(context, 4, 4); const auto eltwise_mult = std::make_shared(context.get_input(1), context.get_input(2)); const auto value = context.get_input(3); diff --git a/src/frontends/pytorch/src/op/addmm.cpp b/src/frontends/pytorch/src/op/addmm.cpp index 4a2e16906b2b09..e8ba9b7e9159ab 100644 --- a/src/frontends/pytorch/src/op/addmm.cpp +++ b/src/frontends/pytorch/src/op/addmm.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_addmm(NodeContext& context) { +OutputVector translate_addmm(const NodeContext& context) { num_inputs_check(context, 5, 5); auto input = context.get_input(0); auto m1 = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/arange.cpp b/src/frontends/pytorch/src/op/arange.cpp index bacd8d113bc3c3..c238cd12bbc394 100644 --- a/src/frontends/pytorch/src/op/arange.cpp +++ b/src/frontends/pytorch/src/op/arange.cpp @@ -17,7 +17,7 @@ namespace op { using namespace ov::op; -OutputVector translate_arange(NodeContext& context) { +OutputVector translate_arange(const NodeContext& context) { auto zero = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); auto one = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); int dtype_port = -1; diff --git a/src/frontends/pytorch/src/op/as_tensor.cpp b/src/frontends/pytorch/src/op/as_tensor.cpp index 114f4e6fe6ad34..ae2c15d0a1eba9 100644 --- a/src/frontends/pytorch/src/op/as_tensor.cpp +++ b/src/frontends/pytorch/src/op/as_tensor.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_as_tensor(NodeContext& context) { +OutputVector translate_as_tensor(const NodeContext& context) { // aten::tensor(t[] data, *, ScalarType? dtype=None, Device? device=None, bool requires_grad=False) -> Tensor num_inputs_check(context, 1, 4); auto dtype = element::f32; diff --git a/src/frontends/pytorch/src/op/avg_poolnd.cpp b/src/frontends/pytorch/src/op/avg_poolnd.cpp index bb1d16b99df2cb..77f35a0569e76b 100644 --- a/src/frontends/pytorch/src/op/avg_poolnd.cpp +++ b/src/frontends/pytorch/src/op/avg_poolnd.cpp @@ -18,7 +18,7 @@ namespace op { using namespace ov::op; -OutputVector translate_avg_poolnd(NodeContext& context) { +OutputVector translate_avg_poolnd(const NodeContext& context) { num_inputs_check(context, 6, 7); auto input = context.get_input(0); auto kernel = context.const_input(1); diff --git a/src/frontends/pytorch/src/op/batch_norm.cpp b/src/frontends/pytorch/src/op/batch_norm.cpp index 495295e212e213..a306dd21832e82 100644 --- a/src/frontends/pytorch/src/op/batch_norm.cpp +++ b/src/frontends/pytorch/src/op/batch_norm.cpp @@ -32,7 +32,7 @@ Output broadcast_const_to_channel_dim(const NodeContext& context, } } // namespace -OutputVector translate_batch_norm(NodeContext& context) { +OutputVector translate_batch_norm(const NodeContext& context) { // Schema: aten::batch_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, // bool training, float momentum, float eps, bool cudnn_enabled) -> Tensor num_inputs_check(context, 8, 9); diff --git a/src/frontends/pytorch/src/op/bitwise_not.cpp b/src/frontends/pytorch/src/op/bitwise_not.cpp index dcb953990e0d82..55ba1203b80a04 100644 --- a/src/frontends/pytorch/src/op/bitwise_not.cpp +++ b/src/frontends/pytorch/src/op/bitwise_not.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_bitwise_not(NodeContext& context) { +OutputVector translate_bitwise_not(const NodeContext& context) { num_inputs_check(context, 1, 2); auto x = context.get_input(0); FRONT_END_OP_CONVERSION_CHECK(x.get_element_type().compatible(element::boolean), diff --git a/src/frontends/pytorch/src/op/bool.cpp b/src/frontends/pytorch/src/op/bool.cpp index 60d7a4e0158631..0d0d53ad608fb1 100644 --- a/src/frontends/pytorch/src/op/bool.cpp +++ b/src/frontends/pytorch/src/op/bool.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_bool(NodeContext& context) { +OutputVector translate_bool(const NodeContext& context) { num_inputs_check(context, 1, 1); return {context.mark_node(std::make_shared(context.get_input(0), element::boolean))}; }; diff --git a/src/frontends/pytorch/src/op/cat.cpp b/src/frontends/pytorch/src/op/cat.cpp index ce359c6384885f..8fbdd0f3e6f32f 100644 --- a/src/frontends/pytorch/src/op/cat.cpp +++ b/src/frontends/pytorch/src/op/cat.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_cat(NodeContext& context) { +OutputVector translate_cat(const NodeContext& context) { // This translator is only needed to get axis as constant from external scope num_inputs_check(context, 2, 2); const auto&& list_elems = get_list_as_outputs(context.get_input(0)); diff --git a/src/frontends/pytorch/src/op/clamp.cpp b/src/frontends/pytorch/src/op/clamp.cpp index afbe349cf4a25f..fa28ca301df48a 100644 --- a/src/frontends/pytorch/src/op/clamp.cpp +++ b/src/frontends/pytorch/src/op/clamp.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_clamp(NodeContext& context) { +OutputVector translate_clamp(const NodeContext& context) { num_inputs_check(context, 1, 3); auto x = context.get_input(0); if (!context.input_is_none(1)) { diff --git a/src/frontends/pytorch/src/op/constant.cpp b/src/frontends/pytorch/src/op/constant.cpp index 944a1e197867dc..6fc6444f04fa35 100644 --- a/src/frontends/pytorch/src/op/constant.cpp +++ b/src/frontends/pytorch/src/op/constant.cpp @@ -9,7 +9,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_constant(NodeContext& context) { +OutputVector translate_constant(const NodeContext& context) { return context.as_constant(); }; diff --git a/src/frontends/pytorch/src/op/conv_transposend.cpp b/src/frontends/pytorch/src/op/conv_transposend.cpp index d4a0cc2e047114..1f281f90486fad 100644 --- a/src/frontends/pytorch/src/op/conv_transposend.cpp +++ b/src/frontends/pytorch/src/op/conv_transposend.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_conv_transposend(NodeContext& context) { +OutputVector translate_conv_transposend(const NodeContext& context) { num_inputs_check(context, 8, 8); auto strides = context.const_input(3); // PyTorch support only symmetric padding, padding sizes are the same for begins and ends for each dimension diff --git a/src/frontends/pytorch/src/op/convnd.cpp b/src/frontends/pytorch/src/op/convnd.cpp index 9b91985182479a..861cb68f1fa30f 100644 --- a/src/frontends/pytorch/src/op/convnd.cpp +++ b/src/frontends/pytorch/src/op/convnd.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_convnd(NodeContext& context) { +OutputVector translate_convnd(const NodeContext& context) { num_inputs_check(context, 7, 7); auto strides = context.const_input(3); // In torch pads at beginning are same as at end diff --git a/src/frontends/pytorch/src/op/convolution.cpp b/src/frontends/pytorch/src/op/convolution.cpp index d0e77e4adadafe..7a250115b54a51 100644 --- a/src/frontends/pytorch/src/op/convolution.cpp +++ b/src/frontends/pytorch/src/op/convolution.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_convolution(NodeContext& context) { +OutputVector translate_convolution(const NodeContext& context) { // Schema: aten::_convolution(Tensor input, Tensor weight, Tensor? bias, int[] stride, int[] padding, int[] // dilation, bool transposed, int[] output_padding, int groups, bool benchmark, bool deterministic, bool // cudnn_enabled, bool allow_tf32) -> Tensor diff --git a/src/frontends/pytorch/src/op/convolution_mode.cpp b/src/frontends/pytorch/src/op/convolution_mode.cpp index c732aa6688941f..7aa9d8f991b97b 100644 --- a/src/frontends/pytorch/src/op/convolution_mode.cpp +++ b/src/frontends/pytorch/src/op/convolution_mode.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_convolution_mode(NodeContext& context) { +OutputVector translate_convolution_mode(const NodeContext& context) { // Schema: aten::_convolution_mode(Tensor input, Tensor weight, Tensor? bias, int[] stride, str padding, int[] // dilation, int groups) -> Tensor num_inputs_check(context, 7, 7); diff --git a/src/frontends/pytorch/src/op/cumsum.cpp b/src/frontends/pytorch/src/op/cumsum.cpp index 55d46500427e6d..c396521a9e402b 100644 --- a/src/frontends/pytorch/src/op/cumsum.cpp +++ b/src/frontends/pytorch/src/op/cumsum.cpp @@ -13,7 +13,7 @@ namespace op { using namespace ov::op; -OutputVector translate_cumsum(NodeContext& context) { +OutputVector translate_cumsum(const NodeContext& context) { // aten::cumsum(Tensor self, int dim, *, ScalarType? dtype=None, Tensor out=None) num_inputs_check(context, 2, 4); auto x = context.get_input(0); diff --git a/src/frontends/pytorch/src/op/deform_conv.cpp b/src/frontends/pytorch/src/op/deform_conv.cpp new file mode 100644 index 00000000000000..1a9ae5900ce738 --- /dev/null +++ b/src/frontends/pytorch/src/op/deform_conv.cpp @@ -0,0 +1,83 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/deformable_convolution.hpp" +#include "pt_framework_node.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +using namespace ov::op; + +OutputVector translate_deform_conv(const NodeContext& context) { + // torchvision::deform_conv2d(Tensor input, Tensor weight, Tensor offset, + // Tensor mask, Tensor bias, int64_t stride_h, int64_t stride_w, + // int64_t pad_h, int64_t pad_w, int64_t dilation_h, int64_t dilation_w, + // int64_t n_weight_grps, int64_t n_offset_grps, bool use_mask) -> Tensor + num_inputs_check(context, 14, 14); + auto pt_input = context.get_input(0); + auto pt_weight = context.get_input(1); + auto pt_offset = context.get_input(2); + auto pt_mask = context.get_input(3); + + int32_t pt_stride_h = context.const_input(5); + int32_t pt_stride_w = context.const_input(6); + auto strides = Strides({(size_t)pt_stride_h, (size_t)pt_stride_w}); + + int32_t pt_pad_h = context.const_input(7); + int32_t pt_pad_w = context.const_input(8); + auto pads = CoordinateDiff({pt_pad_h, pt_pad_w}); + + int32_t pt_dilation_h = context.const_input(9); + int32_t pt_dilation_w = context.const_input(10); + auto dilations = Strides({(size_t)pt_dilation_h, (size_t)pt_dilation_w}); + + int32_t pt_n_weight_grps = context.const_input(11); + int32_t pt_n_offset_grps = context.const_input(12); + bool pt_use_mask = context.const_input(13); + + std::shared_ptr deformable_convolution; + if (!pt_use_mask) { + deformable_convolution = context.mark_node(std::make_shared(pt_input, + pt_offset, + pt_weight, + strides, + pads, + pads, + dilations, + PadType::EXPLICIT, + pt_n_weight_grps, + pt_n_offset_grps, + true)); + } else { + deformable_convolution = context.mark_node(std::make_shared(pt_input, + pt_offset, + pt_weight, + pt_mask, + strides, + pads, + pads, + dilations, + PadType::EXPLICIT, + pt_n_weight_grps, + pt_n_offset_grps, + true)); + } + + if (!context.input_is_none(4)) { + auto bias = context.get_input(4); + bias = reshape_channelwise(context, bias, deformable_convolution); + deformable_convolution = context.mark_node(std::make_shared(deformable_convolution, bias)); + } + return {context.mark_output(deformable_convolution)}; +} +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/op/dim.cpp b/src/frontends/pytorch/src/op/dim.cpp index 7af4aa8fe3147e..2d69cb3e37a796 100644 --- a/src/frontends/pytorch/src/op/dim.cpp +++ b/src/frontends/pytorch/src/op/dim.cpp @@ -12,7 +12,7 @@ namespace op { using namespace ov::op; -OutputVector translate_dim(NodeContext& context) { +OutputVector translate_dim(const NodeContext& context) { num_inputs_check(context, 1, 1); Output rank; std::tie(std::ignore, rank) = get_shape_rank(context, context.get_input(0), true); diff --git a/src/frontends/pytorch/src/op/div.cpp b/src/frontends/pytorch/src/op/div.cpp index 54cab6325ae7f2..e9dd7136e35bef 100644 --- a/src/frontends/pytorch/src/op/div.cpp +++ b/src/frontends/pytorch/src/op/div.cpp @@ -17,7 +17,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_div(NodeContext& context) { +OutputVector translate_div(const NodeContext& context) { num_inputs_check(context, 2, 3); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/elu.cpp b/src/frontends/pytorch/src/op/elu.cpp index f60d76b96638e5..4f96371ee83ebd 100644 --- a/src/frontends/pytorch/src/op/elu.cpp +++ b/src/frontends/pytorch/src/op/elu.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_elu(NodeContext& context) { +OutputVector translate_elu(const NodeContext& context) { // aten::elu(Tensor self, Scalar alpha=1, Scalar scale=1, Scalar input_scale=1) -> Tensor num_inputs_check(context, 2, 4); auto x = context.get_input(0); diff --git a/src/frontends/pytorch/src/op/embedding.cpp b/src/frontends/pytorch/src/op/embedding.cpp index c920992bdaafec..e5dc85a0ddfebf 100644 --- a/src/frontends/pytorch/src/op/embedding.cpp +++ b/src/frontends/pytorch/src/op/embedding.cpp @@ -13,7 +13,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_embedding(NodeContext& context) { +OutputVector translate_embedding(const NodeContext& context) { // aten::embedding(Tensor weight, Tensor indices, SymInt padding_idx=-1, bool scale_grad_by_freq=False, bool // sparse=False) num_inputs_check(context, 5, 5); diff --git a/src/frontends/pytorch/src/op/expand.cpp b/src/frontends/pytorch/src/op/expand.cpp index 34f0a9d70c323b..9210cedc6eba6f 100644 --- a/src/frontends/pytorch/src/op/expand.cpp +++ b/src/frontends/pytorch/src/op/expand.cpp @@ -30,7 +30,7 @@ OutputVector base_expand(const NodeContext& context, const Output& x, cons }; } // namespace -OutputVector translate_expand(NodeContext& context) { +OutputVector translate_expand(const NodeContext& context) { // aten::expand(Tensor(a) self, SymInt[] size, *, bool implicit=False) -> Tensor(a) num_inputs_check(context, 2, 3); auto x = context.get_input(0); @@ -41,7 +41,7 @@ OutputVector translate_expand(NodeContext& context) { return base_expand(context, x, sizes); }; -OutputVector translate_expand_as(NodeContext& context) { +OutputVector translate_expand_as(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/eye.cpp b/src/frontends/pytorch/src/op/eye.cpp index ab35c56569c5c4..9b7f7ef8c3bc29 100644 --- a/src/frontends/pytorch/src/op/eye.cpp +++ b/src/frontends/pytorch/src/op/eye.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_eye(NodeContext& context) { +OutputVector translate_eye(const NodeContext& context) { size_t num_inputs = context.get_input_size(); auto x = context.get_input(0); // num rows and cols should be integer, but at the moment conversion their data type can be unknown yet diff --git a/src/frontends/pytorch/src/op/flatten.cpp b/src/frontends/pytorch/src/op/flatten.cpp index 6d9005a64b8643..6022661c3aa8cf 100644 --- a/src/frontends/pytorch/src/op/flatten.cpp +++ b/src/frontends/pytorch/src/op/flatten.cpp @@ -18,7 +18,7 @@ namespace op { using namespace ov::op; -OutputVector translate_flatten(NodeContext& context) { +OutputVector translate_flatten(const NodeContext& context) { num_inputs_check(context, 1, 3); auto x = context.get_input(0); int64_t start_dim = 0; diff --git a/src/frontends/pytorch/src/op/floor_divide.cpp b/src/frontends/pytorch/src/op/floor_divide.cpp index 5731006dd770aa..4fb1b230d44c21 100644 --- a/src/frontends/pytorch/src/op/floor_divide.cpp +++ b/src/frontends/pytorch/src/op/floor_divide.cpp @@ -14,7 +14,7 @@ namespace op { using namespace ov::op; -OutputVector translate_floor_divide(NodeContext& context) { +OutputVector translate_floor_divide(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/floordiv.cpp b/src/frontends/pytorch/src/op/floordiv.cpp index b85cacf3fc5145..91c03e74d7f0b1 100644 --- a/src/frontends/pytorch/src/op/floordiv.cpp +++ b/src/frontends/pytorch/src/op/floordiv.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_floordiv(NodeContext& context) { +OutputVector translate_floordiv(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/full.cpp b/src/frontends/pytorch/src/op/full.cpp index abfacbf872f913..cf60d096555007 100644 --- a/src/frontends/pytorch/src/op/full.cpp +++ b/src/frontends/pytorch/src/op/full.cpp @@ -42,7 +42,7 @@ Output base_translate_full_with_convert(const NodeContext& context, } } // namespace -OutputVector translate_full(NodeContext& context) { +OutputVector translate_full(const NodeContext& context) { num_inputs_check(context, 2, 6); auto sizes = context.get_input(0); auto value = context.get_input(1); @@ -59,7 +59,7 @@ OutputVector translate_full(NodeContext& context) { return {base_translate_full_with_convert(context, sizes, value, dtype_id)}; }; -OutputVector translate_full_like(NodeContext& context) { +OutputVector translate_full_like(const NodeContext& context) { num_inputs_check(context, 2, 7); auto input = context.get_input(0); auto value = context.get_input(1); @@ -71,7 +71,7 @@ OutputVector translate_full_like(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, out)}; }; -OutputVector translate_fill_(NodeContext& context) { +OutputVector translate_fill_(const NodeContext& context) { num_inputs_check(context, 2, 2); auto input = context.get_input(0); auto value = context.get_input(1); @@ -79,7 +79,7 @@ OutputVector translate_fill_(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, input)}; }; -OutputVector translate_new_full(NodeContext& context) { +OutputVector translate_new_full(const NodeContext& context) { num_inputs_check(context, 3, 7); auto input = context.get_input(0); auto sizes = context.get_input(1); @@ -90,7 +90,7 @@ OutputVector translate_new_full(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, input)}; }; -OutputVector translate_zeros(NodeContext& context) { +OutputVector translate_zeros(const NodeContext& context) { num_inputs_check(context, 2, 5); auto sizes = context.get_input(0); auto value = context.mark_node(v0::Constant::create(element::f32, Shape{}, {0})); @@ -107,7 +107,7 @@ OutputVector translate_zeros(NodeContext& context) { return {base_translate_full_with_convert(context, sizes, value, dtype_id)}; }; -OutputVector translate_zeros_like(NodeContext& context) { +OutputVector translate_zeros_like(const NodeContext& context) { num_inputs_check(context, 1, 6); auto input = context.get_input(0); auto value = context.mark_node(v0::Constant::create(element::f32, Shape{}, {0})); @@ -119,7 +119,7 @@ OutputVector translate_zeros_like(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, out)}; }; -OutputVector translate_new_zeros(NodeContext& context) { +OutputVector translate_new_zeros(const NodeContext& context) { num_inputs_check(context, 2, 6); auto input = context.get_input(0); auto sizes = context.get_input(1); @@ -130,7 +130,7 @@ OutputVector translate_new_zeros(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, input)}; }; -OutputVector translate_ones(NodeContext& context) { +OutputVector translate_ones(const NodeContext& context) { num_inputs_check(context, 1, 5); auto sizes = context.get_input(0); auto value = context.mark_node(v0::Constant::create(element::f32, Shape{}, {1})); @@ -147,7 +147,7 @@ OutputVector translate_ones(NodeContext& context) { return {base_translate_full_with_convert(context, sizes, value, dtype_id)}; }; -OutputVector translate_ones_like(NodeContext& context) { +OutputVector translate_ones_like(const NodeContext& context) { num_inputs_check(context, 1, 6); auto input = context.get_input(0); auto value = context.mark_node(v0::Constant::create(element::f32, Shape{}, {1})); @@ -159,7 +159,7 @@ OutputVector translate_ones_like(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, out)}; }; -OutputVector translate_new_ones(NodeContext& context) { +OutputVector translate_new_ones(const NodeContext& context) { num_inputs_check(context, 2, 6); auto input = context.get_input(0); auto sizes = context.get_input(1); @@ -170,13 +170,13 @@ OutputVector translate_new_ones(NodeContext& context) { return {base_translate_full_with_convertlike(context, sizes, value, input)}; }; -OutputVector translate_empty(NodeContext& context) { +OutputVector translate_empty(const NodeContext& context) { // aten::empty(SymInt[] size, *, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? // pin_memory=None, MemoryFormat? memory_format=None) -> Tensor layout, device and work with memory ignored on our // side, so just skip these parameters num_inputs_check(context, 1, 6); auto sizes = context.get_input(0); - // In OV uninitialised data is not supported, so we create a tensor filled with zeros with a given shape and type. + // In OV uninitialized data is not supported, so we create a tensor filled with zeros with a given shape and type. auto value = context.mark_node(v0::Constant::create(element::f32, Shape{}, {0})); int dtype_id = 1; Output empty; diff --git a/src/frontends/pytorch/src/op/gelu.cpp b/src/frontends/pytorch/src/op/gelu.cpp index 598f6865b3e2b0..c64ea647688584 100644 --- a/src/frontends/pytorch/src/op/gelu.cpp +++ b/src/frontends/pytorch/src/op/gelu.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_gelu(NodeContext& context) { +OutputVector translate_gelu(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto approximate = context.const_input(1); diff --git a/src/frontends/pytorch/src/op/get_attr.cpp b/src/frontends/pytorch/src/op/get_attr.cpp index 3575a5210a8518..1d0ae0e4d13d1b 100644 --- a/src/frontends/pytorch/src/op/get_attr.cpp +++ b/src/frontends/pytorch/src/op/get_attr.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_get_attr(NodeContext& context) { +OutputVector translate_get_attr(const NodeContext& context) { auto res = context.get_decoder()->try_decode_get_attr(); FRONT_END_OP_CONVERSION_CHECK(res.size() > 0, "GetAttr must have at least one output."); return res; diff --git a/src/frontends/pytorch/src/op/getitem.cpp b/src/frontends/pytorch/src/op/getitem.cpp index 1bf9f4a0e8a274..0a1243196f4d6c 100644 --- a/src/frontends/pytorch/src/op/getitem.cpp +++ b/src/frontends/pytorch/src/op/getitem.cpp @@ -13,7 +13,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_getitem(NodeContext& context) { +OutputVector translate_getitem(const NodeContext& context) { num_inputs_check(context, 2, 2); auto input = context.get_input(0); if (std::dynamic_pointer_cast(input.get_node_shared_ptr())) { diff --git a/src/frontends/pytorch/src/op/glu.cpp b/src/frontends/pytorch/src/op/glu.cpp index e650e9c4a54c05..dbe979fb1f2870 100644 --- a/src/frontends/pytorch/src/op/glu.cpp +++ b/src/frontends/pytorch/src/op/glu.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_glu(NodeContext& context) { +OutputVector translate_glu(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto dim = context.input_is_none(1) ? context.mark_node(v0::Constant::create(element::i32, Shape{}, {-1})) diff --git a/src/frontends/pytorch/src/op/grid_sampler.cpp b/src/frontends/pytorch/src/op/grid_sampler.cpp index 9011abd8566a67..8c603813d888f7 100644 --- a/src/frontends/pytorch/src/op/grid_sampler.cpp +++ b/src/frontends/pytorch/src/op/grid_sampler.cpp @@ -13,7 +13,7 @@ namespace op { using namespace ov::op; -OutputVector translate_grid_sampler(NodeContext& context) { +OutputVector translate_grid_sampler(const NodeContext& context) { num_inputs_check(context, 4, 5); auto x = context.get_input(0); auto grid = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/group_norm.cpp b/src/frontends/pytorch/src/op/group_norm.cpp index 7b3ac53bbb4772..6ce36aac6601b1 100644 --- a/src/frontends/pytorch/src/op/group_norm.cpp +++ b/src/frontends/pytorch/src/op/group_norm.cpp @@ -20,7 +20,7 @@ namespace op { using namespace ov::op; -OutputVector translate_group_norm(NodeContext& context) { +OutputVector translate_group_norm(const NodeContext& context) { // aten::group_norm(Tensor input, int num_groups, Tensor? weight=None, Tensor? bias=None, float // eps=1.0000000000000001e-05, bool cudnn_enabled=True) -> Tensor num_inputs_check(context, 2, 6); diff --git a/src/frontends/pytorch/src/op/hardtanh.cpp b/src/frontends/pytorch/src/op/hardtanh.cpp index 52551a259a97d2..a85bedbf00626b 100644 --- a/src/frontends/pytorch/src/op/hardtanh.cpp +++ b/src/frontends/pytorch/src/op/hardtanh.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_hardtanh(NodeContext& context) { +OutputVector translate_hardtanh(const NodeContext& context) { num_inputs_check(context, 1, 3); float min = -1; float max = 1; diff --git a/src/frontends/pytorch/src/op/if.cpp b/src/frontends/pytorch/src/op/if.cpp index 1e5d3a26778cc1..77015fb1deea6b 100644 --- a/src/frontends/pytorch/src/op/if.cpp +++ b/src/frontends/pytorch/src/op/if.cpp @@ -13,7 +13,32 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_if(NodeContext& context) { +namespace { +// TODO: Ticket 106627. This is a WA and will work only if both branches of if will eventually go to the operation that +// will have same output type for both types +void align_result_types(const NodeContext& context, + std::shared_ptr r1, + std::shared_ptr r2) { + auto r1_tensor = r1->input_value(0); + auto r2_tensor = r2->input_value(0); + auto r1_type = r1_tensor.get_element_type(); + auto r2_type = r2_tensor.get_element_type(); + if (r1_type.is_dynamic() || r2_type.is_dynamic()) + return; + element::Type merged_type; + if (!element::Type::merge(merged_type, r1_type, r2_type)) { + if (r1_type.bitwidth() >= r2_type.bitwidth()) { + auto convert = std::make_shared(r2_tensor, r1_type); + r2->set_argument(0, convert); + } else { + auto convert = std::make_shared(r1_tensor, r2_type); + r1->set_argument(0, convert); + } + } +} +} // namespace + +OutputVector translate_if(const NodeContext& context) { auto if_node = std::make_shared(context.get_input(0)); context.mark_node(if_node); auto decoder = context.get_decoder(); @@ -62,6 +87,7 @@ OutputVector translate_if(NodeContext& context) { FRONT_END_OP_CONVERSION_CHECK(then_results.size() >= num_outs && else_results.size() >= num_outs, "Else or then body have less outputs than prim::If requires."); for (size_t i = 0; i < num_outs; i++) { + align_result_types(context, then_results[i], else_results[i]); res.push_back(if_node->set_output(then_results[i], else_results[i])); } // Each body can have mutated outputs that are not included into pytorch node outputs. @@ -136,6 +162,7 @@ OutputVector translate_if(NodeContext& context) { } } for (const auto& output_idx : extra_output_idxs) { + align_result_types(context, extra_then_body_results.at(output_idx), extra_else_body_results.at(output_idx)); context.add_tensor_to_context( output_idx, if_node->set_output(extra_then_body_results.at(output_idx), extra_else_body_results.at(output_idx))); diff --git a/src/frontends/pytorch/src/op/im2col.cpp b/src/frontends/pytorch/src/op/im2col.cpp index 12fb4f3b7c4a04..718e0eadaa4ca0 100644 --- a/src/frontends/pytorch/src/op/im2col.cpp +++ b/src/frontends/pytorch/src/op/im2col.cpp @@ -56,7 +56,7 @@ std::shared_ptr get_im2col_indices_along_dim(const NodeContext& context, } } // namespace -OutputVector translate_im2col(NodeContext& context) { +OutputVector translate_im2col(const NodeContext& context) { num_inputs_check(context, 5, 5); auto input = context.get_input(0); auto kernel_size = context.const_input>(1); diff --git a/src/frontends/pytorch/src/op/index_put_.cpp b/src/frontends/pytorch/src/op/index_put_.cpp index 1ce4ea0e96d6a1..d8a599eaf5abcf 100644 --- a/src/frontends/pytorch/src/op/index_put_.cpp +++ b/src/frontends/pytorch/src/op/index_put_.cpp @@ -10,9 +10,7 @@ namespace frontend { namespace pytorch { namespace op { -using namespace ov::op; - -OutputVector translate_index_put_(NodeContext& context) { +OutputVector translate_index_put_(const NodeContext& context) { // Pass as PtFrameworkNode to register as `inplace_op`. Conversion to OV operators is done as transformation. auto node = std::make_shared(context.get_decoder(), context.inputs()); return {context.mark_node(node)}; diff --git a/src/frontends/pytorch/src/op/instance_norm.cpp b/src/frontends/pytorch/src/op/instance_norm.cpp index b00b7bff260bd1..ff53bd11e92ffb 100644 --- a/src/frontends/pytorch/src/op/instance_norm.cpp +++ b/src/frontends/pytorch/src/op/instance_norm.cpp @@ -88,7 +88,7 @@ OutputVector translate_instance_norm_train(const NodeContext& context, } // namespace -OutputVector translate_instance_norm(NodeContext& context) { +OutputVector translate_instance_norm(const NodeContext& context) { num_inputs_check(context, 8, 9); auto input = context.get_input(0); auto eps = context.const_input(7); diff --git a/src/frontends/pytorch/src/op/int.cpp b/src/frontends/pytorch/src/op/int.cpp index e11397cb6f11e3..5a407a1a7254c3 100644 --- a/src/frontends/pytorch/src/op/int.cpp +++ b/src/frontends/pytorch/src/op/int.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_int(NodeContext& context) { +OutputVector translate_int(const NodeContext& context) { num_inputs_check(context, 1, 1); return {context.mark_node(std::make_shared(context.get_input(0), element::i32))}; }; diff --git a/src/frontends/pytorch/src/op/layer_norm.cpp b/src/frontends/pytorch/src/op/layer_norm.cpp index c954110111e799..204d7164531c72 100644 --- a/src/frontends/pytorch/src/op/layer_norm.cpp +++ b/src/frontends/pytorch/src/op/layer_norm.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_layer_norm(NodeContext& context) { +OutputVector translate_layer_norm(const NodeContext& context) { num_inputs_check(context, 5, 6); auto eps = context.const_input(4); auto normalized_shape = context.const_input(1); diff --git a/src/frontends/pytorch/src/op/len.cpp b/src/frontends/pytorch/src/op/len.cpp index 71f8bdf3a32b88..9a22658500913f 100644 --- a/src/frontends/pytorch/src/op/len.cpp +++ b/src/frontends/pytorch/src/op/len.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_len(NodeContext& context) { +OutputVector translate_len(const NodeContext& context) { num_inputs_check(context, 1, 1); auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); auto const_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {1})); diff --git a/src/frontends/pytorch/src/op/linear.cpp b/src/frontends/pytorch/src/op/linear.cpp index e94ff7c9168003..8288220f320251 100644 --- a/src/frontends/pytorch/src/op/linear.cpp +++ b/src/frontends/pytorch/src/op/linear.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_linear(NodeContext& context) { +OutputVector translate_linear(const NodeContext& context) { // schema: aten::linear(Tensor input, Tensor weight, Tensor? bias=None) -> Tensor num_inputs_check(context, 2, 3); auto x = context.get_input(0); diff --git a/src/frontends/pytorch/src/op/list_construct.cpp b/src/frontends/pytorch/src/op/list_construct.cpp index e69188e23d89a3..e58a3c4744ff61 100644 --- a/src/frontends/pytorch/src/op/list_construct.cpp +++ b/src/frontends/pytorch/src/op/list_construct.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_list_construct(NodeContext& context) { +OutputVector translate_list_construct(const NodeContext& context) { // Process the case when prim::ListConstruct has all inputs constant auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); ov::OutputVector consts; diff --git a/src/frontends/pytorch/src/op/log.cpp b/src/frontends/pytorch/src/op/log.cpp index 85947b7694ee06..808dff6ed32822 100644 --- a/src/frontends/pytorch/src/op/log.cpp +++ b/src/frontends/pytorch/src/op/log.cpp @@ -17,7 +17,7 @@ namespace op { using namespace ov::op; -OutputVector translate_log(NodeContext& context) { +OutputVector translate_log(const NodeContext& context) { // torch.log returns a tensor with the natural logarithm of the elements of input. num_inputs_check(context, 1, 1); auto x = context.get_input(0); @@ -26,7 +26,7 @@ OutputVector translate_log(NodeContext& context) { return {log}; }; -OutputVector translate_log2(NodeContext& context) { +OutputVector translate_log2(const NodeContext& context) { // torch.log2 returns a tensor with the logarithm to the base 2 of the elements of input. num_inputs_check(context, 1, 1); auto x = context.get_input(0); diff --git a/src/frontends/pytorch/src/op/loop.cpp b/src/frontends/pytorch/src/op/loop.cpp index 7bf03cfcd30138..36369ea63bd4ee 100644 --- a/src/frontends/pytorch/src/op/loop.cpp +++ b/src/frontends/pytorch/src/op/loop.cpp @@ -13,7 +13,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_loop(NodeContext& context) { +OutputVector translate_loop(const NodeContext& context) { const auto& inputs = context.inputs(); FRONT_END_OP_CONVERSION_CHECK(inputs.size() >= 2, "Loop must have at least 2 inputs."); auto loop = std::make_shared(inputs[0], inputs[1]); diff --git a/src/frontends/pytorch/src/op/masked_fill.cpp b/src/frontends/pytorch/src/op/masked_fill.cpp index 2a071755b3a145..5ed090e0b619b7 100644 --- a/src/frontends/pytorch/src/op/masked_fill.cpp +++ b/src/frontends/pytorch/src/op/masked_fill.cpp @@ -18,7 +18,7 @@ namespace op { using namespace ov::op; -OutputVector translate_masked_fill(NodeContext& context) { +OutputVector translate_masked_fill(const NodeContext& context) { num_inputs_check(context, 3, 3); auto data = context.get_input(0); auto mask = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/max_poolnd.cpp b/src/frontends/pytorch/src/op/max_poolnd.cpp index 0934611f76dcf2..78eab0c87e7a73 100644 --- a/src/frontends/pytorch/src/op/max_poolnd.cpp +++ b/src/frontends/pytorch/src/op/max_poolnd.cpp @@ -13,7 +13,7 @@ namespace op { using namespace ov::op; -OutputVector translate_max_poolnd(NodeContext& context) { +OutputVector translate_max_poolnd(const NodeContext& context) { num_inputs_check(context, 4, 6); auto kernel = context.const_input(1); auto strides = context.const_input(2); diff --git a/src/frontends/pytorch/src/op/mean.cpp b/src/frontends/pytorch/src/op/mean.cpp index b7a5acfb6fb22f..46c42f6be1a4b7 100644 --- a/src/frontends/pytorch/src/op/mean.cpp +++ b/src/frontends/pytorch/src/op/mean.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_mean(NodeContext& context) { +OutputVector translate_mean(const NodeContext& context) { num_inputs_check(context, 3, 4); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/meshgrid.cpp b/src/frontends/pytorch/src/op/meshgrid.cpp index 841de80fcbf494..c9b5833ae68d60 100644 --- a/src/frontends/pytorch/src/op/meshgrid.cpp +++ b/src/frontends/pytorch/src/op/meshgrid.cpp @@ -10,7 +10,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_meshgrid(NodeContext& context) { +OutputVector translate_meshgrid(const NodeContext& context) { std::string indexing = "ij"; if (!context.input_is_none(1)) { indexing = context.const_input(1); diff --git a/src/frontends/pytorch/src/op/min_max.cpp b/src/frontends/pytorch/src/op/min_max.cpp index 34c70219f1137c..898403bf82b7cf 100644 --- a/src/frontends/pytorch/src/op/min_max.cpp +++ b/src/frontends/pytorch/src/op/min_max.cpp @@ -20,7 +20,7 @@ namespace op { using namespace ov::op; -OutputVector translate_max(NodeContext& context) { +OutputVector translate_max(const NodeContext& context) { // torch.max (same for torch.min) actually has two interfaces smashed together: // torch.max(x, dim, keepdim) and torch.max(x, y) num_inputs_check(context, 1, 3); @@ -49,7 +49,7 @@ OutputVector translate_max(NodeContext& context) { return {values, indicies}; }; -OutputVector translate_min(NodeContext& context) { +OutputVector translate_min(const NodeContext& context) { // torch.min (same for torch.max) actually has two interfaces smashed together: // torch.min(x, dim, keepdim) and torch.min(x, y) num_inputs_check(context, 1, 3); diff --git a/src/frontends/pytorch/src/op/narrow.cpp b/src/frontends/pytorch/src/op/narrow.cpp index a212b22503434e..ffae7d2ec55a03 100644 --- a/src/frontends/pytorch/src/op/narrow.cpp +++ b/src/frontends/pytorch/src/op/narrow.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_narrow(NodeContext& context) { +OutputVector translate_narrow(const NodeContext& context) { num_inputs_check(context, 4, 4); auto const_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {1})); diff --git a/src/frontends/pytorch/src/op/neg.cpp b/src/frontends/pytorch/src/op/neg.cpp index e902eb0f21fefb..423118c3b2fc24 100644 --- a/src/frontends/pytorch/src/op/neg.cpp +++ b/src/frontends/pytorch/src/op/neg.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_neg(NodeContext& context) { +OutputVector translate_neg(const NodeContext& context) { num_inputs_check(context, 1, 1); auto x = context.get_input(0); auto const_neg_1 = context.mark_node(v0::Constant::create(element::i32, Shape{}, {-1})); diff --git a/src/frontends/pytorch/src/op/nms.cpp b/src/frontends/pytorch/src/op/nms.cpp index 2454d94a78e6a8..86ecb3df73cf87 100644 --- a/src/frontends/pytorch/src/op/nms.cpp +++ b/src/frontends/pytorch/src/op/nms.cpp @@ -18,7 +18,7 @@ namespace op { using namespace ov::op; -OutputVector translate_nms(NodeContext& context) { +OutputVector translate_nms(const NodeContext& context) { num_inputs_check(context, 3, 3); auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); auto const_1 = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); diff --git a/src/frontends/pytorch/src/op/nonzero.cpp b/src/frontends/pytorch/src/op/nonzero.cpp index 80edef3f079b6b..29a6aa51175008 100644 --- a/src/frontends/pytorch/src/op/nonzero.cpp +++ b/src/frontends/pytorch/src/op/nonzero.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_nonzero(NodeContext& context) { +OutputVector translate_nonzero(const NodeContext& context) { num_inputs_check(context, 1, 1); auto cond = context.get_input(0); auto non_zero = context.mark_node(std::make_shared(cond)); diff --git a/src/frontends/pytorch/src/op/norm.cpp b/src/frontends/pytorch/src/op/norm.cpp index d35c0fec25867f..a743816ee8d845 100644 --- a/src/frontends/pytorch/src/op/norm.cpp +++ b/src/frontends/pytorch/src/op/norm.cpp @@ -5,12 +5,23 @@ #include "openvino/frontend/pytorch/node_context.hpp" #include "openvino/op/abs.hpp" #include "openvino/op/constant.hpp" +#include "openvino/op/convert.hpp" +#include "openvino/op/convert_like.hpp" +#include "openvino/op/gather.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/not_equal.hpp" #include "openvino/op/power.hpp" +#include "openvino/op/range.hpp" #include "openvino/op/reduce_l1.hpp" #include "openvino/op/reduce_l2.hpp" #include "openvino/op/reduce_max.hpp" #include "openvino/op/reduce_min.hpp" #include "openvino/op/reduce_sum.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/shape_of.hpp" +#include "openvino/op/sqrt.hpp" +#include "openvino/op/squeeze.hpp" +#include "pt_framework_node.hpp" #include "utils.hpp" namespace ov { @@ -20,13 +31,12 @@ namespace op { using namespace ov::op; -OutputVector translate_norm(NodeContext& context) { - num_inputs_check(context, 4, 4); - auto input_tensor = context.get_input(0); - auto p = context.const_input(1); - auto dim = context.get_input(2); - auto keep_dim = context.const_input(3); - +namespace { +Output norm_vector(const NodeContext& context, + Output input_tensor, + Output dim, + float p, + bool keep_dim) { Output res; if (p == 1) { res = context.mark_node(std::make_shared(input_tensor, dim, keep_dim)); @@ -38,17 +48,253 @@ OutputVector translate_norm(NodeContext& context) { } else if (p == -std::numeric_limits::infinity()) { auto abs = context.mark_node(std::make_shared(input_tensor)); res = context.mark_node(std::make_shared(abs, dim, keep_dim)); + } else if (p == 0) { + auto input_rank = input_tensor.get_partial_shape().rank(); + FRONT_END_OP_CONVERSION_CHECK(input_rank.is_dynamic() || input_rank.get_length() == 1, + "ord=0 supported only for vector norm"); + auto zero = context.mark_node(v0::Constant::create(element::f32, Shape{}, {0})); + zero = context.mark_node(std::make_shared(zero, input_tensor)); + auto cond = context.mark_node(std::make_shared(input_tensor, zero)); + cond = context.mark_node(std::make_shared(cond, input_tensor)); + res = context.mark_node(std::make_shared(cond, dim, keep_dim)); } else { - auto const_p = context.mark_node(v0::Constant::create(element::f64, Shape{1}, {p})); - auto const_p_inv = context.mark_node(v0::Constant::create(element::f64, Shape{1}, {1.0 / p})); + auto const_p = context.mark_node(v0::Constant::create(element::f32, Shape{1}, {p})); + const_p = context.mark_node(std::make_shared(const_p, input_tensor)); + auto const_p_inv = context.mark_node(v0::Constant::create(element::f32, Shape{1}, {1.0 / p})); + const_p_inv = context.mark_node(std::make_shared(const_p_inv, input_tensor)); auto abs = context.mark_node(std::make_shared(input_tensor)); auto pow = context.mark_node(std::make_shared(abs, const_p)); auto sum = context.mark_node(std::make_shared(pow, dim, keep_dim)); res = context.mark_node(std::make_shared(sum, const_p_inv)); } + return res; +}; + +Output norm_matrix(const NodeContext& context, + Output input_tensor, + Output dim, + float p, + bool keep_dim) { + Output res; + auto one = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); + auto zero = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); + auto first_dim = context.mark_node(std::make_shared(dim, zero, zero)); + auto second_dim = context.mark_node(std::make_shared(dim, one, zero)); + if (p == 1) { + auto abs = context.mark_node(std::make_shared(input_tensor)); + auto sum = context.mark_node(std::make_shared(abs, first_dim, true)); + res = context.mark_node(std::make_shared(sum, second_dim, keep_dim)); + } else if (p == std::numeric_limits::infinity()) { + auto abs = context.mark_node(std::make_shared(input_tensor)); + auto sum = context.mark_node(std::make_shared(abs, second_dim, true)); + res = context.mark_node(std::make_shared(sum, first_dim, keep_dim)); + } else if (p == -std::numeric_limits::infinity()) { + auto abs = context.mark_node(std::make_shared(input_tensor)); + auto sum = context.mark_node(std::make_shared(abs, second_dim, true)); + res = context.mark_node(std::make_shared(sum, first_dim, keep_dim)); + } else if (p == -1) { + auto abs = context.mark_node(std::make_shared(input_tensor)); + auto sum = context.mark_node(std::make_shared(abs, first_dim, true)); + res = context.mark_node(std::make_shared(sum, second_dim, keep_dim)); + } else { + FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported ord ", p, " for matrix norm"); + } + + return res; +}; + +Output frobenius_norm(const NodeContext& context, Output x, Output dim, bool keep_dim) { + auto sqr = context.mark_node(std::make_shared(x, x)); + auto sumsqr = context.mark_node(std::make_shared(sqr, dim, keep_dim)); + return context.mark_node(std::make_shared(sumsqr)); +} +}; // namespace + +OutputVector translate_norm(const NodeContext& context) { + num_inputs_check(context, 4, 6); + auto input_tensor = context.get_input(0); + auto p_node_type = context.get_input_type(1); + Output dim; + if (context.input_is_none(2)) { + auto zero = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); + auto one = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); + auto input_shape = context.mark_node(std::make_shared(input_tensor, element::i32)); + auto rank = context.mark_node(std::make_shared(input_shape, element::i32)); + rank = context.mark_node(std::make_shared(rank, zero)); + dim = context.mark_node(std::make_shared(zero, rank, one)); + } else { + dim = context.get_input(2); + } + auto keep_dim = context.const_input(3); + if (!context.input_is_none(4)) { + input_tensor = apply_dtype(context, 4, input_tensor); + } + Output res; + if (p_node_type.is()) { + auto p_str = context.const_input(1); + if (p_str == "fro") { + res = frobenius_norm(context, input_tensor, dim, keep_dim); + } else { + FRONT_END_OP_CONVERSION_CHECK(false, "Umsupported ord ", p_str); + } + } else { + auto p = context.const_input(1); + res = norm_vector(context, input_tensor, dim, p, keep_dim); + } + // output tensor + if (!context.input_is_none(5)) { + context.mutate_input(5, res); + } return {res}; }; +OutputVector translate_linalg_vector_norm(const NodeContext& context) { + // aten::linalg_vector_norm(Tensor self, Scalar ord=2, int[1]? dim=None, bool keepdim=False, *, ScalarType? + // dtype=None) -> Tensor + // aten::linalg_vector_norm.out(Tensor self, Scalar ord=2, int[1]? dim=None, bool + // keepdim=False, *, ScalarType? dtype=None, Tensor(a!) out) -> Tensor(a!): + num_inputs_check(context, 5, 6); + auto x = context.get_input(0); + // ord defines the vector norm that is computed. + auto ord = context.const_input(1); + bool keep_dim = context.const_input(3); + Output dim; + Output result; + // If dim= None, x will be flattened before the norm is computed. + if (context.input_is_none(2)) { + keep_dim = false; + auto minus_one = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {-1})); + x = context.mark_node(std::make_shared(x, minus_one, false)); + dim = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); + } else { + dim = context.get_input(2); + } + // dtype may be used to perform the computation in a more precise dtype. It is semantically equivalent to calling + // linalg.vector_norm(x.to(dtype)) + if (!context.input_is_none(4)) { + x = apply_dtype(context, 4, x); + } + result = norm_vector(context, x, dim, ord, keep_dim); + // output tensor + if (!context.input_is_none(5)) { + context.mutate_input(5, result); + } + return {result}; +}; + +OutputVector translate_linalg_matrix_norm(const NodeContext& context) { + // aten::linalg_matrix_norm.out(Tensor self, Scalar ord, int[] dim=[-2, -1], bool keepdim=False, *, ScalarType? + // dtype=None, Tensor(a!) out) -> Tensor(a!) aten::linalg_matrix_norm(Tensor self, Scalar ord, int[] dim=[-2, -1], + // bool keepdim=False, *, ScalarType? dtype=None) aten::linalg_matrix_norm.str_ord(Tensor self, str ord="fro", int[] + // dim=[-2, -1], bool keepdim=False, *, ScalarType? dtype=None) + num_inputs_check(context, 5, 6); + auto x = context.get_input(0); + // ord defines the vector norm that is computed can be string or number + auto ord_type = context.get_input_type(1); + auto dim = context.get_input(2); + bool keep_dim = context.const_input(3); + Output result; + + // dtype may be used to perform the computation in a more precise dtype. It is semantically equivalent to calling + // linalg.mtrix_norm(x.to(dtype)) + if (!context.input_is_none(4)) { + x = apply_dtype(context, 4, x); + } + if (ord_type.is()) { + auto p_str = context.const_input(1); + if (p_str == "fro") { + result = frobenius_norm(context, x, dim, keep_dim); + } else { + FRONT_END_OP_CONVERSION_CHECK(false, "Umsupported ord ", p_str); + } + } else { + auto p = context.const_input(1); + result = norm_matrix(context, x, dim, p, keep_dim); + } + // output tensor + if (!context.input_is_none(5)) { + context.mutate_input(5, result); + } + return {result}; +}; + +OutputVector translate_linalg_norm(const NodeContext& context) { + // aten::linalg_norm(Tensor self, Scalar? ord=None, int[1]? dim=None, bool keepdim=False, *, ScalarType? dtype=None) + // aten::linalg_norm.ord_str(Tensor self, str ord, int[1]? dim=None, bool keepdim=False, *, ScalarType? dtype=None) + // aten::linalg_norm.ord_str_out(Tensor self, str ord, int[1]? dim=None, bool keepdim=False, *, ScalarType? + // dtype=None, Tensor(a!) out) -> Tensor(a!) + num_inputs_check(context, 5, 6); + auto x = context.get_input(0); + bool keep_dim = context.const_input(3); + Output result; + Output dim; + // dtype may be used to perform the computation in a more precise dtype. It is semantically equivalent to calling + // linalg.norm(x.to(dtype)) + if (!context.input_is_none(4)) { + x = apply_dtype(context, 4, x); + } + // If dim= None and ord= None, A will be flattened to 1D and the 2-norm of the resulting vector will be computed. + if (context.input_is_none(2) && context.input_is_none(1)) { + auto minus_one = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {-1})); + x = context.mark_node(std::make_shared(x, minus_one, false)); + dim = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); + result = norm_vector(context, x, dim, 2, false); + } else { + // If dim=None apply for all dimesions + if (context.input_is_none(2)) { + auto zero = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); + auto one = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); + auto input_shape = context.mark_node(std::make_shared(x, element::i32)); + auto rank = context.mark_node(std::make_shared(input_shape, element::i32)); + rank = context.mark_node(std::make_shared(rank, zero)); + dim = context.mark_node(std::make_shared(zero, rank, one)); + } else { + dim = context.get_input(2); + } + // default norm for matrix is frobenius norm, for vector - L2, for other ranks are not detrmined + if (context.input_is_none(1)) { + auto input_rank = x.get_partial_shape().rank(); + if (input_rank.is_static() && input_rank.get_length() == 2) { + result = frobenius_norm(context, x, dim, keep_dim); + } else if (input_rank.is_static() && input_rank.get_length() == 1) { + result = norm_vector(context, x, dim, 2, keep_dim); + } else { + FRONT_END_OP_CONVERSION_CHECK(false, + "linalg norm for tensor rank > 2 without ord specification unsupported"); + } + } else { + // ord defines the norm that is computed can be string or number + auto ord_type = context.get_input_type(1); + if (ord_type.is()) { + auto p_str = context.const_input(1); + if (p_str == "fro") { + result = frobenius_norm(context, x, dim, keep_dim); + } else { + FRONT_END_OP_CONVERSION_CHECK(false, "Umsupported ord ", p_str); + } + } else { + auto p = context.const_input(1); + if (!context.input_is_none(2)) { + auto const_dim = context.const_input>(2); + if (const_dim.size() == 2) { + result = norm_matrix(context, x, dim, p, keep_dim); + } else { + result = norm_vector(context, x, dim, p, keep_dim); + } + } else { + result = norm_vector(context, x, dim, p, keep_dim); + } + } + } + } + + // output tensor + if (!context.input_is_none(5)) { + context.mutate_input(5, result); + } + return {result}; +}; + } // namespace op } // namespace pytorch } // namespace frontend diff --git a/src/frontends/pytorch/src/op/numel.cpp b/src/frontends/pytorch/src/op/numel.cpp index 721ed7e173bdc6..a4d2a836c6a2bd 100644 --- a/src/frontends/pytorch/src/op/numel.cpp +++ b/src/frontends/pytorch/src/op/numel.cpp @@ -10,7 +10,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_numel(NodeContext& context) { +OutputVector translate_numel(const NodeContext& context) { num_inputs_check(context, 1, 1); return {numel(context, context.get_input(0))}; }; diff --git a/src/frontends/pytorch/src/op/pad.cpp b/src/frontends/pytorch/src/op/pad.cpp index 8a0568ece9cc3d..170544654542e5 100644 --- a/src/frontends/pytorch/src/op/pad.cpp +++ b/src/frontends/pytorch/src/op/pad.cpp @@ -22,7 +22,7 @@ namespace op { using namespace ov::op; -OutputVector translate_pad(NodeContext& context) { +OutputVector translate_pad(const NodeContext& context) { num_inputs_check(context, 2, 4); auto data = context.get_input(0); auto paddings = context.const_input>(1); diff --git a/src/frontends/pytorch/src/op/pow.cpp b/src/frontends/pytorch/src/op/pow.cpp index d418f3385340bf..d3a39694bf3953 100644 --- a/src/frontends/pytorch/src/op/pow.cpp +++ b/src/frontends/pytorch/src/op/pow.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_pow(NodeContext& context) { +OutputVector translate_pow(const NodeContext& context) { num_inputs_check(context, 2, 2); auto lhs = context.get_input(0); auto rhs = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/pythonop.cpp b/src/frontends/pytorch/src/op/pythonop.cpp index 36a4b388738c02..4aa142f04b58ed 100644 --- a/src/frontends/pytorch/src/op/pythonop.cpp +++ b/src/frontends/pytorch/src/op/pythonop.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_pythonop(NodeContext& context) { +OutputVector translate_pythonop(const NodeContext& context) { auto decoder = context.get_decoder(); FRONT_END_OP_CONVERSION_CHECK(decoder->get_subgraph_size() == 1, "PythonOp must have 1 subgraph to be able to translate it to OV."); diff --git a/src/frontends/pytorch/src/op/reciprocal.cpp b/src/frontends/pytorch/src/op/reciprocal.cpp index 67c5bdba78b39f..38b12fee06cb18 100644 --- a/src/frontends/pytorch/src/op/reciprocal.cpp +++ b/src/frontends/pytorch/src/op/reciprocal.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_reciprocal(NodeContext& context) { +OutputVector translate_reciprocal(const NodeContext& context) { num_inputs_check(context, 1, 1); auto x = context.get_input(0); auto const_neg_1 = context.mark_node(v0::Constant::create(element::i32, Shape{}, {-1})); diff --git a/src/frontends/pytorch/src/op/relu6.cpp b/src/frontends/pytorch/src/op/relu6.cpp index 5dd5906061bc7f..08996811249dcc 100644 --- a/src/frontends/pytorch/src/op/relu6.cpp +++ b/src/frontends/pytorch/src/op/relu6.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_relu6(NodeContext& context) { +OutputVector translate_relu6(const NodeContext& context) { num_inputs_check(context, 1, 1); auto x = context.get_input(0); return {context.mark_node(std::make_shared(x, 0., 6.))}; diff --git a/src/frontends/pytorch/src/op/remainder.cpp b/src/frontends/pytorch/src/op/remainder.cpp index 55d33e00c3f321..622e20eba52fa4 100644 --- a/src/frontends/pytorch/src/op/remainder.cpp +++ b/src/frontends/pytorch/src/op/remainder.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_remainder(NodeContext& context) { +OutputVector translate_remainder(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/repeat.cpp b/src/frontends/pytorch/src/op/repeat.cpp index 574951aaf82c20..15dc03a466ec92 100644 --- a/src/frontends/pytorch/src/op/repeat.cpp +++ b/src/frontends/pytorch/src/op/repeat.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_repeat(NodeContext& context) { +OutputVector translate_repeat(const NodeContext& context) { num_inputs_check(context, 2, 2); auto x = context.get_input(0); auto repeats = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/repeat_interleave.cpp b/src/frontends/pytorch/src/op/repeat_interleave.cpp index 06d8333e04cceb..64971f6e3f28bc 100644 --- a/src/frontends/pytorch/src/op/repeat_interleave.cpp +++ b/src/frontends/pytorch/src/op/repeat_interleave.cpp @@ -34,7 +34,7 @@ OutputVector generate_indices_from_repeats_tensor(const NodeContext& context, co }; } // namespace -OutputVector translate_repeat_interleave(NodeContext& context) { +OutputVector translate_repeat_interleave(const NodeContext& context) { num_inputs_check(context, 2, 3); // constants auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{}, {0})); diff --git a/src/frontends/pytorch/src/op/reshape.cpp b/src/frontends/pytorch/src/op/reshape.cpp index b0d669e47be63b..c5c33f4f6e61da 100644 --- a/src/frontends/pytorch/src/op/reshape.cpp +++ b/src/frontends/pytorch/src/op/reshape.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_reshape(NodeContext& context) { +OutputVector translate_reshape(const NodeContext& context) { // Translation is used by both aten::view and aten::reshape. // Schema: aten::view(Tensor input, int[] shape) -> Tensor // Schema: aten::reshape(Tensor input, int[] shape) -> Tensor diff --git a/src/frontends/pytorch/src/op/reshape_as.cpp b/src/frontends/pytorch/src/op/reshape_as.cpp index 63d18ee468f087..1c9be43d4a2d15 100644 --- a/src/frontends/pytorch/src/op/reshape_as.cpp +++ b/src/frontends/pytorch/src/op/reshape_as.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_reshape_as(NodeContext& context) { +OutputVector translate_reshape_as(const NodeContext& context) { num_inputs_check(context, 2, 2); auto input_tensor = context.get_input(0); auto shape_tesnor = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/roi_align.cpp b/src/frontends/pytorch/src/op/roi_align.cpp index d3a389c59654b9..fb2ad3a41d7455 100644 --- a/src/frontends/pytorch/src/op/roi_align.cpp +++ b/src/frontends/pytorch/src/op/roi_align.cpp @@ -19,7 +19,7 @@ namespace op { using namespace ov::op; -OutputVector translate_roi_align(NodeContext& context) { +OutputVector translate_roi_align(const NodeContext& context) { num_inputs_check(context, 7, 7); auto const_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {1})); auto const_neg_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {-1})); diff --git a/src/frontends/pytorch/src/op/roll.cpp b/src/frontends/pytorch/src/op/roll.cpp index b0aef51a6872b9..9f358368fbce8c 100644 --- a/src/frontends/pytorch/src/op/roll.cpp +++ b/src/frontends/pytorch/src/op/roll.cpp @@ -17,7 +17,7 @@ namespace op { using namespace ov::op; -OutputVector translate_roll(NodeContext& context) { +OutputVector translate_roll(const NodeContext& context) { num_inputs_check(context, 3, 3); const auto data = context.get_input(0); const auto shifts = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/rsqrt.cpp b/src/frontends/pytorch/src/op/rsqrt.cpp index 9e9ba9330c87ca..d4f56040da27c5 100644 --- a/src/frontends/pytorch/src/op/rsqrt.cpp +++ b/src/frontends/pytorch/src/op/rsqrt.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_rsqrt(NodeContext& context) { +OutputVector translate_rsqrt(const NodeContext& context) { num_inputs_check(context, 1, 1); auto data = context.get_input(0); auto input_shape = context.mark_node(std::make_shared(data, element::i32)); diff --git a/src/frontends/pytorch/src/op/rsub.cpp b/src/frontends/pytorch/src/op/rsub.cpp index 21b109e9037182..200094b6eecede 100644 --- a/src/frontends/pytorch/src/op/rsub.cpp +++ b/src/frontends/pytorch/src/op/rsub.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_rsub(NodeContext& context) { +OutputVector translate_rsub(const NodeContext& context) { num_inputs_check(context, 3, 3); auto self = context.get_input(0); auto other = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/select.cpp b/src/frontends/pytorch/src/op/select.cpp index c6d7cb0048f325..7cd898fdf223b5 100644 --- a/src/frontends/pytorch/src/op/select.cpp +++ b/src/frontends/pytorch/src/op/select.cpp @@ -5,11 +5,7 @@ #include "openvino/op/select.hpp" #include "openvino/frontend/pytorch/node_context.hpp" -#include "openvino/op/add.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/less.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/slice.hpp" +#include "openvino/op/gather.hpp" #include "openvino/op/squeeze.hpp" #include "utils.hpp" @@ -20,23 +16,13 @@ namespace op { using namespace ov::op; -OutputVector translate_select(NodeContext& context) { +OutputVector translate_select(const NodeContext& context) { + // aten::select.int(Tensor(a) self, int dim, SymInt index) -> Tensor(a) num_inputs_check(context, 3, 3); - auto const_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {1})); - auto const_minus_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {-1})); - auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); - - auto input_tensor = context.get_input(0); - auto dim = context.mark_node(std::make_shared(context.get_input(1), const_1, false)); - auto start = context.mark_node(std::make_shared(context.get_input(2), const_1, false)); - - auto less = context.mark_node(std::make_shared(start, const_0)); - auto const_1_signed = context.mark_node(std::make_shared(less, const_minus_1, const_1)); - auto stop = context.mark_node(std::make_shared(start, const_1_signed)); - - auto slice_node = context.mark_node(std::make_shared(input_tensor, start, stop, const_1_signed, dim)); - - return {context.mark_node(std::make_shared(slice_node, dim))}; + auto data = context.get_input(0); + auto dim = context.get_input(1); + auto index = context.get_input(2); + return {context.mark_node(std::make_shared(data, index, dim))}; }; } // namespace op diff --git a/src/frontends/pytorch/src/op/selu.cpp b/src/frontends/pytorch/src/op/selu.cpp index 9ec08af77facc8..aef54491e74bd1 100644 --- a/src/frontends/pytorch/src/op/selu.cpp +++ b/src/frontends/pytorch/src/op/selu.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_selu(NodeContext& context) { +OutputVector translate_selu(const NodeContext& context) { num_inputs_check(context, 1, 1); auto x = context.get_input(0); auto alpha = context.mark_node(v0::Constant::create(element::f64, Shape{}, {1.6732632423543772848170429916717})); diff --git a/src/frontends/pytorch/src/op/set_item.cpp b/src/frontends/pytorch/src/op/set_item.cpp index 9ce33fce24e8d2..ef11a2a391c39a 100644 --- a/src/frontends/pytorch/src/op/set_item.cpp +++ b/src/frontends/pytorch/src/op/set_item.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_set_item(NodeContext& context) { +OutputVector translate_set_item(const NodeContext& context) { // schema: aten::_set_item.t(t[](a!) l, int idx, t(b -> *) el) -> t[](a!) // _set_item inserts element in list num_inputs_check(context, 3, 3); diff --git a/src/frontends/pytorch/src/op/size.cpp b/src/frontends/pytorch/src/op/size.cpp index a4d70cef19ad2c..289facd0fe7f44 100644 --- a/src/frontends/pytorch/src/op/size.cpp +++ b/src/frontends/pytorch/src/op/size.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_size(NodeContext& context) { +OutputVector translate_size(const NodeContext& context) { num_inputs_check(context, 1, 2); auto shape = context.mark_node(std::make_shared(context.get_input(0), element::i32)); if (context.input_is_none(1)) { diff --git a/src/frontends/pytorch/src/op/slice.cpp b/src/frontends/pytorch/src/op/slice.cpp index 756efc1590b796..391b1c834fd535 100644 --- a/src/frontends/pytorch/src/op/slice.cpp +++ b/src/frontends/pytorch/src/op/slice.cpp @@ -18,7 +18,7 @@ namespace op { using namespace ov::op; -OutputVector translate_slice(NodeContext& context) { +OutputVector translate_slice(const NodeContext& context) { // aten::slice.t(t[] l, int? start=None, int? end=None, int step=1) -> (t[]) // aten::slice.Tensor(Tensor(a) self, int dim=0, int? start=None, int? end=None, int step=1) -> (Tensor(a)) ov::Output dim; diff --git a/src/frontends/pytorch/src/op/softmax.cpp b/src/frontends/pytorch/src/op/softmax.cpp index 1b94a3560ad972..10c3afea7cda0b 100644 --- a/src/frontends/pytorch/src/op/softmax.cpp +++ b/src/frontends/pytorch/src/op/softmax.cpp @@ -13,7 +13,7 @@ namespace pytorch { namespace op { using namespace ov::op; -OutputVector translate_softmax(NodeContext& context) { +OutputVector translate_softmax(const NodeContext& context) { num_inputs_check(context, 2, 3); auto x = context.get_input(0); auto axis = context.const_input(1); diff --git a/src/frontends/pytorch/src/op/sort.cpp b/src/frontends/pytorch/src/op/sort.cpp index c0e54d54d9be2a..715a7a52d3efe8 100644 --- a/src/frontends/pytorch/src/op/sort.cpp +++ b/src/frontends/pytorch/src/op/sort.cpp @@ -9,7 +9,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_sort(NodeContext& context) { +OutputVector translate_sort(const NodeContext& context) { num_inputs_check(context, 3, 4); const auto input_tensor = context.get_input(0); bool stable, descending; @@ -40,7 +40,7 @@ OutputVector translate_sort(NodeContext& context) { return topk->outputs(); }; -OutputVector translate_argsort(NodeContext& context) { +OutputVector translate_argsort(const NodeContext& context) { auto sort = translate_sort(context); return {sort[1]}; }; diff --git a/src/frontends/pytorch/src/op/square.cpp b/src/frontends/pytorch/src/op/square.cpp index 7194aafd9abb36..2310fda75aa574 100644 --- a/src/frontends/pytorch/src/op/square.cpp +++ b/src/frontends/pytorch/src/op/square.cpp @@ -14,7 +14,7 @@ namespace op { using namespace ov::op; -OutputVector translate_square(NodeContext& context) { +OutputVector translate_square(const NodeContext& context) { num_inputs_check(context, 1, 1); auto input_0 = context.get_input(0); auto const_2 = context.mark_node(v0::Constant::create(input_0.get_element_type(), Shape{1}, {2})); diff --git a/src/frontends/pytorch/src/op/squeeze.cpp b/src/frontends/pytorch/src/op/squeeze.cpp index dacf2c55a4d596..fb15801367a564 100644 --- a/src/frontends/pytorch/src/op/squeeze.cpp +++ b/src/frontends/pytorch/src/op/squeeze.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_squeeze(NodeContext& context) { +OutputVector translate_squeeze(const NodeContext& context) { num_inputs_check(context, 1, 2); auto x = context.get_input(0); if (context.input_is_none(1)) { diff --git a/src/frontends/pytorch/src/op/sub.cpp b/src/frontends/pytorch/src/op/sub.cpp index fd449c12bbd2d3..94963ed9bdb61f 100644 --- a/src/frontends/pytorch/src/op/sub.cpp +++ b/src/frontends/pytorch/src/op/sub.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_sub(NodeContext& context) { +OutputVector translate_sub(const NodeContext& context) { num_inputs_check(context, 2, 3); auto x = context.get_input(0); auto y = context.get_input(1); diff --git a/src/frontends/pytorch/src/op/sum.cpp b/src/frontends/pytorch/src/op/sum.cpp index 3dc4601b1083a9..7a87dc0c507f6c 100644 --- a/src/frontends/pytorch/src/op/sum.cpp +++ b/src/frontends/pytorch/src/op/sum.cpp @@ -11,7 +11,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_sum(NodeContext& context) { +OutputVector translate_sum(const NodeContext& context) { num_inputs_check(context, 1, 3); bool keep_dims = false; ov::Output axes; diff --git a/src/frontends/pytorch/src/op/to.cpp b/src/frontends/pytorch/src/op/to.cpp index 6e5b0ebda639c4..2499b8346f5f02 100644 --- a/src/frontends/pytorch/src/op/to.cpp +++ b/src/frontends/pytorch/src/op/to.cpp @@ -16,7 +16,7 @@ namespace op { using namespace ov::op; -OutputVector translate_to(NodeContext& context) { +OutputVector translate_to(const NodeContext& context) { int dtype_idx; int memory_format_idx; if (context.get_input_size() == 5) { diff --git a/src/frontends/pytorch/src/op/topk.cpp b/src/frontends/pytorch/src/op/topk.cpp index 26addb856c6445..06916c4ea03e2f 100644 --- a/src/frontends/pytorch/src/op/topk.cpp +++ b/src/frontends/pytorch/src/op/topk.cpp @@ -15,7 +15,7 @@ namespace op { using namespace ov::op; -OutputVector translate_topk(NodeContext& context) { +OutputVector translate_topk(const NodeContext& context) { num_inputs_check(context, 5, 5); const auto input_tensor = context.get_input(0); const auto largest = context.const_input(3); diff --git a/src/frontends/pytorch/src/op/transpose.cpp b/src/frontends/pytorch/src/op/transpose.cpp index 60fee576613374..9a6cddb3ffb896 100644 --- a/src/frontends/pytorch/src/op/transpose.cpp +++ b/src/frontends/pytorch/src/op/transpose.cpp @@ -20,7 +20,7 @@ namespace op { using namespace ov::op; -OutputVector translate_transpose(NodeContext& context) { +OutputVector translate_transpose(const NodeContext& context) { num_inputs_check(context, 3, 3); auto dim0 = context.const_input(1); auto dim1 = context.const_input(2); diff --git a/src/frontends/pytorch/src/op/trilu.cpp b/src/frontends/pytorch/src/op/trilu.cpp index 1726cf2f895956..1ef4d50fd6affa 100644 --- a/src/frontends/pytorch/src/op/trilu.cpp +++ b/src/frontends/pytorch/src/op/trilu.cpp @@ -60,11 +60,11 @@ OutputVector translate_base_triu_tril(const NodeContext& context, bool upper) { } }; // namespace -OutputVector translate_triu(NodeContext& context) { +OutputVector translate_triu(const NodeContext& context) { return translate_base_triu_tril(context, true); }; -OutputVector translate_tril(NodeContext& context) { +OutputVector translate_tril(const NodeContext& context) { return translate_base_triu_tril(context, false); }; diff --git a/src/frontends/pytorch/src/op/unfold.cpp b/src/frontends/pytorch/src/op/unfold.cpp index 949f7991391b7e..e7aa129b2935cc 100644 --- a/src/frontends/pytorch/src/op/unfold.cpp +++ b/src/frontends/pytorch/src/op/unfold.cpp @@ -13,7 +13,7 @@ namespace frontend { namespace pytorch { namespace op { -OutputVector translate_unfold(NodeContext& context) { +OutputVector translate_unfold(const NodeContext& context) { num_inputs_check(context, 4, 4); // constants auto const_0 = context.mark_node(Constant::create(element::i32, Shape{}, {0})); diff --git a/src/frontends/pytorch/src/op/upsample.cpp b/src/frontends/pytorch/src/op/upsample.cpp index 111a07a28c70e9..484387b8f3931d 100644 --- a/src/frontends/pytorch/src/op/upsample.cpp +++ b/src/frontends/pytorch/src/op/upsample.cpp @@ -69,32 +69,32 @@ OutputVector base_translate_upsample(const NodeContext& context, }; } // namespace -OutputVector translate_upsample_linear1d(NodeContext& context) { +OutputVector translate_upsample_linear1d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::LINEAR_ONNX, 1); }; -OutputVector translate_upsample_bilinear2d(NodeContext& context) { +OutputVector translate_upsample_bilinear2d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::LINEAR_ONNX, 2); }; -OutputVector translate_upsample_trilinear3d(NodeContext& context) { +OutputVector translate_upsample_trilinear3d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::LINEAR_ONNX, 3); }; -OutputVector translate_upsample_nearest1d(NodeContext& context) { +OutputVector translate_upsample_nearest1d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::NEAREST, 1); }; -OutputVector translate_upsample_nearest2d(NodeContext& context) { +OutputVector translate_upsample_nearest2d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::NEAREST, 2); }; -OutputVector translate_upsample_nearest3d(NodeContext& context) { +OutputVector translate_upsample_nearest3d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::NEAREST, 3); }; // bicubic is only supported for 2d in pytorch -OutputVector translate_upsample_bicubic2d(NodeContext& context) { +OutputVector translate_upsample_bicubic2d(const NodeContext& context) { return base_translate_upsample(context, v4::Interpolate::InterpolateMode::CUBIC, 2); }; diff --git a/src/frontends/pytorch/src/op/var_mean.cpp b/src/frontends/pytorch/src/op/var_mean.cpp index 936038fecdcc2c..f021161722cd39 100644 --- a/src/frontends/pytorch/src/op/var_mean.cpp +++ b/src/frontends/pytorch/src/op/var_mean.cpp @@ -20,7 +20,7 @@ namespace op { using namespace ov::op; -OutputVector translate_var_mean(NodeContext& context) { +OutputVector translate_var_mean(const NodeContext& context) { num_inputs_check(context, 1, 4); auto data = context.get_input(0); bool unbiased = true; @@ -75,7 +75,7 @@ OutputVector translate_var_mean(NodeContext& context) { return {var, mean}; }; -OutputVector translate_var(NodeContext& context) { +OutputVector translate_var(const NodeContext& context) { auto res = translate_var_mean(context); return {res[0]}; } diff --git a/src/frontends/pytorch/src/op/where.cpp b/src/frontends/pytorch/src/op/where.cpp index 454d23938a2c0c..4a9de9f69edab8 100644 --- a/src/frontends/pytorch/src/op/where.cpp +++ b/src/frontends/pytorch/src/op/where.cpp @@ -14,7 +14,7 @@ namespace op { using namespace ov::op; -OutputVector translate_where(NodeContext& context) { +OutputVector translate_where(const NodeContext& context) { num_inputs_check(context, 1, 3); auto cond = context.get_input(0); FRONT_END_OP_CONVERSION_CHECK(!context.input_is_none(1), "aten::where(cond) unsupported"); diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp index 92fc0741e43cdd..84087e2c9b8318 100644 --- a/src/frontends/pytorch/src/op_table.cpp +++ b/src/frontends/pytorch/src/op_table.cpp @@ -12,7 +12,7 @@ namespace frontend { namespace pytorch { namespace op { -#define OP_CONVERTER(op) OutputVector op(NodeContext& node) +#define OP_CONVERTER(op) OutputVector op(const NodeContext& node) OP_CONVERTER(translate_adaptive_avg_pool3d); OP_CONVERTER(translate_adaptive_max_pool2d); @@ -34,6 +34,7 @@ OP_CONVERTER(translate_convnd); OP_CONVERTER(translate_convolution); OP_CONVERTER(translate_convolution_mode); OP_CONVERTER(translate_cumsum); +OP_CONVERTER(translate_deform_conv); OP_CONVERTER(translate_dim); OP_CONVERTER(translate_div); OP_CONVERTER(translate_elu); @@ -62,6 +63,9 @@ OP_CONVERTER(translate_instance_norm); OP_CONVERTER(translate_int); OP_CONVERTER(translate_layer_norm); OP_CONVERTER(translate_len); +OP_CONVERTER(translate_linalg_norm); +OP_CONVERTER(translate_linalg_matrix_norm); +OP_CONVERTER(translate_linalg_vector_norm); OP_CONVERTER(translate_linear); OP_CONVERTER(translate_list_construct); OP_CONVERTER(translate_log); @@ -130,7 +134,7 @@ OP_CONVERTER(translate_zeros_like); } // namespace op -const std::map get_supported_ops() { +const std::map get_supported_ops() { return { {"aten::__and__", op::translate_1to1_match_2_inputs}, // TODO: cover numerical cases {"aten::__getitem__", op::translate_getitem}, @@ -237,6 +241,9 @@ const std::map get_supported_ops() { {"aten::leaky_relu", op::translate_1to1_match_2_inputs}, {"aten::leaky_relu_", op::inplace_op>}, {"aten::len", op::translate_len}, + {"aten::linalg_norm", op::translate_linalg_norm}, + {"aten::linalg_matrix_norm", op::translate_linalg_matrix_norm}, + {"aten::linalg_vector_norm", op::translate_linalg_vector_norm}, {"aten::linear", op::translate_linear}, {"aten::log", op::translate_log}, {"aten::log_", op::inplace_op}, @@ -351,6 +358,7 @@ const std::map get_supported_ops() { {"prim::requires_grad", op::return_false_scalar}, {"prim::PythonOp", op::translate_pythonop}, {"prim::type", op::skip_node}, // Used with prim::device, pass PtFrameworkNode. + {"torchvision::deform_conv2d", op::translate_deform_conv}, {"torchvision::nms", op::translate_nms}, {"torchvision::roi_align", op::translate_roi_align}, }; diff --git a/src/frontends/pytorch/src/op_table.hpp b/src/frontends/pytorch/src/op_table.hpp index 7a67c9101578b0..e15a988e98175b 100644 --- a/src/frontends/pytorch/src/op_table.hpp +++ b/src/frontends/pytorch/src/op_table.hpp @@ -10,7 +10,7 @@ namespace ov { namespace frontend { namespace pytorch { -const std::map get_supported_ops(); +const std::map get_supported_ops(); } // namespace pytorch } // namespace frontend diff --git a/src/frontends/pytorch/src/place.cpp b/src/frontends/pytorch/src/place.cpp index 0bf0da388ba0e0..9fbf55e5c4c94f 100644 --- a/src/frontends/pytorch/src/place.cpp +++ b/src/frontends/pytorch/src/place.cpp @@ -30,6 +30,11 @@ Place::Place(const ov::frontend::InputModel& input_model, size_t tensor_index) if (debug_name != m_names.at(0)) { m_names.push_back(debug_name); } + const auto& signature_name = + im->m_model_decoder->get_input_signature_name(std::distance(inputs.begin(), in_it)); + if (signature_name != m_names.at(0) && signature_name != debug_name) { + m_names.push_back(signature_name); + } } auto out_it = std::find(outputs.begin(), outputs.end(), tensor_index); if (out_it != outputs.end()) { diff --git a/src/frontends/pytorch/src/place.hpp b/src/frontends/pytorch/src/place.hpp index 3f91d20ed83abd..540381efe7d74d 100644 --- a/src/frontends/pytorch/src/place.hpp +++ b/src/frontends/pytorch/src/place.hpp @@ -33,6 +33,14 @@ class Place : public ov::frontend::Place { return m_tensor_index; } + bool is_equal_data(const Ptr& another) const override { + const auto another_pt = dynamic_cast(another.get()); + if (!another_pt) { + return false; + } + return m_tensor_index == another_pt->get_tensor_index(); + } + private: const ov::frontend::InputModel& m_input_model; const size_t m_tensor_index; diff --git a/src/frontends/pytorch/src/transforms/aten_index_replacer.cpp b/src/frontends/pytorch/src/transforms/aten_index_replacer.cpp index 93d4214add1fdd..cf9bd5c9fa2d02 100644 --- a/src/frontends/pytorch/src/transforms/aten_index_replacer.cpp +++ b/src/frontends/pytorch/src/transforms/aten_index_replacer.cpp @@ -47,12 +47,12 @@ std::shared_ptr flatten(const Output& value, size_t axis) { } else { const auto value_shape = std::make_shared(value, element::i32); const auto value_rank = std::make_shared(value_shape, element::i32); - const auto axis_node = v0::Constant::create(element::i32, Shape{}, {axis}); - auto start = v0::Constant::create(element::i32, Shape{}, {0}); - auto step = v0::Constant::create(element::i32, Shape{}, {1}); + const auto axis_node = v0::Constant::create(element::i32, Shape{1}, {axis}); + auto start = v0::Constant::create(element::i32, Shape{1}, {0}); + auto step = v0::Constant::create(element::i32, Shape{1}, {1}); const auto first_part_dims = std::make_shared(value_shape, start, axis_node, step); auto zero = v0::Constant::create(element::i32, {}, {0}); - auto first_part_dims_length = std::make_shared(first_part_dims, zero, true); + auto first_part_dims_length = std::make_shared(first_part_dims, zero, true); auto remaining_part_length = v0::Constant::create(element::i32, {1}, {-1}); @@ -70,7 +70,7 @@ AtenIndexToSelect::AtenIndexToSelect() { if (!index_op) { return false; } - auto input_node = index_op->input_value(0).get_node_shared_ptr(); + auto input_node = index_op->input_value(0); auto indicies = index_op->input_value(1).get_node_shared_ptr(); auto list_indicies = cast_fw_node(indicies, "prim::ListConstruct"); if (list_indicies) { @@ -108,10 +108,10 @@ AtenIndexToSelect::AtenIndexToSelect() { continue; } } - auto id_dtype = ids[i].get_node_shared_ptr()->get_element_type(); + auto id_dtype = ids[i].get_element_type(); if (id_dtype == element::boolean || id_dtype == element::u8) { - auto idx = std::make_shared(ids[i], element::u8); - auto nonzero = std::make_shared(idx); + auto idx = std::make_shared(ids[i], element::u8); + auto nonzero = std::make_shared(idx, element::i32); auto input_order = v0::Constant::create(element::i32, Shape{2}, {1, 0}); auto masked_id = std::make_shared(nonzero, input_order); masked_indicies.push_back(masked_id); @@ -125,30 +125,32 @@ AtenIndexToSelect::AtenIndexToSelect() { // all indicies prim::Constant(None), return input as is if (advanced_ids.size() == 0) { - copy_runtime_info({index_op, input_node}, input_node); - replace_node(index_op, input_node); + replace_node(index_op, input_node.get_node_shared_ptr()); return true; } // perform gather for single element case if (advanced_ids.size() == 1) { auto index = masked_indicies[advanced_ids[0]]; - index = std::make_shared(index, element::i32); if (is_masked_bool[advanced_ids[0]]) { auto gather = std::make_shared(input_node, index); - copy_runtime_info({index_op, input_node, indicies}, gather); + copy_runtime_info({index_op, indicies}, gather); + gather->set_friendly_name(index_op->get_friendly_name()); replace_node(index_op, gather); return true; } + index = std::make_shared(index, element::i32); auto dim = v0::Constant::create(element::i32, Shape{}, {advanced_ids[0]}); auto gather = std::make_shared(input_node, index, dim); - copy_runtime_info({index_op, input_node, indicies}, gather); + copy_runtime_info({index_op, indicies}, gather); + gather->set_friendly_name(index_op->get_friendly_name()); replace_node(index_op, gather); return true; } auto adv_idx_count = advanced_ids.size(); - auto rank = input_node->get_input_partial_shape(0).rank(); + auto rank = input_node.get_partial_shape().rank(); + // index transformation supports only tensors with static rank if (rank.is_dynamic()) { - FRONT_END_CHECK_IMPLEMENTED(false, "indexing for tensor with dynamic rank is not implemented "); + return false; } auto input_shape = std::make_shared(input_node, element::i32); auto zero = v0::Constant::create(element::i32, Shape{}, {0}); @@ -166,9 +168,11 @@ AtenIndexToSelect::AtenIndexToSelect() { auto transposed_input = std::make_shared(input_node, transpose_dims); auto flatten_input = flatten(transposed_input, adv_idx_count); auto cum_adv_index = masked_indicies[advanced_ids[adv_idx_count - 1]]; + cum_adv_index = std::make_shared(cum_adv_index, element::i32); auto multiplier = input_dims->output(advanced_ids[adv_idx_count - 1]); - for (int i = static_cast(adv_idx_count) - 2; i > 0; i--) { - auto adv_index = std::make_shared(masked_indicies[i], multiplier); + for (int i = static_cast(adv_idx_count) - 2; i > -1; i--) { + auto m_idx = std::make_shared(masked_indicies[i], element::i32); + auto adv_index = std::make_shared(m_idx, multiplier); cum_adv_index = std::make_shared(cum_adv_index, adv_index); auto input_id = advanced_ids[i]; multiplier = std::make_shared(multiplier, input_dims->output(input_id)); @@ -204,7 +208,7 @@ AtenIndexToSelect::AtenIndexToSelect() { v0::Constant::create(element::i32, Shape{adv_idx_permute.size()}, adv_idx_permute); gather = std::make_shared(gather, permute_indicies); // unfold advanced index axes - for (size_t i = 0; i <= advanced_ids[0]; i++) { + for (size_t i = 0; i < advanced_ids[0]; i++) { concat_dims.push_back(input_dims->output(i)); } concat_dims.push_back(cum_adv_index_shape_tensor); @@ -223,8 +227,9 @@ AtenIndexToSelect::AtenIndexToSelect() { } auto final_shape = std::make_shared(concat_dims, 0); gather = std::make_shared(gather, final_shape, false); - copy_runtime_info({index_op, input_node, indicies}, gather); + copy_runtime_info({index_op, indicies}, gather); replace_node(index_op, gather); + gather->set_friendly_name(index_op->get_friendly_name()); return true; } else { @@ -234,28 +239,28 @@ AtenIndexToSelect::AtenIndexToSelect() { // index is None, stay input as is const auto& attrs = const_input->get_attrs(); if (attrs.find("none_value") != attrs.end()) { - copy_runtime_info({index_op, input_node, indicies}, input_node); - replace_node(index_op, input_node); + replace_node(index_op, input_node.get_node_shared_ptr()); return true; } } auto index_dtype = indicies->get_output_element_type(0); if (index_dtype == element::boolean || index_dtype == element::u8) { - auto nonzero = std::make_shared(indicies); + auto nonzero = std::make_shared(indicies, element::i32); auto input_order = v0::Constant::create(element::i32, Shape{2}, {1, 0}); auto masked_id = std::make_shared(nonzero, input_order); auto gather = std::make_shared(input_node, masked_id); - copy_runtime_info({index_op, input_node, indicies}, gather); + copy_runtime_info({index_op, indicies}, gather); replace_node(index_op, gather); return true; } - if (index_dtype != element::i32 && index_dtype != element::i32) { + if (index_dtype != element::i32) { indicies = std::make_shared(indicies, element::i32); } auto dim = v0::Constant::create(element::i32, Shape{}, {0}); auto gather = std::make_shared(input_node, indicies, dim); - copy_runtime_info({index_op, input_node, indicies}, gather); + copy_runtime_info({index_op, indicies}, gather); replace_node(index_op, gather); + gather->set_friendly_name(index_op->get_friendly_name()); return true; } return false; diff --git a/src/frontends/pytorch/src/transforms/listconstruct_replacer.cpp b/src/frontends/pytorch/src/transforms/listconstruct_replacer.cpp index a5496501dba283..6b1792f7a63e0d 100644 --- a/src/frontends/pytorch/src/transforms/listconstruct_replacer.cpp +++ b/src/frontends/pytorch/src/transforms/listconstruct_replacer.cpp @@ -17,6 +17,7 @@ #include "openvino/op/tile.hpp" #include "openvino/op/transpose.hpp" #include "openvino/op/util/framework_node.hpp" +#include "openvino/op/variadic_split.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" @@ -49,6 +50,8 @@ ListConstructReplacer::ListConstructReplacer() { auto tile_op = pattern::wrap_type({pattern::any_input(), list}); // replace aten::permute(tensor, prim::ListConstruct) auto transpose_op = pattern::wrap_type({pattern::any_input(), list}); + // aten::split_with_sizes case + auto vsplit_op = pattern::wrap_type({pattern::any_input(), pattern::any_input(), list}); auto lc_pattern = std::make_shared(OutputVector{reshape_op, roll_op, broadcast_op, @@ -57,7 +60,8 @@ ListConstructReplacer::ListConstructReplacer() { equal_op, select_op, tile_op, - transpose_op}); + transpose_op, + vsplit_op}); ov::matcher_pass_callback callback = [=](pattern::Matcher& m) { auto& pattern_map = m.get_pattern_value_map(); diff --git a/src/frontends/pytorch/src/transforms/min_max_prim_list_construct_replacer.cpp b/src/frontends/pytorch/src/transforms/min_max_prim_list_construct_replacer.cpp index eed50b174f263f..d4602ee162cbc4 100644 --- a/src/frontends/pytorch/src/transforms/min_max_prim_list_construct_replacer.cpp +++ b/src/frontends/pytorch/src/transforms/min_max_prim_list_construct_replacer.cpp @@ -49,7 +49,8 @@ MinMaxPrimListConstructReplacer::MinMaxPrimListConstructReplacer() { auto step = std::make_shared(element::i32, Shape{}, 1); auto shape = std::make_shared(input, element::i32); auto rank = std::make_shared(shape, element::i32); - auto reduced_rank = std::make_shared(rank); + auto axis_0 = ov::op::v0::Constant::create(element::i32, Shape{}, {0}); + auto reduced_rank = std::make_shared(rank, axis_0); auto axes = std::make_shared(start, reduced_rank, step, element::i32); std::shared_ptr reduce_op; if (!is_min) { diff --git a/src/frontends/pytorch/src/transforms/prim_list_unpack_replacer.cpp b/src/frontends/pytorch/src/transforms/prim_list_unpack_replacer.cpp index a4ba9a8b3cd4c3..931901b19337e1 100644 --- a/src/frontends/pytorch/src/transforms/prim_list_unpack_replacer.cpp +++ b/src/frontends/pytorch/src/transforms/prim_list_unpack_replacer.cpp @@ -33,6 +33,7 @@ PrimListUnpackReplacer::PrimListUnpackReplacer() { if (rank.is_dynamic()) { return false; } + std::shared_ptr split; if (rank.get_length() == 0) { // Create split_lenghts tensor from split_size int, // allow for last chunk to be smaller if data is not equally divisible. @@ -45,18 +46,17 @@ PrimListUnpackReplacer::PrimListUnpackReplacer() { auto split_lenghts_m_1 = std::make_shared(split_size, num_out_m_1); NodeVector concat_inputs{split_lenghts_m_1, const_neg_1}; auto split_lenghts = std::make_shared(concat_inputs, 0); - auto split = std::make_shared(torch_split->get_input_source_output(0), - torch_split->get_input_source_output(2), - split_lenghts); - copy_runtime_info({list_unpack, input_node}, split); - replace_node(list_unpack, split); + split = std::make_shared(torch_split->get_input_source_output(0), + torch_split->get_input_source_output(2), + split_lenghts); } else { - auto split = std::make_shared(torch_split->get_input_source_output(0), - torch_split->get_input_source_output(2), - torch_split->get_input_source_output(1)); - copy_runtime_info({list_unpack, input_node}, split); - replace_node(list_unpack, split); + split = std::make_shared(torch_split->get_input_source_output(0), + torch_split->get_input_source_output(2), + torch_split->get_input_source_output(1)); } + copy_runtime_info({list_unpack, input_node}, split); + split->set_friendly_name(input_node->get_friendly_name()); + replace_node(list_unpack, split); return true; } @@ -67,20 +67,58 @@ PrimListUnpackReplacer::PrimListUnpackReplacer() { split_with_sizes->get_input_source_output(1)); copy_runtime_info({list_unpack, input_node}, split); + split->set_friendly_name(input_node->get_friendly_name()); replace_node(list_unpack, split); return true; } if (auto chunk = cast_fw_node(input_node, "aten::chunk")) { - // Using number of ListUnpack outputs instead of 1st input to chunk. - // TODO: confirm it works for all cases - auto split = std::make_shared(chunk->get_input_source_output(0), - chunk->get_input_source_output(2), - list_unpack->get_output_size()); - - copy_runtime_info({list_unpack, input_node}, split); - replace_node(list_unpack, split); + auto input_tensor = chunk->get_input_source_output(0); + auto chunks_i32 = chunk->get_input_source_output(1); + auto dim = chunk->get_input_source_output(2); + + auto chunks = std::make_shared(chunks_i32, element::i64); + auto const_0 = opset10::Constant::create(element::i64, Shape{1}, {0}); + auto const_1 = opset10::Constant::create(element::i64, Shape{1}, {1}); + auto const_0_nodim = opset10::Constant::create(element::i64, Shape{}, {0}); + auto const_1_nodim = opset10::Constant::create(element::i64, Shape{}, {1}); + auto const_shape = opset10::Constant::create(element::i64, Shape{1}, {list_unpack->get_output_size()}); + + auto input_shape = std::make_shared(input_tensor); + auto input_dimension = std::make_shared(input_shape, dim, const_0); + auto input_size = std::make_shared(input_dimension); + + auto chunk_size = std::make_shared(input_size, chunks, true); + auto last_chunk_size = std::make_shared(input_size, chunks); + auto is_last_nonzero = std::make_shared(last_chunk_size, const_0_nodim); + auto is_last_nonzero_int = std::make_shared(is_last_nonzero, element::i64); + + auto computed_chunk_size = std::make_shared(chunk_size, is_last_nonzero_int); + auto computed_chunk_size_incr = std::make_shared(computed_chunk_size, const_1_nodim); + auto computed_last_chunk_size = std::make_shared(input_size, computed_chunk_size); + auto computed_is_last_nonzero = std::make_shared(computed_last_chunk_size, const_0_nodim); + auto computed_is_last_nonzero_int = + std::make_shared(computed_is_last_nonzero, element::i64); + auto computed_is_last_nonzero_int_unsq = + std::make_shared(computed_is_last_nonzero_int, const_0); + auto computed_chunks = std::make_shared(input_size, computed_chunk_size, true); + auto computed_chunks_unsq = std::make_shared(computed_chunks, const_0); + + auto chunk_lengths = std::make_shared(computed_chunks_unsq, + computed_chunk_size, + computed_chunk_size_incr, + element::i64); + auto split_lengths = std::make_shared(chunk_lengths, + const_0, + computed_is_last_nonzero_int_unsq, + computed_last_chunk_size, + ov::op::PadMode::CONSTANT); + auto split_lengths_static = std::make_shared(split_lengths, const_shape, false); + auto sliced_chunks = std::make_shared(input_tensor, dim, split_lengths_static); + + copy_runtime_info({list_unpack, input_node}, sliced_chunks); + replace_node(list_unpack, sliced_chunks); return true; } diff --git a/src/frontends/pytorch/src/translate_session.cpp b/src/frontends/pytorch/src/translate_session.cpp index e4ddd31e5be922..f8b50ee6cde437 100644 --- a/src/frontends/pytorch/src/translate_session.cpp +++ b/src/frontends/pytorch/src/translate_session.cpp @@ -20,11 +20,22 @@ namespace pytorch { using namespace ov::op; TranslateSession::TranslateSession(const ov::frontend::InputModel::Ptr& input_model, - const std::map& translator_map) + const std::map& translator_map, + const std::shared_ptr& telemetry) : m_input_model(input_model), m_translator_map(translator_map), + m_telemetry(telemetry), m_ov_model(nullptr) {} +TranslateSession::~TranslateSession() { + if (m_telemetry) { + // Send statistics + for (const auto& op : m_op_statistics) { + m_telemetry->send_event("op_count", "pytorch_" + op.first, static_cast(op.second)); + } + } +} + std::shared_ptr TranslateSession::get_converted_model() { if (m_ov_model) { return m_ov_model; @@ -50,9 +61,9 @@ std::shared_ptr TranslateSession::convert_pytorch_model( std::shared_ptr resulting_model; // define here to make a conversion in a nested scope { - ParameterVector parameters; - TensorMap tensor_map; // tensor map of the current context - std::set mutated_tensors; + auto parameters = std::make_shared(); + auto tensor_map = std::make_shared(); // tensor map of the current context + auto mutated_tensors = std::make_shared>(); // Go over all pytorch_model inputs and register them in the tensor map: auto inputs = pytorch_model->inputs(); @@ -78,8 +89,11 @@ std::shared_ptr TranslateSession::convert_pytorch_model( } if (!input_node) { auto parameter = std::make_shared(type, pshape); - encode_tensor_name(parameter->output(0), inputs.at(i), pytorch_model->get_input_debug_name(i)); - parameters.push_back(parameter); + encode_tensor_name( + parameter->output(0), + inputs.at(i), + {pytorch_model->get_input_debug_name(i), pytorch_model->get_input_signature_name(i)}); + parameters->push_back(parameter); input_node = parameter; auto order = pytorch_model->get_input_transpose_order(i); if (order.size() > 0 && !std::is_sorted(order.begin(), order.end())) { @@ -96,7 +110,7 @@ std::shared_ptr TranslateSession::convert_pytorch_model( input_node = transpose; } } - tensor_map[inputs.at(i)] = input_node; + (*tensor_map)[inputs.at(i)] = input_node; } auto node_visitor = [&](std::shared_ptr node) { @@ -108,54 +122,59 @@ std::shared_ptr TranslateSession::convert_pytorch_model( for (size_t i = 0; i < inlined_inputs.size(); ++i) { size_t fw_tensor_id = inlined_nodes_counter; ++inlined_nodes_counter; // TODO: Make sure that Decoder side use the same increment for producing ids - if (tensor_map.find(fw_tensor_id) != tensor_map.end()) { + if (tensor_map->find(fw_tensor_id) != tensor_map->end()) { throw std::runtime_error("Duplicated producer for PT value with unique ID: " + std::to_string(fw_tensor_id) + " produced by inlined_inputs"); } - tensor_map[fw_tensor_id] = inlined_inputs[i]; + (*tensor_map)[fw_tensor_id] = inlined_inputs[i]; } auto raw_inputs = node->inputs(); for (size_t i = 0; i < raw_inputs.size(); ++i) { auto input = raw_inputs.at(i); - if (tensor_map.find(input) == tensor_map.end()) { + if (tensor_map->find(input) == tensor_map->end()) { // Input refers value in the outer scope, need to create a new Parameter in the current scope // Linkage to external scope will be performed on the level of the parent operation (if or loop) // TODO: Eliminate duplication with the main code for Parameters creation PartialShape ps = node->get_input_shape(i); - auto type = simplified_type_interpret(node->get_input_type(i)).as(); + + auto type = simplified_type_interpret(node->get_input_type(i)); // TODO: Use special API to set custom type specification - auto parameter = std::make_shared(type, ps); + std::shared_ptr parameter; + // TODO: Use decoder type or explore adding the missing cast types to Torchscript path + if (std::strcmp(std::getenv("PYTORCH_TRACING_MODE"),"TORCHFX") == 0) + parameter = std::make_shared(type.as(), ps); + else + parameter = std::make_shared(element::dynamic, ps); // TODO: Missing get_input_transpose_order handling for not trivial layouts - tensor_map[input] = parameter; + (*tensor_map)[input] = parameter; // set name of parameter to the index of node in the model encode_tensor_name(parameter->output(0), input); - parameters.push_back(parameter); + parameters->push_back(parameter); } } - auto context = NodeContext(node, &tensor_map, ¶meters, external_tensor_map, this); + auto context = NodeContext(node, external_tensor_map, tensor_map, parameters, mutated_tensors, this); + // Add op type in the statistics + m_op_statistics[context.get_op_type()]++; auto converted_outputs = convert_node(context); - auto mutated_t = context.get_mutated_tensors(); - mutated_tensors.insert(mutated_t.begin(), mutated_t.end()); - auto fw_outputs = node->outputs(); // Ops with subgraphs or with mutated inputs may have more outputs after conversion compared to pytorch ones FRONT_END_OP_CONVERSION_CHECK(fw_outputs.size() <= converted_outputs.size(), "Number of ", - node->get_op_type(), + context.get_op_type(), " outputs greater then number of converted outputs."); // TODO: Make sure that mapping of fw_outputs to converted_outputs does always work // FIXME: Now it is not true for at least prim::Constant for (size_t i = 0; i < fw_outputs.size(); ++i) { size_t fw_tensor_id = node->output(i); - FRONT_END_GENERAL_CHECK(tensor_map.find(fw_tensor_id) == tensor_map.end(), + FRONT_END_GENERAL_CHECK(tensor_map->find(fw_tensor_id) == tensor_map->end(), "Duplicated producer for PT value with unique ID: ", fw_tensor_id); - tensor_map[fw_tensor_id] = converted_outputs[i]; - encode_tensor_name(converted_outputs[i], fw_tensor_id, node->get_output_debug_name(i)); + (*tensor_map)[fw_tensor_id] = converted_outputs[i]; + encode_tensor_name(converted_outputs[i], fw_tensor_id, {node->get_output_debug_name(i)}); } }; @@ -165,14 +184,14 @@ std::shared_ptr TranslateSession::convert_pytorch_model( ResultVector results; for (size_t i = 0; i < pytorch_model->num_of_outputs(); ++i) { size_t id = pytorch_model->output(i); - if (tensor_map.find(id) == tensor_map.end()) { + if (tensor_map->find(id) == tensor_map->end()) { // Not found in this scope, adding Parameter to connect to external scope auto parameter = std::make_shared(element::dynamic, PartialShape::dynamic()); encode_tensor_name(parameter->output(0), id); - parameters.push_back(parameter); - tensor_map[id] = parameter; + parameters->push_back(parameter); + (*tensor_map)[id] = parameter; } - auto ov_output = tensor_map[id]; + auto ov_output = tensor_map->at(id); auto order = pytorch_model->get_output_transpose_order(i); FRONT_END_GENERAL_CHECK(order.size() == 0 || std::is_sorted(order.begin(), order.end()), "Output strides have wrong order."); @@ -185,32 +204,32 @@ std::shared_ptr TranslateSession::convert_pytorch_model( // Since parameters can be added we need to list all current parameters std::set param_names; - for (const auto& param : parameters) { + for (const auto& param : *parameters) { auto input_idx = decode_tensor_name(param->output(0)); param_names.insert(input_idx); } - for (const auto& tensor_id : mutated_tensors) { + for (const auto& tensor_id : *mutated_tensors) { if (param_names.count(tensor_id)) { - FRONT_END_GENERAL_CHECK(tensor_map.count(tensor_id), + FRONT_END_GENERAL_CHECK(tensor_map->count(tensor_id), "Tensor with id: ", tensor_id, " doesn't exist in tensor map."); // model input was mutated we need to make a result for it - auto mutated_tensor = tensor_map.at(tensor_id); + auto mutated_tensor = tensor_map->at(tensor_id); // empty external_tensor_map means this is main body of the model and we don't want to create // additional outputs in that case. if (mutated_tensor.get_target_inputs().empty() && !external_tensor_map.empty()) - results.push_back(std::make_shared(tensor_map.at(tensor_id))); + results.push_back(std::make_shared(tensor_map->at(tensor_id))); } } - resulting_model = std::make_shared(results, parameters); + resulting_model = std::make_shared(results, *parameters); // Did a conversion in a nested scope to automatically remove any holders of nodes except those in the graph } return resulting_model; } -OutputVector TranslateSession::convert_node(NodeContext& context) { +OutputVector TranslateSession::convert_node(const NodeContext& context) { try { auto it = m_translator_map.find(context.get_op_type()); if (it != m_translator_map.end()) { @@ -228,32 +247,29 @@ OutputVector TranslateSession::convert_node(NodeContext& context) { return make_framework_node(context); } -void TranslateSession::encode_tensor_name(Output output, size_t tensor_idx, std::string debug_name) { +void TranslateSession::encode_tensor_name(Output output, + size_t tensor_idx, + std::vector additional_names) { if (!output.get_names().empty()) { OPENVINO_DEBUG << "Tensor names already exist: " << output.get_any_name() << ". Rewriting with " << tensor_idx; } - auto has_dname = !debug_name.empty(); auto name = std::to_string(tensor_idx); - if (has_dname && name == debug_name) - has_dname = false; + std::unordered_set names; + names.insert(name); + if (additional_names.size() > 0) { + names.insert(additional_names.begin(), additional_names.end()); + } if (m_counter_map.count(tensor_idx)) { auto&& pair = m_counter_map[tensor_idx]; auto new_name = name + '_' + std::to_string(++pair.first); pair.second.set_names({new_name}); pair.second = output; - if (has_dname) { - output.set_names({name, debug_name}); - } else { - output.set_names({name}); - } + output.set_names(names); + } else { m_counter_map[tensor_idx] = {0, output}; - if (has_dname) { - output.set_names({name, debug_name}); - } else { - output.set_names({name}); - } + output.set_names(names); } } diff --git a/src/frontends/pytorch/src/translate_session.hpp b/src/frontends/pytorch/src/translate_session.hpp index e33ea31c63091d..fb1c06a48e8ce8 100644 --- a/src/frontends/pytorch/src/translate_session.hpp +++ b/src/frontends/pytorch/src/translate_session.hpp @@ -5,6 +5,7 @@ #pragma once #include "input_model.hpp" +#include "openvino/frontend/extension/telemetry.hpp" #include "openvino/frontend/pytorch/node_context.hpp" namespace ov { @@ -17,7 +18,9 @@ namespace pytorch { class TranslateSession { public: TranslateSession(const frontend::InputModel::Ptr& input_model, - const std::map& translator_map); + const std::map& translator_map, + const std::shared_ptr& telemetry); + ~TranslateSession(); std::shared_ptr get_converted_model(); std::shared_ptr translate_graph(const frontend::InputModel::Ptr& input_model); @@ -32,19 +35,23 @@ class TranslateSession { const TensorMap& external_tensor_map = {}, const std::unordered_map& external_descriptors = {}); - void encode_tensor_name(Output tensor_desc, size_t tensor_idx, std::string debug_name = ""); + void encode_tensor_name(Output tensor_desc, + size_t tensor_idx, + std::vector additional_names = {}); size_t decode_tensor_name(const Output& tensor_desc); size_t m_friendly_name_counter = 0; private: - OutputVector convert_node(NodeContext& context); + OutputVector convert_node(const NodeContext& context); const frontend::InputModel::Ptr m_input_model; - const std::map& m_translator_map; - + const std::map& m_translator_map; + std::shared_ptr m_telemetry; std::shared_ptr m_ov_model; + std::map>> m_counter_map; + std::map m_op_statistics; }; } // namespace pytorch diff --git a/src/frontends/pytorch/src/utils.cpp b/src/frontends/pytorch/src/utils.cpp index b4027e7c04147f..42592dd65345db 100644 --- a/src/frontends/pytorch/src/utils.cpp +++ b/src/frontends/pytorch/src/utils.cpp @@ -66,7 +66,8 @@ std::tuple, Output> get_shape_rank(const NodeContext& context auto shape = context.mark_node(std::make_shared(x, output_type)); Output rank = context.mark_node(std::make_shared(shape, output_type)); if (as_scalar) { - rank = context.mark_node(std::make_shared(rank)); + auto axis_0 = context.mark_node(opset10::Constant::create(output_type, Shape{}, {0})); + rank = context.mark_node(std::make_shared(rank, axis_0)); } return std::make_tuple(shape, rank); } @@ -110,9 +111,8 @@ std::shared_ptr get_axes_range(const NodeContext& context, int input_id) { auto x = context.get_input(input_id); auto start = std::make_shared(element::i32, Shape{}, 0); auto step = std::make_shared(element::i32, Shape{}, 1); - auto shape = context.mark_node(std::make_shared(x, element::i32)); - auto rank = context.mark_node(std::make_shared(shape, element::i32)); - auto reduced_rank = context.mark_node(std::make_shared(rank)); + Output reduced_rank; + std::tie(std::ignore, reduced_rank) = get_shape_rank(context, x, true); return context.mark_node(std::make_shared(start, reduced_rank, step, element::i32)); }; @@ -177,7 +177,7 @@ std::shared_ptr concat_list_construct(std::shared_ptr input) { return input; } -OutputVector make_framework_node(NodeContext& context) { +OutputVector make_framework_node(const NodeContext& context) { auto schema = context.get_schema(); // TODO: properly process schema to get the actual position of mutable input // Hack. Can indicate mutable inputs, but can it be reliable? diff --git a/src/frontends/pytorch/src/utils.hpp b/src/frontends/pytorch/src/utils.hpp index 07235a2152d8b0..bf83fa95ac3506 100644 --- a/src/frontends/pytorch/src/utils.hpp +++ b/src/frontends/pytorch/src/utils.hpp @@ -48,7 +48,7 @@ op::PadType convert_pad(const std::string& pt_pad); std::shared_ptr concat_list_construct(std::shared_ptr input); -OutputVector make_framework_node(NodeContext& context); +OutputVector make_framework_node(const NodeContext& context); std::shared_ptr cast_fw_node(std::shared_ptr node, const std::string& type); @@ -63,8 +63,8 @@ void align_eltwise_input_types(const NodeContext& context, std::deque> get_list_as_outputs(const Output& start); namespace op { -template -OutputVector inplace_op(NodeContext& context) { +template +OutputVector inplace_op(const NodeContext& context) { auto translation_res = T(context); FRONT_END_OP_CONVERSION_CHECK(translation_res.size() == 1, "inplace_op function must be used on single output translators"); @@ -73,34 +73,34 @@ OutputVector inplace_op(NodeContext& context) { } template -OutputVector translate_1to1_match_1_inputs(NodeContext& context) { +OutputVector translate_1to1_match_1_inputs(const NodeContext& context) { num_inputs_check(context, 1, 1); FRONT_END_OP_CONVERSION_CHECK(!context.input_is_none(0), "Input should not be None."); return {context.mark_node(std::make_shared(context.get_input(0)))}; } template -OutputVector translate_1to1_match_2_inputs(NodeContext& context) { +OutputVector translate_1to1_match_2_inputs(const NodeContext& context) { num_inputs_check(context, 2, 2); FRONT_END_OP_CONVERSION_CHECK(!context.input_is_none(0) && !context.input_is_none(1), "Inputs should not be None."); return {context.mark_node(std::make_shared(context.get_input(0), context.get_input(1)))}; } template -OutputVector translate_1to1_match_2_inputs_align_types(NodeContext& context) { +OutputVector translate_1to1_match_2_inputs_align_types(const NodeContext& context) { num_inputs_check(context, 2, 2); FRONT_END_OP_CONVERSION_CHECK(!context.input_is_none(0) && !context.input_is_none(1), "Inputs should not be None."); auto lhs = context.get_input(0); auto rhs = context.get_input(1); - align_eltwise_input_types(context, lhs, rhs); + align_eltwise_input_types(context, lhs, rhs, true); return {context.mark_node(std::make_shared(lhs, rhs))}; } -inline OutputVector return_false_scalar(NodeContext& context) { +inline OutputVector return_false_scalar(const NodeContext& context) { return {context.mark_node(ov::op::v0::Constant::create(element::boolean, Shape{}, {false}))}; } -inline OutputVector skip_node(NodeContext& context) { +inline OutputVector skip_node(const NodeContext& context) { return {context.get_input(0).get_node_shared_ptr()}; } diff --git a/src/frontends/tensorflow/include/openvino/frontend/tensorflow/extension/conversion.hpp b/src/frontends/tensorflow/include/openvino/frontend/tensorflow/extension/conversion.hpp index e27516b60cb1be..d17cc5083e756d 100644 --- a/src/frontends/tensorflow/include/openvino/frontend/tensorflow/extension/conversion.hpp +++ b/src/frontends/tensorflow/include/openvino/frontend/tensorflow/extension/conversion.hpp @@ -23,14 +23,19 @@ class TENSORFLOW_API ConversionExtension : public ConversionExtensionBase { : ConversionExtensionBase(op_type), m_converter(converter) {} - const ov::frontend::CreatorFunction& get_converter() const { + ConversionExtension(const std::string& op_type, + const ov::frontend::tensorflow::CreatorFunctionNamedAndIndexed& converter) + : ConversionExtensionBase(op_type), + m_converter(converter) {} + + const ov::frontend::tensorflow::CreatorFunction& get_converter() const { return m_converter; } ~ConversionExtension() override; private: - ov::frontend::CreatorFunction m_converter; + ov::frontend::tensorflow::CreatorFunction m_converter; }; } // namespace tensorflow diff --git a/src/frontends/tensorflow/include/openvino/frontend/tensorflow/node_context.hpp b/src/frontends/tensorflow/include/openvino/frontend/tensorflow/node_context.hpp index 1d002270cf7d8c..fb26992d753c60 100644 --- a/src/frontends/tensorflow/include/openvino/frontend/tensorflow/node_context.hpp +++ b/src/frontends/tensorflow/include/openvino/frontend/tensorflow/node_context.hpp @@ -68,7 +68,31 @@ class NodeContext : public ov::frontend::NodeContext { const OutputVector& m_inputs; }; -using CreatorFunction = std::function; +using CreatorFunctionIndexed = std::function; +using CreatorFunctionNamedAndIndexed = std::function; + +class CreatorFunction { +public: + CreatorFunction() = default; + CreatorFunction(CreatorFunctionIndexed _func) : func_indexed(_func) {} + CreatorFunction(CreatorFunctionNamedAndIndexed _func) : func_named_and_indexed(_func) {} + + NamedOutputVector operator()(const ov::frontend::tensorflow::NodeContext& node) const { + if (func_indexed) { + auto outputs = func_indexed(node); + return NamedOutputVector(outputs.begin(), outputs.end()); + } else if (func_named_and_indexed) { + return func_named_and_indexed(node); + } else { + FRONT_END_GENERAL_CHECK(false, "No conversion function exist in this CreatorFunction"); + } + } + +private: + CreatorFunctionIndexed func_indexed = nullptr; + CreatorFunctionNamedAndIndexed func_named_and_indexed = nullptr; +}; + using TranslatorDictionaryType = std::map; } // namespace tensorflow diff --git a/src/frontends/tensorflow/src/CMakeLists.txt b/src/frontends/tensorflow/src/CMakeLists.txt index 42eb26e93c42cd..af1cb8b00cdbd0 100644 --- a/src/frontends/tensorflow/src/CMakeLists.txt +++ b/src/frontends/tensorflow/src/CMakeLists.txt @@ -6,3 +6,8 @@ ov_add_frontend(NAME tensorflow LINKABLE_FRONTEND FILEDESCRIPTION "FrontEnd to load and convert TensorFlow file format" LINK_LIBRARIES openvino::core::dev openvino::frontend::tensorflow_common) + +if(ENABLE_SNAPPY_COMPRESSION) + target_link_libraries(openvino_tensorflow_frontend PRIVATE Snappy::snappy) + target_compile_definitions(openvino_tensorflow_frontend PRIVATE ENABLE_SNAPPY_COMPRESSION) +endif() diff --git a/src/frontends/tensorflow/src/decoder_argdef.cpp b/src/frontends/tensorflow/src/decoder_argdef.cpp index 4f59f851dff980..cf3cb3379b1ab4 100644 --- a/src/frontends/tensorflow/src/decoder_argdef.cpp +++ b/src/frontends/tensorflow/src/decoder_argdef.cpp @@ -32,22 +32,14 @@ const std::string& DecoderArgDef::get_op_name() const { void DecoderArgDef::get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const { // Body graph nodes may have two colons `:`, for example, // producer_name:z:2 means that producer operation name is `producer_name` // and output port is 2 FRONT_END_GENERAL_CHECK(m_op_type == "output_arg", "[TensorFlow Frontend] Internal error: get_input_node is supported only for output_arg."); - parse_producer_name(m_producer_name, producer_name, producer_output_port_index, {}); -} - -void DecoderArgDef::get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const { - FRONT_END_GENERAL_CHECK(m_op_type == "output_arg", - "[TensorFlow Frontend] Internal error: get_input_node is supported only for output_arg."); - parse_producer_name(m_producer_name, producer_name, producer_output_port_index, op_type_by_name); + parse_producer_name(m_producer_name, producer_name, producer_output_port_name, producer_output_port_index); } ov::Any DecoderArgDef::get_attribute(const std::string& name) const { diff --git a/src/frontends/tensorflow/src/decoder_argdef.hpp b/src/frontends/tensorflow/src/decoder_argdef.hpp index 5b01025ee4df7a..dfee9b21e1481c 100644 --- a/src/frontends/tensorflow/src/decoder_argdef.hpp +++ b/src/frontends/tensorflow/src/decoder_argdef.hpp @@ -10,6 +10,8 @@ #include "openvino/frontend/tensorflow/decoder.hpp" namespace tensorflow { +class GraphDef; +class FunctionDef; class OpDef_ArgDef; } // namespace tensorflow @@ -19,14 +21,23 @@ namespace tensorflow { class DecoderArgDef : public ov::frontend::tensorflow::DecoderBase { public: - explicit DecoderArgDef(const ::tensorflow::OpDef_ArgDef* arg_def, const std::string& op_type) + explicit DecoderArgDef(const ::tensorflow::OpDef_ArgDef* arg_def, + const std::shared_ptr<::tensorflow::GraphDef>& graph_def, + const std::shared_ptr<::tensorflow::FunctionDef>& func_def, + const std::string& op_type) : m_arg_def(arg_def), + m_graph_def(graph_def), + m_func_def(func_def), m_op_type(op_type) {} explicit DecoderArgDef(const ::tensorflow::OpDef_ArgDef* arg_def, + const std::shared_ptr<::tensorflow::GraphDef>& graph_def, + const std::shared_ptr<::tensorflow::FunctionDef>& func_def, const std::string& op_type, const std::string& producer_name) : m_arg_def(arg_def), + m_graph_def(graph_def), + m_func_def(func_def), m_op_type(op_type), m_producer_name(producer_name) {} @@ -36,19 +47,21 @@ class DecoderArgDef : public ov::frontend::tensorflow::DecoderBase { void get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const override; - void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const override; - const std::string& get_op_type() const override; const std::string& get_op_name() const override; private: const ::tensorflow::OpDef_ArgDef* m_arg_def; + // For existence of OpDef_ArgDef object corresponding to the main graph node, + // GraphDef object must live in the memory + const std::shared_ptr<::tensorflow::GraphDef> m_graph_def; + // For existence of OpDef_ArgDef object corresponding to the body graph node, + // both GraphDef and FunctionDef objects must be alive in the memory + const std::shared_ptr<::tensorflow::FunctionDef> m_func_def; const std::string m_op_type; const std::string m_producer_name; }; diff --git a/src/frontends/tensorflow/src/decoder_proto.cpp b/src/frontends/tensorflow/src/decoder_proto.cpp index 98413e884cd82c..19e758e5c53f73 100644 --- a/src/frontends/tensorflow/src/decoder_proto.cpp +++ b/src/frontends/tensorflow/src/decoder_proto.cpp @@ -129,7 +129,12 @@ ov::Any DecoderProto::get_attribute(const std::string& name) const { } case ::tensorflow::AttrValue::ValueCase::kType: { - return get_ov_type(attrs[0].type()); + auto atype = attrs[0].type(); + if (atype != ::tensorflow::DT_STRING) { + return get_ov_type(attrs[0].type()); + } else { + return ov::Any("DT_STRING"); + } } case ::tensorflow::AttrValue::ValueCase::kList: { @@ -168,7 +173,11 @@ ov::Any DecoderProto::get_attribute(const std::string& name) const { if (list.type_size()) { std::vector res; for (int idx = 0; idx < list.type_size(); ++idx) { - res.emplace_back(get_ov_type(list.type(idx))); + if (list.type(idx) != ::tensorflow::DataType::DT_STRING) { + res.emplace_back(get_ov_type(list.type(idx))); + } else { + res.emplace_back(ov::element::undefined); + } } return res; } @@ -194,9 +203,22 @@ ov::Any DecoderProto::get_attribute(const std::string& name) const { FRONT_END_GENERAL_CHECK(pshape.is_static(), "Dynamic shapes are not supported for Tensor attribute."); const auto& tf_type = tensor_proto.dtype(); auto ov_type = get_ov_type(tf_type); - FRONT_END_GENERAL_CHECK( - ov_type.is_static(), - "Encountered unknown element type " + DataType_Name(tf_type) + " on an empty tensor_proto"); + if (tf_type != ::tensorflow::DataType::DT_STRING) { + FRONT_END_GENERAL_CHECK( + ov_type.is_static(), + "Encountered unknown element type " + DataType_Name(tf_type) + " on an empty tensor_proto"); + } else { + ov_type = ov::element::u64; + pshape.resize(0); + pshape.push_back(tensor_proto.string_val_size()); + } + if (tf_type == ::tensorflow::DataType::DT_STRING) { + auto data = std::vector(); + for (auto& item : tensor_proto.string_val()) { + data.push_back(item); + } + return data; + } ov::Tensor res(ov_type, pshape.get_shape()); auto tensor_content = tensor_proto.tensor_content(); if (!tensor_content.empty() && tensor_proto.has_tensor_shape()) { @@ -286,27 +308,8 @@ size_t DecoderProto::get_input_size() const { void parse_producer_name(const std::string& producer_port_name, std::string& producer_name, - size_t& producer_output_port_index, - const DecoderBase::OpTypeByName& op_type_by_name) { - using OutputPortIdxMax = std::unordered_map; - // create a table of operation type and its output ports - // for which we specify output port indices manually - // it is mainly affects multiple output operations - // extract this information from tensorflow/core/ops/*.cc files - const OutputPortIdxMax output_port_idx_map = { - {"TopK:indices", 1}, - {"TopKV2:indices", 1}, - {"CTCGreedyDecoder:decoded_values", 1}, - {"CTCGreedyDecoder:decoded_shape", 2}, - {"CTCGreedyDecoder:log_probability", 3}, - {"CTCGreedyDecoder:log_probability", 3}, - {"FusedBatchNorm:batch_mean", 1}, - {"FusedBatchNorm:batch_variance", 2}, - {"FusedBatchNormV2:batch_mean", 1}, - {"FusedBatchNormV2:batch_variance", 2}, - {"FusedBatchNormV3:batch_mean", 1}, - {"FusedBatchNormV3:batch_variance", 2}, - }; + std::string& producer_output_port_name, + size_t& producer_output_port_index) { // Body graph nodes may have two colons `:` input names, for example, // `TopKV2Name:indices:0` means that producer operation name is `TopKV2Name` // the middle name is output port name of the producer `indices` that means @@ -323,11 +326,7 @@ void parse_producer_name(const std::string& producer_port_name, "Port id is not specified or not a number. Value: ", port_id); producer_output_port_index = std::stoi(port_id); - auto producer_op_type = - (op_type_by_name.count(producer_name) > 0) ? op_type_by_name.at(producer_name) : "Unknown"; - auto producer_key = producer_op_type + ":" + port_name; - producer_output_port_index = output_port_idx_map.count(producer_key) > 0 ? output_port_idx_map.at(producer_key) - : producer_output_port_index; + producer_output_port_name = port_name; return; } else if (first_colon != std::string::npos) { // just one colon case @@ -345,17 +344,10 @@ void parse_producer_name(const std::string& producer_port_name, void DecoderProto::get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const { const std::string producer_port_name = m_node_def->input(static_cast(input_port_idx)); - parse_producer_name(producer_port_name, producer_name, producer_output_port_index, {}); -} - -void DecoderProto::get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const { - const std::string producer_port_name = m_node_def->input(static_cast(input_port_idx)); - parse_producer_name(producer_port_name, producer_name, producer_output_port_index, op_type_by_name); + parse_producer_name(producer_port_name, producer_name, producer_output_port_name, producer_output_port_index); } const std::string& DecoderProto::get_op_type() const { diff --git a/src/frontends/tensorflow/src/decoder_proto.hpp b/src/frontends/tensorflow/src/decoder_proto.hpp index db1f113882451d..6d1bdbd11eebd6 100644 --- a/src/frontends/tensorflow/src/decoder_proto.hpp +++ b/src/frontends/tensorflow/src/decoder_proto.hpp @@ -12,6 +12,8 @@ #include "types.pb.h" namespace tensorflow { +class GraphDef; +class FunctionDef; class NodeDef; class AttrValue; } // namespace tensorflow @@ -24,12 +26,23 @@ ov::element::Type get_ov_type(const ::tensorflow::DataType& type); void parse_producer_name(const std::string& producer_port_name, std::string& producer_name, - size_t& producer_output_port_index, - const DecoderBase::OpTypeByName& op_type_by_name); + std::string& producer_output_port_name, + size_t& producer_output_port_index); class DecoderProto : public ov::frontend::tensorflow::DecoderBase { public: - explicit DecoderProto(const ::tensorflow::NodeDef* node_def) : m_node_def(node_def) {} + explicit DecoderProto(const ::tensorflow::NodeDef* node_def, + const std::shared_ptr<::tensorflow::GraphDef>& graph_def) + : m_node_def(node_def), + m_graph_def(graph_def), + m_func_def(nullptr) {} + + explicit DecoderProto(const ::tensorflow::NodeDef* node_def, + const std::shared_ptr<::tensorflow::GraphDef>& graph_def, + const std::shared_ptr<::tensorflow::FunctionDef>& func_def) + : m_node_def(node_def), + m_graph_def(graph_def), + m_func_def(func_def) {} ov::Any get_attribute(const std::string& name) const override; @@ -37,13 +50,9 @@ class DecoderProto : public ov::frontend::tensorflow::DecoderBase { void get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const override; - void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const override; - const std::string& get_op_type() const override; const std::string& get_op_name() const override; @@ -51,6 +60,12 @@ class DecoderProto : public ov::frontend::tensorflow::DecoderBase { private: std::vector<::tensorflow::AttrValue> decode_attribute_helper(const std::string& name) const; const ::tensorflow::NodeDef* m_node_def; + // For existence of NodeDef object corresponding to the main graph node, + // GraphDef object must live in the memory + const std::shared_ptr<::tensorflow::GraphDef> m_graph_def; + // For existence of NodeDef object corresponding to the body graph node, + // both GraphDef and FunctionDef objects must be alive in the memory + const std::shared_ptr<::tensorflow::FunctionDef> m_func_def; }; } // namespace tensorflow } // namespace frontend diff --git a/src/frontends/tensorflow/src/frontend.cpp b/src/frontends/tensorflow/src/frontend.cpp index 4893ea0a9236c8..6a6d9cd85c007d 100644 --- a/src/frontends/tensorflow/src/frontend.cpp +++ b/src/frontends/tensorflow/src/frontend.cpp @@ -5,10 +5,13 @@ #include "openvino/frontend/tensorflow/frontend.hpp" #include "graph_iterator_proto.hpp" +#include "graph_iterator_proto_txt.hpp" +#include "graph_iterator_saved_model.hpp" #include "helper_transforms/block_lstm_replacer.hpp" #include "helper_transforms/const_to_result_remover.hpp" #include "helper_transforms/embedding_segments_feature_fusing.hpp" #include "helper_transforms/gru_block_cell_replacer.hpp" +#include "helper_transforms/saved_model_unused_remover.hpp" #include "input_model.hpp" #include "op_table.hpp" #include "openvino/frontend/tensorflow/extension/conversion.hpp" @@ -17,7 +20,6 @@ #include "openvino/pass/manager.hpp" #include "openvino/util/common_util.hpp" #include "openvino/util/log.hpp" -#include "pass/transpose_sinking.hpp" #include "so_extension.hpp" #include "tf_framework_node.hpp" #include "transformations/common_optimizations/reverse_shape_and_type_infer.hpp" @@ -29,24 +31,32 @@ using namespace ov; using namespace ov::frontend::tensorflow; namespace { -std::vector get_unconverted_types_from_model(const std::shared_ptr& model) { - std::vector unconverted_ops_types; + +void get_unsupported_operations_and_failures(const std::shared_ptr& model, + std::vector& unsupported_operations, + std::unordered_map& failures) { for (const auto& node : model->get_ordered_ops()) { if (const auto& fw_node = ov::as_type_ptr(node)) { auto op_type = fw_node->get_decoder()->get_op_type(); - unconverted_ops_types.push_back(op_type); + auto fw_node_attrs = fw_node->get_attrs(); + if (fw_node_attrs.find(FrameworkNode::failed_conversion_key) != fw_node_attrs.end() && + failures.count(op_type) == 0) { + // save only the first encountered failure that is more improtant for developer + // that means the translator is found but the conversion is failed + failures[op_type] = fw_node_attrs.at(FrameworkNode::failed_conversion_key); + } else if (std::find(unsupported_operations.begin(), unsupported_operations.end(), op_type) == + unsupported_operations.end()) { + // found new unsupported operation + unsupported_operations.push_back(op_type); + } } if (const auto& fw_node = ov::as_type_ptr(node)) { int subgraphs_size = static_cast(fw_node->get_internal_subgraphs_size()); for (int i = 0; i < subgraphs_size; ++i) { - auto internal_types = get_unconverted_types_from_model(fw_node->get_function(i)); - unconverted_ops_types.insert(unconverted_ops_types.begin(), - internal_types.begin(), - internal_types.end()); + get_unsupported_operations_and_failures(fw_node->get_function(i), unsupported_operations, failures); } } } - return unconverted_ops_types; } void translate_framework_node(const std::shared_ptr& node, @@ -66,7 +76,7 @@ void translate_framework_node(const std::shared_ptr& node, auto old_output = old_outputs.begin(); for (; new_output != new_node_outputs.end() && old_output != old_outputs.end(); ++old_output, ++new_output) { - old_output->replace(*new_output); + old_output->replace(new_output->port); } } } // namespace @@ -75,8 +85,10 @@ FrontEnd::FrontEnd() : m_op_translators(tensorflow::op::get_supported_ops()) {} /// \brief Check if FrontEndTensorflow can recognize model from given parts bool FrontEnd::supported_impl(const std::vector& variants) const { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; // TODO: Support other TensorFlow formats: SavedModel, .meta, checkpoint, pbtxt - if (variants.size() != 1) + if (variants.size() != 1 + extra_variants_num) return false; if (variants[0].is()) { @@ -86,6 +98,11 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { // for automatic deduction of the frontend to convert the model // we have more strict rule that is to have `.pb` extension in the path return true; + } else if (GraphIteratorSavedModel::is_supported(model_path)) { + return true; + } else if (GraphIteratorProtoTxt::is_supported(model_path)) { + // handle text protobuf format + return true; } } #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) @@ -97,6 +114,11 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { // for automatic deduction of the frontend to convert the model // we have more strict rule that is to have `.pb` extension in the path return true; + } else if (GraphIteratorSavedModel::is_supported(model_path)) { + return true; + } else if (GraphIteratorProtoTxt::is_supported(model_path)) { + // handle text protobuf format + return true; } } #endif @@ -108,8 +130,11 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { } ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { - // TODO: Support other TensorFlow formats: SavedModel, .meta, checkpoint, pbtxt - FRONT_END_GENERAL_CHECK(variants.size() == 1, + // TODO: Support other TensorFlow formats: SavedModel, .meta, checkpoint + + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + FRONT_END_GENERAL_CHECK(variants.size() == 1 + extra_variants_num, "[TensorFlow Frontend] Internal error or inconsistent input model: the frontend supports " "only frozen binary protobuf format."); @@ -118,6 +143,21 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va if (GraphIteratorProto::is_supported(model_path)) { // handle binary protobuf format return std::make_shared(std::make_shared(model_path), m_telemetry); + } else if (GraphIteratorSavedModel::is_supported(model_path)) { + std::shared_ptr graph_iterator; + if (variants.size() > 1 && variants[1].is()) { + graph_iterator = std::make_shared(model_path, variants[1].as()); + } else { + graph_iterator = std::make_shared(model_path, std::string("serve")); + } + return std::make_shared(graph_iterator, + m_telemetry, + graph_iterator->get_variables_index(), + graph_iterator->get_saved_model_input_names(), + graph_iterator->get_saved_model_output_names()); + } else if (GraphIteratorProtoTxt::is_supported(model_path)) { + // handle text protobuf format + return std::make_shared(std::make_shared(model_path), m_telemetry); } } #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) @@ -126,6 +166,23 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va if (GraphIteratorProto::is_supported(model_path)) { // handle binary protobuf format with a path in Unicode return std::make_shared(std::make_shared(model_path), m_telemetry); + } else if (GraphIteratorSavedModel::is_supported(model_path)) { + std::shared_ptr graph_iterator; + if (variants.size() > 1 && variants[1].is()) { + graph_iterator = std::make_shared( + model_path, + ov::util::wstring_to_string(variants[1].as())); + } else { + graph_iterator = std::make_shared(model_path, std::string("serve")); + } + return std::make_shared(graph_iterator, + m_telemetry, + graph_iterator->get_variables_index(), + graph_iterator->get_saved_model_input_names(), + graph_iterator->get_saved_model_output_names()); + } else if (GraphIteratorProtoTxt::is_supported(model_path)) { + // handle text protobuf format with a path in Unicode + return std::make_shared(std::make_shared(model_path), m_telemetry); } } #endif @@ -145,15 +202,34 @@ ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& va std::shared_ptr FrontEnd::convert(const ov::frontend::InputModel::Ptr& model) const { auto f = convert_partially(model); - auto unsupported_operations = get_unconverted_types_from_model(f); + std::unordered_map failures; + std::vector unsupported_operations; + get_unsupported_operations_and_failures(f, unsupported_operations, failures); + + std::stringstream exception_message; + for (const auto& failure : failures) { + if (m_telemetry) { + // TODO: 105173 support anonymization of exception message in order to send to telemetry + } + exception_message << "[TensorFlow Frontend] Internal error: conversion is failed for " + failure.first + + " operation with a message:\n" + failure.second + "\n"; + } + if (m_telemetry) { for (const auto& unsupported_operation : unsupported_operations) { m_telemetry->send_event("error_cause", "tf_" + unsupported_operation); } } - FRONT_END_OP_CONVERSION_CHECK( - unsupported_operations.size() == 0, - "[TensorFlow Frontend] Internal error: No translator found for " + unsupported_operations[0] + " node."); + // TODO 107500: report the full list of unsupported operations + // also, communicate with MO for the fallback to the legacy FE + // via OpConversionFailure exception that will store all failures and unsupported_operations + if (unsupported_operations.size() > 0) { + exception_message << "[TensorFlow Frontend] Internal error: No translator found for " + + unsupported_operations[0] + " node."; + } + + bool is_conversion_successful = ((unsupported_operations.size() == 0) && (failures.size() == 0)); + FRONT_END_OP_CONVERSION_CHECK(is_conversion_successful, exception_message.str()); return f; } @@ -232,6 +308,7 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { // run transformations to convert sub-graphs with intermediate (or FrameworkNode) operations // into sub-graphs with only OpenVINO operations ov::pass::Manager manager; + manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); @@ -239,16 +316,18 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { manager.run_passes(model); } - // TODO: TSGeneral can fail on models with Framework nodes (not converted to OV opset) - auto unsupported_ops = get_unconverted_types_from_model(model); - if (unsupported_ops.size() > 0) { + // TODO 107554: TSGeneral can fail on models with Framework nodes (not converted to OV opset) + std::unordered_map failures; + std::vector unsupported_operations; + get_unsupported_operations_and_failures(model, unsupported_operations, failures); + if (unsupported_operations.size() > 0 || failures.size() > 0) { return; } { // perform transpose sinking and reverse infer if the model contains only OpenVINO operations ov::pass::Manager manager; - manager.register_pass(); + manager.register_pass(); manager.register_pass(); manager.run_passes(model); } @@ -264,13 +343,22 @@ void FrontEnd::add_extension(const std::shared_ptr& extension) { m_extensions.push_back(so_ext); } else if (auto common_conv_ext = std::dynamic_pointer_cast(extension)) { m_conversion_extensions.push_back(common_conv_ext); - m_op_translators[common_conv_ext->get_op_type()] = [=](const NodeContext& context) { - return common_conv_ext->get_converter()(context); - }; - } else if (const auto& tensorflow_conv_ext = std::dynamic_pointer_cast(extension)) { + if (common_conv_ext->get_converter()) { + m_op_translators[common_conv_ext->get_op_type()] = + ov::frontend::tensorflow::CreatorFunctionIndexed([=](const tensorflow::NodeContext& context) { + return common_conv_ext->get_converter()(context); + }); + } else if (common_conv_ext->get_converter_named_and_indexed()) { + m_op_translators[common_conv_ext->get_op_type()] = + ov::frontend::tensorflow::CreatorFunctionNamedAndIndexed([=](const tensorflow::NodeContext& context) { + return common_conv_ext->get_converter_named_and_indexed()(context); + }); + } + // Ignore other types of extensions in particular CreatorFunctionNamed which cannot be used with tensorflow + // frontend + } else if (const auto& tensorflow_conv_ext = + std::dynamic_pointer_cast(extension)) { m_conversion_extensions.push_back(tensorflow_conv_ext); - m_op_translators[tensorflow_conv_ext->get_op_type()] = [=](const NodeContext& context) { - return tensorflow_conv_ext->get_converter()(context); - }; + m_op_translators[tensorflow_conv_ext->get_op_type()] = tensorflow_conv_ext->get_converter(); } } diff --git a/src/frontends/tensorflow/src/graph_iterator_proto.hpp b/src/frontends/tensorflow/src/graph_iterator_proto.hpp index 1fa836e3b036e1..a69d3b05ffaa7e 100644 --- a/src/frontends/tensorflow/src/graph_iterator_proto.hpp +++ b/src/frontends/tensorflow/src/graph_iterator_proto.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include #include "decoder_argdef.hpp" #include "decoder_proto.hpp" @@ -18,6 +19,7 @@ namespace frontend { namespace tensorflow { class GraphIteratorProto : public GraphIterator { +protected: std::shared_ptr<::tensorflow::GraphDef> m_graph_def; std::shared_ptr<::tensorflow::FunctionDef> m_func_def; @@ -27,6 +29,29 @@ class GraphIteratorProto : public GraphIterator { std::vector m_input_names; std::vector m_output_names; + GraphIteratorProto() + : m_graph_def(std::make_shared<::tensorflow::GraphDef>()), + m_func_def(nullptr), + m_library_map() {} + + void initialize_decoders_and_library() { + FRONT_END_GENERAL_CHECK(m_graph_def, "GraphDef is not initialized."); + + auto nodes_size = m_graph_def->node_size(); + m_decoders.resize(static_cast(nodes_size)); + for (int node_ind = 0; node_ind < nodes_size; ++node_ind) { + m_decoders[node_ind] = std::make_shared(&m_graph_def->node(node_ind), m_graph_def); + } + + // initialize a library map + auto num_funcs = m_graph_def->library().function_size(); + for (int func_ind = 0; func_ind < num_funcs; ++func_ind) { + auto func = m_graph_def->library().function(func_ind); + auto func_name = func.signature().name(); + m_library_map.insert(std::pair(func_name, func_ind)); + } + } + public: GraphIteratorProto(const std::shared_ptr<::tensorflow::GraphDef>& graph_def, const std::shared_ptr<::tensorflow::FunctionDef>& func_def, @@ -45,12 +70,13 @@ class GraphIteratorProto : public GraphIterator { for (int input_ind = 0; input_ind < input_size; ++input_ind) { auto input_arg = &m_func_def->signature().input_arg(input_ind); m_input_names.push_back(input_arg->name()); - m_decoders.push_back(std::make_shared(input_arg, "input_arg")); + m_decoders.push_back(std::make_shared(input_arg, m_graph_def, m_func_def, "input_arg")); } // fill all node defs from library functions for (int node_ind = 0; node_ind < nodes_size; ++node_ind) { - m_decoders.push_back(std::make_shared(&(m_func_def->node_def(node_ind)))); + m_decoders.push_back( + std::make_shared(&(m_func_def->node_def(node_ind)), m_graph_def, m_func_def)); } // fill all outputs from library functions @@ -60,7 +86,8 @@ class GraphIteratorProto : public GraphIterator { auto output_arg = &m_func_def->signature().output_arg(output_ind); m_output_names.push_back(output_arg->name()); auto producer_name = ret_map.at(output_arg->name()); - m_decoders.push_back(std::make_shared(output_arg, "output_arg", producer_name)); + m_decoders.push_back( + std::make_shared(output_arg, m_graph_def, m_func_def, "output_arg", producer_name)); } } @@ -73,19 +100,7 @@ class GraphIteratorProto : public GraphIterator { FRONT_END_GENERAL_CHECK(pb_stream && pb_stream.is_open(), "Model file does not exist"); FRONT_END_GENERAL_CHECK(m_graph_def->ParseFromIstream(&pb_stream), "Model cannot be parsed"); - auto nodes_size = m_graph_def->node_size(); - m_decoders.resize(static_cast(nodes_size)); - for (int node_ind = 0; node_ind < nodes_size; ++node_ind) { - m_decoders[node_ind] = std::make_shared(&m_graph_def->node(node_ind)); - } - - // initialize a library map - auto num_funcs = m_graph_def->library().function_size(); - for (int func_ind = 0; func_ind < num_funcs; ++func_ind) { - auto func = m_graph_def->library().function(func_ind); - auto func_name = func.signature().name(); - m_library_map.insert(std::pair(func_name, func_ind)); - } + initialize_decoders_and_library(); } /// \brief Check if the input file is supported @@ -148,6 +163,7 @@ class GraphIteratorProto : public GraphIterator { return m_output_names; } }; + } // namespace tensorflow } // namespace frontend } // namespace ov diff --git a/src/frontends/tensorflow/src/graph_iterator_proto_txt.hpp b/src/frontends/tensorflow/src/graph_iterator_proto_txt.hpp new file mode 100644 index 00000000000000..b193aaaef52010 --- /dev/null +++ b/src/frontends/tensorflow/src/graph_iterator_proto_txt.hpp @@ -0,0 +1,53 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/text_format.h" +#include "graph_iterator_proto.hpp" +#include "openvino/frontend/exception.hpp" + +namespace ov { +namespace frontend { +namespace tensorflow { + +class GraphIteratorProtoTxt : public GraphIteratorProto { +public: + template + GraphIteratorProtoTxt(const std::basic_string& path) : GraphIteratorProto() { + std::ifstream pbtxt_stream(path, std::ios::in); + FRONT_END_GENERAL_CHECK(pbtxt_stream && pbtxt_stream.is_open(), "Model file does not exist"); + auto input_stream = std::make_shared<::google::protobuf::io::IstreamInputStream>(&pbtxt_stream); + FRONT_END_GENERAL_CHECK(input_stream, "Model cannot be read"); + auto is_parsed = ::google::protobuf::TextFormat::Parse(input_stream.get(), m_graph_def.get()); + FRONT_END_GENERAL_CHECK( + is_parsed, + "[TensorFlow Frontend] Incorrect model or internal error: Model in text Protobuf format cannot be parsed."); + + initialize_decoders_and_library(); + } + + /// \brief Check if the input file is supported + template + static bool is_supported(const std::basic_string& path) { + std::ifstream pbtxt_stream(path, std::ios::in); + bool model_exists = (pbtxt_stream && pbtxt_stream.is_open()); + if (!model_exists) { + return false; + } + auto input_stream = std::make_shared<::google::protobuf::io::IstreamInputStream>(&pbtxt_stream); + if (!input_stream) { + return false; + } + auto graph_def = std::make_shared<::tensorflow::GraphDef>(); + auto is_parsed = ::google::protobuf::TextFormat::Parse(input_stream.get(), graph_def.get()); + return is_parsed; + } +}; +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp b/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp new file mode 100644 index 00000000000000..160912657c7a1a --- /dev/null +++ b/src/frontends/tensorflow/src/graph_iterator_saved_model.hpp @@ -0,0 +1,291 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "graph_iterator_proto.hpp" +#include "openvino/util/file_util.hpp" +#include "saved_model.pb.h" + +namespace ov { +namespace frontend { +namespace tensorflow { + +struct VIBlock; + +template +std::basic_string get_saved_model_name() {} +template +std::basic_string get_variables_index_name() {} + +template <> +std::basic_string get_saved_model_name(); +template <> +std::basic_string get_variables_index_name(); + +#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) +template <> +std::basic_string get_saved_model_name(); +template <> +std::basic_string get_variables_index_name(); +#endif + +// Stores information about variables index +class SavedModelVariablesIndex { + // Contains maximum amount of shards, used for creating corrext extension + int32_t m_total_shards; + // Contains BundleEntryProto variables list, readed from .index file + std::map> m_variables_index; + // List of opened data files for using with BundleEntryProto + std::map> m_data_files; + // List of mapped variables which could be read using TrackableObjectGraph + std::map m_variables_map; + +public: + /// \brief Reads variables from opened variable index file. Can cause an asserts in case of issues. + /// \param vi_stream Opened stream file, file pointer doesn't matter, it will be rewind internally. + /// \param path A path to file with variables data + /// \returns Returns true in case of everything loads successfully, false otherwise + bool read_variables(std::ifstream& vi_stream, const std::string& path); +#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) + /// \brief Reads variables from opened variable index file. Can cause an asserts in case of issues. + /// \param vi_stream Opened stream file, file pointer doesn't matter, it will be rewind internally. + /// \param path A path to file with variables data + /// \returns Returns true in case of everything loads successfully, false otherwise + bool read_variables(std::ifstream& vi_stream, const std::wstring& path); +#endif + + /// \brief Returns data and size of data of stored variable + /// \param name Name of variable + /// \param data Pointer on a pointer where data pointer will be returned + /// \param size Pointer on a variable which will stores data size + /// \returns Returns true in case variable was found, false otherwise (data and size will be untouched) + bool get_variable(const std::string& name, const char** data, size_t* size) const { + auto varItem = m_variables_index.find(name); + if (varItem == m_variables_index.end()) { + return false; + } + if (data != nullptr) { + *data = varItem->second.data(); + } + if (size != nullptr) { + *size = varItem->second.size(); + } + return true; + } + + /// \brief Returns data and size of data of mapped variable from trackable object graph to variables index + /// \param name Name of a mapping variable + /// \param data Pointer on a pointer where data pointer will be returned + /// \param size Pointer on a variable which will stores data size + /// \returns Returns true in case variable was found, false otherwise (data and size will be untouched) + bool get_mapped_variable(const std::string& name, const char** data, size_t* size) const { + auto mapItem = m_variables_map.find(name); + if (mapItem == m_variables_map.end()) { + return false; + } + return get_variable(mapItem->second, data, size); + } + + /// \brief Checks if variable has a mapped pair + /// \param name Name of variable for checking existance + /// \returns True in case variable has mapped value and false otherwise + bool has_mapped_variable(const std::string& name) const { + auto mapItem = m_variables_map.find(name); + return mapItem != m_variables_map.end(); + } + + /// \brief Returns shared pointer to a requested shard_id, or nullptr in case of shard_id isn't found + /// \param shard_id Requested shard_id + /// \returns Valid shared_ptr with ifstream or with nullptr if shard isn't found + std::shared_ptr get_data_file(const int32_t shard_id) const { + auto result = m_data_files.find(shard_id); + return result != m_data_files.end() ? result->second : nullptr; + } + + /// \brief Adds variable mapping to the variables map + /// \param var_name Variable full name (from .index file) + /// \param map_name Mapped name + /// \param rewrite Rewrite mapped value in case it exists + /// \returns True if map updated. False if nothing changed (if variable exists and rewrite is false). + bool map_variable(const std::string& var_name, const std::string& map_name, bool rewrite = false) { + if (m_variables_map.find(var_name) != m_variables_map.end() && rewrite == false) { + return false; + } + + m_variables_map[var_name] = map_name; + return true; + } + +private: + /// \brief Reads block structure of .index file + /// \param[in,out] fs Filestream of .index file, position in file will be updated + /// \param[in] index Variables index block which stores information about block + /// \param[out] data Block data will be readed + /// \param[out] offset Offset of block start + /// \param[out] offset_end Offset of block end + void read_variables_index_block(std::ifstream& fs, + const VIBlock& index, + std::vector& data, + uint32_t& offset, + uint32_t& offset_end); + /// \brief Reads key=value pair from provided pointer + /// \param[in,out] ptr Actual pointer, will be moved to the end of readed pair (to read next) + /// \param[in] ptr_end End of memory which shouldn't be passed in case of broken structure + /// \param[out] key Key name + /// \param[out] value Stored value for key (isn't a pure string, data block) + /// \param[out] val_lenght Length of readed value + void read_variables_index_pair(char*& ptr, + const char* ptr_end, + std::string& key, + char*& value, + uint32_t& val_length); + /// \brief Reads .index file and stores key=value map in provided varIndex + /// \param[in,out] fs Filestream should be parsed. Position in file will be updated + /// \param[out] varIndex Variables indx (key=value) from given filestream + void read_variables_index(std::ifstream& fs, std::map>& varIndex); + /// \brief Reads bundle header if it is available. Checks version and saves info about amount of shards + void read_bundle_header(); + /// \brief Reads key=value map from storef _CHECKPOINTABLE_OBJECT_GRAPH variable + void read_checkpointable_object_graph(); +}; + +// Loads graph from Tensorflow Saved Model file (saved_model.pb) +class GraphIteratorSavedModel : public GraphIteratorProto { + std::shared_ptr<::tensorflow::SavedModel> m_saved_model; + std::shared_ptr m_variables_index; + std::shared_ptr> m_inputs_map; + std::shared_ptr> m_outputs_map; + +public: + template + GraphIteratorSavedModel(const std::basic_string& path, const std::string& tags) + : m_saved_model(std::make_shared<::tensorflow::SavedModel>()) { + this->read_saved_model(path, tags); + } + + static bool is_supported(const std::string& path); +#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) + static bool is_supported(const std::wstring& path); +#endif + + std::shared_ptr get_variables_index() { + return m_variables_index; + } + + std::shared_ptr> get_saved_model_input_names() const { + return m_inputs_map; + } + + std::shared_ptr> get_saved_model_output_names() const { + return m_outputs_map; + } + +private: + bool is_valid_signature(const ::tensorflow::SignatureDef& signature) const; + + template + bool read_saved_model(const std::basic_string& path, const std::string& tags) { + std::ifstream sm_stream{path + get_saved_model_name(), std::ifstream::in | std::ifstream::binary}; + FRONT_END_GENERAL_CHECK(sm_stream && sm_stream.is_open(), "Model file does not exist"); + + std::basic_string varIndexPath = path + get_variables_index_name(); + if (ov::util::file_exists(varIndexPath)) { + m_variables_index = std::make_shared(); + std::ifstream vi_stream{varIndexPath, std::ifstream::in | std::ifstream::binary}; + FRONT_END_GENERAL_CHECK(vi_stream && vi_stream.is_open(), + "Saved Model's variable index file does not exist"); + FRONT_END_GENERAL_CHECK(m_variables_index->read_variables(vi_stream, path), + "Saved Model's variable index file cannot be parsed"); + } + + bool res = m_saved_model->ParseFromIstream(&sm_stream); + FRONT_END_GENERAL_CHECK(res && m_saved_model->meta_graphs_size(), "Saved Model cannot be parsed"); + + for (const auto& meta_graph : m_saved_model->meta_graphs()) { + if (!meta_graph.has_graph_def()) { + continue; + } + + if (m_saved_model->meta_graphs_size() > 1) { + bool tag_found = false; + for (const auto& tag : meta_graph.meta_info_def().tags()) { + if (tags.find(tag) != std::string::npos) { + tag_found = true; + break; + } + } + if (!tag_found) { + continue; + } + } + + std::map validSignatures = {}; + for (const auto& sit : meta_graph.signature_def()) { + const std::string& key = sit.first; + const ::tensorflow::SignatureDef& val = sit.second; + if (is_valid_signature(val)) { + validSignatures[key] = &val; + } + } + + auto serving_default = validSignatures.find("serving_default"); + + if (serving_default != validSignatures.end()) { + m_inputs_map = std::make_shared>(); + m_outputs_map = std::make_shared>(); + for (const auto& input : serving_default->second->inputs()) { + (*m_inputs_map)[input.second.name()] = input.first; + } + for (const auto& output : serving_default->second->outputs()) { + (*m_outputs_map)[output.second.name()] = output.first; + } + } + + m_graph_def = std::make_shared<::tensorflow::GraphDef>(meta_graph.graph_def()); + + // Update variables map using information by resolving AssignVariableOp graph nodes + std::map var_map; + map_assignvariable(m_graph_def, var_map); + for (auto var : var_map) { + m_variables_index->map_variable(var.first, var.second); + } + + auto nodes_size = m_graph_def->node_size(); + m_decoders.resize(static_cast(nodes_size)); + for (int node_ind = 0; node_ind < nodes_size; ++node_ind) { + m_decoders[node_ind] = std::make_shared(&m_graph_def->node(node_ind), m_graph_def); + } + + // initialize a library map + auto num_funcs = m_graph_def->library().function_size(); + for (int func_ind = 0; func_ind < num_funcs; ++func_ind) { + auto func = m_graph_def->library().function(func_ind); + auto func_name = func.signature().name(); + m_library_map.insert(std::pair(func_name, func_ind)); + } + + return true; + } + + FRONT_END_GENERAL_CHECK(false, "Saved Model doesn't contain MetaGraph with requested tag"); + + return false; + } + + /// \brief Reads relationship between VarHandleOp - RestoreV2 - AssignVariableOp and + /// stores this information in a provided key=value map. Where key - name of VarHandleOp, + /// value - long variable name which is stored in RestoreV2. + /// It needs to map VarHandleOp to right place in .index file. + /// \param[in] graph_def GraphDef object for analysis + /// \param[out] variables_map Map of variables found in graph_def + void map_assignvariable(const std::shared_ptr<::tensorflow::GraphDef> graph_def, + std::map& variables_map) const; +}; // GraphIteratorSavedModel + +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow/src/input_model.cpp b/src/frontends/tensorflow/src/input_model.cpp index f6292da7dc964a..251dbc4d43beb7 100644 --- a/src/frontends/tensorflow/src/input_model.cpp +++ b/src/frontends/tensorflow/src/input_model.cpp @@ -55,7 +55,10 @@ class InputModel::InputModelTFImpl { InputModelTFImpl(const GraphIterator::Ptr& graph_iterator, const ov::frontend::InputModel& input_model); InputModelTFImpl(const GraphIterator::Ptr& graph_iterator, const ov::frontend::InputModel& input_model, - const std::shared_ptr& telemetry); + const std::shared_ptr& telemetry, + const std::shared_ptr& variables_index, + const std::shared_ptr> saved_model_input_names, + const std::shared_ptr> saved_model_output_names); std::vector get_inputs() const; std::vector get_outputs() const; ov::frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensorName) const; @@ -79,6 +82,9 @@ class InputModel::InputModelTFImpl { std::shared_ptr get_body_input_model(const std::string& body_model_name) const; std::vector get_input_names() const; std::vector get_output_names() const; + std::shared_ptr get_variables_index() const; + std::shared_ptr> get_saved_model_input_names() const; + std::shared_ptr> get_saved_model_output_names() const; private: void load_places(); @@ -99,6 +105,10 @@ class InputModel::InputModelTFImpl { std::shared_ptr m_telemetry; + std::shared_ptr m_variables_index; + std::shared_ptr> m_saved_model_input_names; + std::shared_ptr> m_saved_model_output_names; + // shows if some nodes might be deleted from graph bool m_graph_changed = false; }; @@ -152,10 +162,10 @@ void InputModel::InputModelTFImpl::load_places() { } auto dtype_any = node_decoder->get_attribute("dtype"); auto placeholder_name = node_decoder->get_op_name(); - FRONT_END_GENERAL_CHECK( - dtype_any.is(), - "Incorrect input model: Placeholder node " + placeholder_name + " has unspecified type."); - auto type = dtype_any.as(); + ov::element::Type type = ov::element::dynamic; + if (dtype_any.is()) { + type = dtype_any.as(); + } std::vector names = {op_name}; auto tensor_place = std::make_shared(m_input_model, pshape, type, names); m_tensor_places[op_name] = tensor_place; @@ -166,9 +176,13 @@ void InputModel::InputModelTFImpl::load_places() { } for (size_t input_port_idx = 0; input_port_idx < node_decoder->get_input_size(); ++input_port_idx) { std::string producer_op_name; + std::string producer_output_port_name; size_t producer_output_port_idx; try { - node_decoder->get_input_node(input_port_idx, producer_op_name, producer_output_port_idx); + node_decoder->get_input_node(input_port_idx, + producer_op_name, + producer_output_port_name, + producer_output_port_idx); op_names_with_consumers.insert(producer_op_name); } catch (const std::exception&) { FRONT_END_THROW("[ ERROR ] Exception happened when preparing input " + std::to_string(input_port_idx) + @@ -202,6 +216,17 @@ void InputModel::InputModelTFImpl::load_places() { m_outputs.push_back(output_place); } } +std::shared_ptr InputModel::InputModelTFImpl::get_variables_index() const { + return m_variables_index; +} + +std::shared_ptr> InputModel::InputModelTFImpl::get_saved_model_input_names() const { + return m_saved_model_input_names; +} + +std::shared_ptr> InputModel::InputModelTFImpl::get_saved_model_output_names() const { + return m_saved_model_output_names; +} std::vector> InputModel::InputModelTFImpl::get_op_places() const { return topologically_sort_op_nodes(); @@ -255,9 +280,13 @@ std::vector> InputModel::InputModelTFImpl::topologicall for (size_t input_port_idx = 0; input_port_idx < input_count; ++input_port_idx) { std::string producer_name; + std::string producer_output_port_name; size_t producer_output_port_idx; try { - current_operation_decoder->get_input_node(input_port_idx, producer_name, producer_output_port_idx); + current_operation_decoder->get_input_node(input_port_idx, + producer_name, + producer_output_port_name, + producer_output_port_idx); } catch (const std::exception&) { FRONT_END_THROW("[ ERROR ] Exception happened when preparing input " + std::to_string(input_port_idx) + " for op '" + @@ -337,12 +366,19 @@ std::shared_ptr InputModel::InputModelTFImpl::get_body_input_model( return std::make_shared(body_graph_iterator, m_telemetry); } -InputModel::InputModelTFImpl::InputModelTFImpl(const GraphIterator::Ptr& graph_iterator, - const ov::frontend::InputModel& input_model, - const std::shared_ptr& telemetry) +InputModel::InputModelTFImpl::InputModelTFImpl( + const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + const std::shared_ptr& telemetry, + const std::shared_ptr& variables_index, + const std::shared_ptr> saved_model_input_names, + const std::shared_ptr> saved_model_output_names) : m_graph_iterator(graph_iterator), m_input_model(input_model), - m_telemetry(telemetry) { + m_telemetry(telemetry), + m_variables_index(variables_index), + m_saved_model_input_names(saved_model_input_names), + m_saved_model_output_names(saved_model_output_names) { FRONT_END_GENERAL_CHECK(m_graph_iterator, "Null pointer specified for GraphIterator"); m_input_names = graph_iterator->get_input_names(); m_output_names = graph_iterator->get_output_names(); @@ -445,8 +481,29 @@ void InputModel::InputModelTFImpl::set_tensor_value(ov::frontend::Place::Ptr pla m_tensor_values[name] = constant; } -InputModel::InputModel(const GraphIterator::Ptr& graph_iterator, const std::shared_ptr& telemetry) - : _impl{std::make_shared(graph_iterator, *this, telemetry)} {} +InputModel::InputModel(const GraphIterator::Ptr& graph_iterator, + const std::shared_ptr& telemetry, + const std::shared_ptr& variables_index, + const std::shared_ptr> saved_model_input_names, + const std::shared_ptr> saved_model_output_names) + : _impl{std::make_shared(graph_iterator, + *this, + telemetry, + variables_index, + saved_model_input_names, + saved_model_output_names)} {} + +std::shared_ptr InputModel::get_variables_index() { + return _impl->get_variables_index(); +} + +std::shared_ptr> InputModel::get_saved_model_input_names() const { + return _impl->get_saved_model_input_names(); +} + +std::shared_ptr> InputModel::get_saved_model_output_names() const { + return _impl->get_saved_model_output_names(); +} std::vector InputModel::get_input_names() const { return _impl->get_input_names(); diff --git a/src/frontends/tensorflow/src/input_model.hpp b/src/frontends/tensorflow/src/input_model.hpp index 9cfacafb0f0df4..ce75194467d9fa 100644 --- a/src/frontends/tensorflow/src/input_model.hpp +++ b/src/frontends/tensorflow/src/input_model.hpp @@ -16,6 +16,7 @@ namespace tensorflow { class OpPlace; class TensorPlace; +class SavedModelVariablesIndex; class InputModel : public ov::frontend::InputModel { friend class TranslateSession; @@ -31,7 +32,10 @@ class InputModel : public ov::frontend::InputModel { public: explicit InputModel(const GraphIterator::Ptr& graph_iterator, - const std::shared_ptr& telemetry = {}); + const std::shared_ptr& telemetry = {}, + const std::shared_ptr& variables_index = {}, + const std::shared_ptr> saved_model_input_names = nullptr, + const std::shared_ptr> saved_model_output_names = nullptr); std::vector get_inputs() const override; std::vector get_outputs() const override; @@ -45,6 +49,9 @@ class InputModel : public ov::frontend::InputModel { void set_element_type(const ov::frontend::Place::Ptr& place, const ov::element::Type&) override; ov::element::Type get_element_type(const ov::frontend::Place::Ptr& place) const override; void set_tensor_value(const ov::frontend::Place::Ptr& place, const void* value) override; + std::shared_ptr get_variables_index(); + std::shared_ptr> get_saved_model_input_names() const; + std::shared_ptr> get_saved_model_output_names() const; }; } // namespace tensorflow diff --git a/src/frontends/tensorflow/src/op/lookup_table_insert.cpp b/src/frontends/tensorflow/src/op/lookup_table_insert.cpp deleted file mode 100644 index 3fb679e170be38..00000000000000 --- a/src/frontends/tensorflow/src/op/lookup_table_insert.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "common_op_table.hpp" -#include "openvino/frontend/tensorflow/node_context.hpp" -#include "utils.hpp" - -namespace ov { -namespace frontend { -namespace tensorflow { -namespace op { - -OutputVector translate_lookup_table_insert_op(const ov::frontend::tensorflow::NodeContext& node) { - // auto-pruning of unsupported sub-graphs that contain - // operations working with dictionaries - default_op_checks(node, 3, {"LookupTableInsert", "LookupTableInsertV2"}); - return {}; -} - -} // namespace op -} // namespace tensorflow -} // namespace frontend -} // namespace ov diff --git a/src/frontends/tensorflow/src/op/queue_dequeue.cpp b/src/frontends/tensorflow/src/op/queue_dequeue.cpp index 16a29006604dc3..c4fb43b0cfa5f1 100644 --- a/src/frontends/tensorflow/src/op/queue_dequeue.cpp +++ b/src/frontends/tensorflow/src/op/queue_dequeue.cpp @@ -84,7 +84,9 @@ OutputVector translate_queue_dequeue_many_op(const ov::frontend::tensorflow::Nod // compute batch dimension for outputs // this is a number of batch objects emitted from QueueDequeue Dimension batch_dim = Dimension::dynamic(); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto n_const = get_constant_from_source(n)) { + OPENVINO_SUPPRESS_DEPRECATED_END auto n_value = n_const->cast_vector(); if (n_value.size() > 0 && n_value[0] > 0) { batch_dim = n_value[0]; diff --git a/src/frontends/tensorflow/src/op/var_handle.cpp b/src/frontends/tensorflow/src/op/var_handle.cpp new file mode 100644 index 00000000000000..723a9d1ac21802 --- /dev/null +++ b/src/frontends/tensorflow/src/op/var_handle.cpp @@ -0,0 +1,203 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_op_table.hpp" +#include "graph_iterator_saved_model.hpp" +#include "helper_ops/string_constant.hpp" +#include "helper_ops/unsupported_constant.hpp" +#include "input_model.hpp" +#include "openvino/opsets/opset8.hpp" +#include "tensor_bundle.pb.h" + +using namespace std; +using namespace ov::opset8; +using namespace ov; + +namespace ov { +namespace frontend { +namespace tensorflow { +namespace op { + +// Reading variable from shard file +template +static std::shared_ptr read_variable(std::shared_ptr var_index, + const ov::element::Type ov_type, + const ov::Shape shape, + const ::tensorflow::BundleEntryProto& entry, + const NodeContext& node) { + std::vector var_data; + google::protobuf::int64 size = 1; + for (uint64_t i = 0; i < shape.size(); ++i) { + size *= static_cast(shape[i]); + } + var_data.resize(size); + TENSORFLOW_OP_VALIDATION(node, + size == static_cast(entry.size() / sizeof(T)), + "[TensorFlow Frontend] Internal error: Available data size isn't equal to calculated."); + auto fs = var_index->get_data_file(entry.shard_id()); + if (!fs.get()) { + TENSORFLOW_OP_VALIDATION(node, var_index, "[TensorFlow Frontend] Internal error: Cannot get shard file."); + } + fs->seekg(entry.offset(), std::ios::beg); + fs->read(reinterpret_cast(var_data.data()), entry.size()); + return std::make_shared(ov_type, shape, var_data); +} + +OutputVector translate_varhandle_op(const NodeContext& node) { + default_op_checks(node, 0, {"VarHandleOp"}); + auto translate_session = node.get_translate_session(); + TENSORFLOW_OP_VALIDATION(node, + translate_session, + "[TensorFlow Frontend] Internal error: Translate session is nullptr."); + auto model = reinterpret_cast(translate_session->get_input_model().get()); + auto var_index = model->get_variables_index(); + auto ov_type = node.get_attribute("dtype"); + std::shared_ptr const_node; + if (ov_type == element::undefined) { + const_node = std::make_shared(); + } else { + // Getting variable description from variables index + const char* entry_data = nullptr; + size_t entry_size = 0; + auto var_name = node.get_name(); + auto shape = node.get_attribute<::ov::PartialShape>("shape").get_shape(); + bool result = var_index->get_mapped_variable(var_name, &entry_data, &entry_size); + + TENSORFLOW_OP_VALIDATION(node, result, "[TensorFlow Frontend] Internal error: Cannot find requested variable."); + + ::tensorflow::BundleEntryProto entry; + TENSORFLOW_OP_VALIDATION(node, + entry.ParseFromArray(entry_data, static_cast(entry_size)), + "[TensorFlow Frontend] Internal error: Cannot get read bundle entry."); + + switch (ov_type) { + case ov::element::u8: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::i8: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::i16: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::i32: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::i64: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::f16: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::f32: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::f64: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + case ov::element::bf16: + const_node = read_variable(var_index, ov_type, shape, entry, node); + break; + default: + FRONT_END_THROW("Encountered unknown element type " + ov_type.get_type_name()); + } + } + set_node_name(node.get_name(), const_node); + return {const_node}; +} + +OutputVector translate_varisinitialized_op(const NodeContext& node) { + auto const_node = std::make_shared(::ov::element::boolean, Shape{}, true); + set_node_name(node.get_name(), const_node); + return {const_node}; +} + +OutputVector translate_readvariable_op(const NodeContext& node) { + default_op_checks(node, 1, {"ReadVariableOp"}); + // Documentation says it should return only one tensor with dtype, but + // _output_shapes in a vector of shapes and it means it could have multiple outputs + // https://www.tensorflow.org/api_docs/python/tf/raw_ops/ReadVariableOp + auto output_shapes = node.get_attribute>("_output_shapes"); + + OutputVector outs = {}; + + for (size_t i = 0; i < output_shapes.size(); ++i) { + std::shared_ptr output_node; + if (node.get_input(0).get_partial_shape().is_static() && + output_shapes[i].get_shape() != node.get_input(0).get_shape()) { + auto reshape_shape = make_shared(ov::element::i32, output_shapes[i].get_shape()); + output_node = make_shared(node.get_input(0), reshape_shape, false); + } else { + output_node = node.get_input(0).get_node_shared_ptr(); + } + if (i == 0) { + set_out_name(node.get_name(), output_node); + set_out_name(node.get_name() + ":" + "0", output_node); + } else { + set_node_name(node.get_name() + ":" + std::to_string(i), output_node); + } + outs.push_back(output_node); + } + return outs; +} + +OutputVector translate_assignvariable_op(const NodeContext& node) { + default_op_checks(node, 2, {"AssignVariableOp"}); + auto assignvariableop_node = std::make_shared(); + set_node_name(node.get_name(), assignvariableop_node); + return {assignvariableop_node}; +} + +OutputVector translate_restorev2_op(const NodeContext& node) { + default_op_checks(node, 3, {"RestoreV2"}); + auto translate_session = node.get_translate_session(); + TENSORFLOW_OP_VALIDATION(node, + translate_session, + "[TensorFlow Frontend] Internal error: Translate session is nullptr."); + auto model = reinterpret_cast(translate_session->get_input_model().get()); + auto var_index = model->get_variables_index(); + auto tensor_names = + reinterpret_cast(node.get_input(1).get_node())->get_data().as>(); + auto tensor_types = node.get_attribute>("dtypes"); + + OutputVector outs = {}; + + for (size_t i = 0; i < tensor_names.size(); ++i) { + auto const_node = std::make_shared(); + if (i == 0) + set_node_name(node.get_name(), const_node); + else + set_node_name(node.get_name() + ":" + std::to_string(i), const_node); + outs.push_back(const_node); + } + + return outs; +} + +OutputVector translate_staticregexfullmatch_op(const NodeContext& node) { + default_op_checks(node, 1, {"StaticRegexFullMatch"}); + // auto pattern = node.get_attribute_as_any("pattern").as(); + auto const_node = std::make_shared(ov::element::boolean, ov::Shape{}, true); + set_node_name(node.get_name(), const_node); + return {const_node}; +} + +OutputVector translate_stringjoin_op(const NodeContext& node) { + default_op_checks(node, 1, {"StringJoin"}); + auto const_node = std::make_shared(); + set_node_name(node.get_name(), const_node); + return {const_node}; +} + +OutputVector translate_mergev2checkpoint_op(const NodeContext& node) { + default_op_checks(node, 1, {"MergeV2Checkpoint"}); + auto const_node = std::make_shared(); + set_node_name(node.get_name(), const_node); + return {const_node}; +} + +} // namespace op +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow/src/op_table.cpp b/src/frontends/tensorflow/src/op_table.cpp index dddab20848841f..63f05fffa2e9a2 100644 --- a/src/frontends/tensorflow/src/op_table.cpp +++ b/src/frontends/tensorflow/src/op_table.cpp @@ -6,6 +6,7 @@ #include "common_op_table.hpp" #include "openvino/opsets/opset10.hpp" +#include "openvino/opsets/opset8.hpp" #include "openvino/opsets/opset9.hpp" using namespace std; @@ -26,239 +27,255 @@ TF_OP_CONVERTER(translate_gru_block_cell_op); TF_OP_CONVERTER(translate_hash_table_op); TF_OP_CONVERTER(translate_iterator_get_next_op); TF_OP_CONVERTER(translate_iterator_op); -TF_OP_CONVERTER(translate_lookup_table_insert_op); TF_OP_CONVERTER(translate_partitioned_call_op); TF_OP_CONVERTER(translate_queue_dequeue_op); TF_OP_CONVERTER(translate_queue_dequeue_many_op); TF_OP_CONVERTER(translate_sparse_fill_empty_rows_op); TF_OP_CONVERTER(translate_sparse_reshape_op); TF_OP_CONVERTER(translate_sparse_segment_sum_op); +TF_OP_CONVERTER(translate_varisinitialized_op); +TF_OP_CONVERTER(translate_readvariable_op); +TF_OP_CONVERTER(translate_assignvariable_op); +TF_OP_CONVERTER(translate_varhandle_op); +TF_OP_CONVERTER(translate_restorev2_op); +TF_OP_CONVERTER(translate_staticregexfullmatch_op); +TF_OP_CONVERTER(translate_stringjoin_op); +TF_OP_CONVERTER(translate_mergev2checkpoint_op); TF_OP_CONVERTER(translate_while_op); const std::map get_supported_ops() { return { // note: UnaryOp translator declaration for each op must to be added in unary_op.cpp file - {"Abs", translate_unary_op}, - {"Acos", translate_unary_op}, - {"Acosh", translate_unary_op}, - {"Asin", translate_unary_op}, - {"Asinh", translate_unary_op}, - {"Atan", translate_unary_op}, - {"Atanh", translate_unary_op}, - {"Ceil", translate_unary_op}, - {"Cos", translate_unary_op}, - {"Cosh", translate_unary_op}, - {"Erf", translate_unary_op}, - {"Exp", translate_unary_op}, - {"Floor", translate_unary_op}, - {"IsFinite", translate_unary_op}, - {"IsInf", translate_unary_op}, - {"IsNan", translate_unary_op}, - {"Log", translate_unary_op}, - {"LogicalNot", translate_unary_op}, - {"Mish", translate_unary_op}, - {"Neg", translate_unary_op}, - {"Relu", translate_unary_op}, - {"Sigmoid", translate_unary_op}, - {"Sin", translate_unary_op}, - {"Sinh", translate_unary_op}, - {"Sign", translate_unary_op}, - {"Softplus", translate_unary_op}, - {"Softsign", translate_unary_op}, - {"Tan", translate_unary_op}, - {"Tanh", translate_unary_op}, - {"Swish", translate_unary_op}, + {"Abs", CreatorFunction(translate_unary_op)}, + {"Acos", CreatorFunction(translate_unary_op)}, + {"Acosh", CreatorFunction(translate_unary_op)}, + {"Asin", CreatorFunction(translate_unary_op)}, + {"Asinh", CreatorFunction(translate_unary_op)}, + {"Atan", CreatorFunction(translate_unary_op)}, + {"Atanh", CreatorFunction(translate_unary_op)}, + {"Ceil", CreatorFunction(translate_unary_op)}, + {"Cos", CreatorFunction(translate_unary_op)}, + {"Cosh", CreatorFunction(translate_unary_op)}, + {"Erf", CreatorFunction(translate_unary_op)}, + {"Exp", CreatorFunction(translate_unary_op)}, + {"Floor", CreatorFunction(translate_unary_op)}, + {"IsFinite", CreatorFunction(translate_unary_op)}, + {"IsInf", CreatorFunction(translate_unary_op)}, + {"IsNan", CreatorFunction(translate_unary_op)}, + {"Log", CreatorFunction(translate_unary_op)}, + {"LogicalNot", CreatorFunction(translate_unary_op)}, + {"Mish", CreatorFunction(translate_unary_op)}, + {"Neg", CreatorFunction(translate_unary_op)}, + {"Relu", CreatorFunction(translate_unary_op)}, + {"Sigmoid", CreatorFunction(translate_unary_op)}, + {"Sin", CreatorFunction(translate_unary_op)}, + {"Sinh", CreatorFunction(translate_unary_op)}, + {"Sign", CreatorFunction(translate_unary_op)}, + {"Softplus", CreatorFunction(translate_unary_op)}, + {"Softsign", CreatorFunction(translate_unary_op)}, + {"Tan", CreatorFunction(translate_unary_op)}, + {"Tanh", CreatorFunction(translate_unary_op)}, + {"Swish", CreatorFunction(translate_unary_op)}, // note: BinaryOp translator declaration for each op must to be added in binary_op.cpp file - {"Add", translate_binary_op}, - {"AddV2", translate_binary_op}, - {"Equal", translate_binary_op}, - {"FloorMod", translate_binary_op}, - {"Greater", translate_binary_op}, - {"GreaterEqual", translate_binary_op}, - {"Less", translate_binary_op}, - {"LessEqual", translate_binary_op}, - {"LogicalAnd", translate_binary_op}, - {"LogicalOr", translate_binary_op}, - {"LogicalXor", translate_binary_op}, - {"Maximum", translate_binary_op}, - {"Minimum", translate_binary_op}, - {"Mul", translate_binary_op}, - {"Mod", translate_binary_op}, - {"NotEqual", translate_binary_op}, - {"Pow", translate_binary_op}, - {"RealDiv", translate_binary_op}, - {"SquaredDifference", translate_binary_op}, - {"Sub", translate_binary_op}, + {"Add", CreatorFunction(translate_binary_op)}, + {"AddV2", CreatorFunction(translate_binary_op)}, + {"Equal", CreatorFunction(translate_binary_op)}, + {"FloorMod", CreatorFunction(translate_binary_op)}, + {"Greater", CreatorFunction(translate_binary_op)}, + {"GreaterEqual", CreatorFunction(translate_binary_op)}, + {"Less", CreatorFunction(translate_binary_op)}, + {"LessEqual", CreatorFunction(translate_binary_op)}, + {"LogicalAnd", CreatorFunction(translate_binary_op)}, + {"LogicalOr", CreatorFunction(translate_binary_op)}, + {"LogicalXor", CreatorFunction(translate_binary_op)}, + {"Maximum", CreatorFunction(translate_binary_op)}, + {"Minimum", CreatorFunction(translate_binary_op)}, + {"Mul", CreatorFunction(translate_binary_op)}, + {"Mod", CreatorFunction(translate_binary_op)}, + {"NotEqual", CreatorFunction(translate_binary_op)}, + {"Pow", CreatorFunction(translate_binary_op)}, + {"RealDiv", CreatorFunction(translate_binary_op)}, + {"SquaredDifference", CreatorFunction(translate_binary_op)}, + {"Sub", CreatorFunction(translate_binary_op)}, // note: ReduceOp translator declaration for each op must to be added in reduce.cpp file - {"Any", translate_direct_reduce_op}, - {"All", translate_direct_reduce_op}, - {"EuclideanNorm", translate_direct_reduce_op}, - {"Max", translate_direct_reduce_op}, - {"Mean", translate_direct_reduce_op}, - {"Min", translate_direct_reduce_op}, - {"Prod", translate_direct_reduce_op}, - {"Sum", translate_direct_reduce_op}, + {"Any", CreatorFunction(translate_direct_reduce_op)}, + {"All", CreatorFunction(translate_direct_reduce_op)}, + {"EuclideanNorm", CreatorFunction(translate_direct_reduce_op)}, + {"Max", CreatorFunction(translate_direct_reduce_op)}, + {"Mean", CreatorFunction(translate_direct_reduce_op)}, + {"Min", CreatorFunction(translate_direct_reduce_op)}, + {"Prod", CreatorFunction(translate_direct_reduce_op)}, + {"Sum", CreatorFunction(translate_direct_reduce_op)}, // Separate translators: - {"AddN", translate_add_n_op}, - {"ArgMax", translate_arg_max_op}, - {"ArgMin", translate_arg_min_op}, - {"Assert", translate_assert_op}, - {"AvgPool", translate_avg_pool_op}, - {"AvgPool3D", translate_avg_pool_op}, - {"BatchMatMul", translate_batch_mat_mul_op}, - {"BatchMatMulV2", translate_batch_mat_mul_op}, - {"BatchToSpaceND", translate_batch_to_space_nd_op}, - {"BroadcastArgs", translate_broadcast_args_op}, - {"BroadcastTo", translate_broadcast_to_op}, - {"Bucketize", translate_bucketize_op}, - {"BiasAdd", translate_bias_add_op}, - {"Cast", translate_cast_op}, - {"ClipByValue", translate_clip_by_value_op}, - {"Concat", translate_concat_op}, - {"ConcatV2", translate_concat_op}, - {"Const", translate_const_op}, - {"Conv2D", translate_conv_2d_op}, - {"Conv2DBackpropInput", translate_conv_2d_backprop_input_op}, - {"Conv3D", translate_conv_3d_op}, - {"Conv3DBackpropInputV2", translate_conv_3d_backprop_input_v2_op}, - {"CropAndResize", translate_crop_and_resize_op}, - {"CTCGreedyDecoder", translate_ctc_greedy_decoder_op}, - {"CTCLoss", translate_ctc_loss_op}, - {"Cumsum", translate_cumsum_op}, - {"DepthToSpace", translate_depth_to_space_op}, - {"DepthwiseConv2dNative", translate_depthwise_conv_2d_native_op}, - {"DynamicPartition", translate_dynamic_partition_op}, - {"Einsum", translate_einsum_op}, - {"Elu", translate_elu_op}, - {"EmptyTensorList", translate_tensor_list_reserve_op}, - {"ExpandDims", translate_expand_dims_op}, - {"ExtractImagePatches", translate_extract_image_patches_op}, - {"FakeQuantWithMinMaxVars", translate_fake_quant_op}, - {"FakeQuantWithMinMaxVarsPerChannel", translate_fake_quant_op}, - {"FIFOQueue", translate_fifo_queue_op}, - {"FIFOQueueV2", translate_fifo_queue_op}, - {"Fill", translate_fill_op}, - {"FloorDiv", translate_floor_div_op}, - {"FusedBatchNorm", translate_fused_batch_norm_op}, - {"FusedBatchNormV2", translate_fused_batch_norm_op}, - {"FusedBatchNormV3", translate_fused_batch_norm_op}, - {"Gather", translate_gather_op}, - {"GatherV2", translate_gather_v2_op}, - {"GatherNd", translate_gather_nd_op}, - {"HashTable", translate_hash_table_op}, - {"HashTableV2", translate_hash_table_op}, - {"Identity", translate_identity_op}, - {"IdentityN", translate_identity_n_op}, - {"If", translate_if_op}, - {"input_arg", translate_input_arg_op}, - {"Iterator", translate_iterator_op}, - {"IteratorGetNext", translate_iterator_get_next_op}, - {"IteratorV2", translate_iterator_op}, - {"output_arg", translate_output_arg_op}, - {"L2Loss", translate_l2_loss_op}, - {"LeakyRelu", translate_leaky_relu_op}, - {"LinSpace", translate_linspace_op}, - {"ListDiff", translate_list_diff_op}, - {"LogSoftmax", translate_log_softmax_op}, - {"Log1p", translate_log_1p_op}, - {"LookupTableInsert", translate_lookup_table_insert_op}, - {"LookupTableInsertV2", translate_lookup_table_insert_op}, - {"LRN", translate_lrn_op}, - {"MatMul", translate_mat_mul_op}, - {"MatrixDiag", translate_matrix_diag_op}, - {"MaxPool", translate_max_pool_op}, - {"MaxPoolV2", translate_max_pool_op}, - {"MaxPool3D", translate_max_pool_op}, - {"MirrorPad", translate_mirror_pad_op}, - {"MutableHashTable", translate_hash_table_op}, - {"MutableHashTableV2", translate_hash_table_op}, - {"NonMaxSuppression", translate_non_max_suppression_op}, - {"NonMaxSuppressionV2", translate_non_max_suppression_op}, - {"NonMaxSuppressionV3", translate_non_max_suppression_op}, - {"NonMaxSuppressionV4", translate_non_max_suppression_op}, - {"NonMaxSuppressionV5", translate_non_max_suppression_op}, - {"NoOp", translate_no_op}, // do nothing - {"OneHot", translate_one_hot_op}, - {"OneShotIterator", translate_iterator_op}, - {"Pack", translate_pack_op}, - {"Pad", translate_pad_op}, - {"PadV2", translate_padv2_op}, - {"QueueDequeue", translate_queue_dequeue_op}, - {"QueueDequeueV2", translate_queue_dequeue_op}, - {"QueueDequeueUpTo", translate_queue_dequeue_many_op}, - {"QueueDequeueUpToV2", translate_queue_dequeue_many_op}, - {"QueueDequeueMany", translate_queue_dequeue_many_op}, - {"DynamicStitch", translate_parallel_dynamic_stitch_op}, - {"ParallelDynamicStitch", translate_parallel_dynamic_stitch_op}, - {"PartitionedCall", translate_partitioned_call_op}, - {"Placeholder", translate_placeholder_op}, - {"PlaceholderWithDefault", translate_placeholder_with_default_op}, - {"PreventGradient", translate_identity_op}, - {"Range", translate_range_op}, - {"Rank", translate_rank_op}, - {"RandomUniform", translate_random_uniform_op}, - {"RandomUniformInt", translate_random_uniform_int_op}, - {"Reciprocal", translate_reciprocal_op}, - {"Relu6", translate_relu_6_op}, - {"Reshape", translate_reshape_op}, - {"Reverse", translate_reverse_op}, - {"ReverseSequence", translate_reverse_sequence_op}, - {"ReverseV2", translate_reverse_v2_op}, - {"ResizeBilinear", translate_interpolate_op}, - {"ResizeNearestNeighbor", translate_interpolate_op}, - {"ResourceGather", translate_resource_gather_op}, - {"Roll", translate_roll_op}, - {"Round", translate_round_op}, - {"Rsqrt", translate_rsqrt_op}, - {"SaveV2", translate_no_op}, - {"ScatterNd", translate_scatter_nd_op}, - {"SegmentSum", translate_segment_sum_op}, - {"SparseToDense", translate_sparse_to_dense_op}, - {"Select", translate_select_op}, - {"SelectV2", translate_select_v2_op}, - {"Shape", translate_shape_op}, - {"Size", translate_size_op}, - {"Slice", translate_slice_op}, - {"Snapshot", translate_identity_op}, - {"Softmax", translate_softmax_op}, - {"SpaceToDepth", translate_space_to_depth_op}, - {"SparseReshape", translate_sparse_reshape_op}, - {"Split", translate_split_op}, - {"SplitV", translate_split_v_op}, - {"StopGradient", translate_identity_op}, - {"Sqrt", translate_sqrt_op}, - {"Square", translate_square_op}, - {"Squeeze", translate_squeeze_op}, - {"SpaceToBatchND", translate_space_to_batch_nd_op}, - {"StatefulPartitionedCall", translate_partitioned_call_op}, - {"StatelessIf", translate_if_op}, - {"StatelessWhile", translate_while_op}, - {"StridedSlice", translate_strided_slice_op}, - {"TensorListFromTensor", translate_tensor_list_from_tensor_op}, - {"TensorListGetItem", translate_tensor_list_get_item_op}, - {"TensorListPushBack", translate_tensor_list_push_back_op}, - {"TensorListSetItem", translate_tensor_list_set_item_op}, - {"TensorListStack", translate_tensor_list_stack_op}, - {"TensorListReserve", translate_tensor_list_reserve_op}, - {"Tile", translate_tile_op}, - {"TopK", translate_top_k_op}, - {"TopKV2", translate_top_k_v2_op}, - {"Transpose", translate_transpose_op}, - {"Unpack", translate_unpack_op}, - {"While", translate_while_op}, - {"Where", translate_where_op}, - {"Xdivy", translate_x_div_y_op}, - {"ZerosLike", translate_zeros_like_op}, + {"AddN", CreatorFunction(translate_add_n_op)}, + {"ArgMax", CreatorFunction(translate_arg_max_op)}, + {"ArgMin", CreatorFunction(translate_arg_min_op)}, + {"Assert", CreatorFunction(translate_no_op)}, + {"AvgPool", CreatorFunction(translate_avg_pool_op)}, + {"AvgPool3D", CreatorFunction(translate_avg_pool_op)}, + {"BatchMatMul", CreatorFunction(translate_batch_mat_mul_op)}, + {"BatchMatMulV2", CreatorFunction(translate_batch_mat_mul_op)}, + {"BatchToSpaceND", CreatorFunction(translate_batch_to_space_nd_op)}, + {"BroadcastArgs", CreatorFunction(translate_broadcast_args_op)}, + {"BroadcastTo", CreatorFunction(translate_broadcast_to_op)}, + {"Bucketize", CreatorFunction(translate_bucketize_op)}, + {"BiasAdd", CreatorFunction(translate_bias_add_op)}, + {"Cast", CreatorFunction(translate_cast_op)}, + {"ClipByValue", CreatorFunction(translate_clip_by_value_op)}, + {"Concat", CreatorFunction(translate_concat_op)}, + {"ConcatV2", CreatorFunction(translate_concat_op)}, + {"Const", CreatorFunction(translate_const_op)}, + {"Conv2D", CreatorFunction(translate_conv_2d_op)}, + {"Conv2DBackpropInput", CreatorFunction(translate_conv_2d_backprop_input_op)}, + {"Conv3D", CreatorFunction(translate_conv_3d_op)}, + {"Conv3DBackpropInputV2", CreatorFunction(translate_conv_3d_backprop_input_v2_op)}, + {"CropAndResize", CreatorFunction(translate_crop_and_resize_op)}, + {"CTCGreedyDecoder", CreatorFunction(translate_ctc_greedy_decoder_op)}, + {"CTCLoss", CreatorFunction(translate_ctc_loss_op)}, + {"Cumsum", CreatorFunction(translate_cumsum_op)}, + {"DepthToSpace", CreatorFunction(translate_depth_to_space_op)}, + {"DepthwiseConv2dNative", CreatorFunction(translate_depthwise_conv_2d_native_op)}, + {"DynamicPartition", CreatorFunction(translate_dynamic_partition_op)}, + {"Einsum", CreatorFunction(translate_einsum_op)}, + {"Elu", CreatorFunction(translate_elu_op)}, + {"EmptyTensorList", CreatorFunction(translate_tensor_list_reserve_op)}, + {"ExpandDims", CreatorFunction(translate_expand_dims_op)}, + {"ExtractImagePatches", CreatorFunction(translate_extract_image_patches_op)}, + {"FakeQuantWithMinMaxVars", CreatorFunction(translate_fake_quant_op)}, + {"FakeQuantWithMinMaxVarsPerChannel", CreatorFunction(translate_fake_quant_op)}, + {"FIFOQueue", CreatorFunction(translate_fifo_queue_op)}, + {"FIFOQueueV2", CreatorFunction(translate_fifo_queue_op)}, + {"Fill", CreatorFunction(translate_fill_op)}, + {"FloorDiv", CreatorFunction(translate_floor_div_op)}, + {"FusedBatchNorm", CreatorFunction(translate_fused_batch_norm_op)}, + {"FusedBatchNormV2", CreatorFunction(translate_fused_batch_norm_op)}, + {"FusedBatchNormV3", CreatorFunction(translate_fused_batch_norm_op)}, + {"Gather", CreatorFunction(translate_gather_op)}, + {"GatherV2", CreatorFunction(translate_gather_v2_op)}, + {"GatherNd", CreatorFunction(translate_gather_nd_op)}, + {"HashTable", CreatorFunction(translate_hash_table_op)}, + {"HashTableV2", CreatorFunction(translate_hash_table_op)}, + {"Identity", CreatorFunction(translate_identity_op)}, + {"IdentityN", CreatorFunction(translate_identity_n_op)}, + {"If", CreatorFunction(translate_if_op)}, + {"input_arg", CreatorFunction(translate_input_arg_op)}, + {"Iterator", CreatorFunction(translate_iterator_op)}, + {"IteratorGetNext", CreatorFunction(translate_iterator_get_next_op)}, + {"IteratorV2", CreatorFunction(translate_iterator_op)}, + {"output_arg", CreatorFunction(translate_output_arg_op)}, + {"L2Loss", CreatorFunction(translate_l2_loss_op)}, + {"LeakyRelu", CreatorFunction(translate_leaky_relu_op)}, + {"LinSpace", CreatorFunction(translate_linspace_op)}, + {"ListDiff", CreatorFunction(translate_list_diff_op)}, + {"LogSoftmax", CreatorFunction(translate_log_softmax_op)}, + {"Log1p", CreatorFunction(translate_log_1p_op)}, + {"LookupTableInsert", CreatorFunction(translate_no_op)}, + {"LookupTableInsertV2", CreatorFunction(translate_no_op)}, + {"LRN", CreatorFunction(translate_lrn_op)}, + {"MatMul", CreatorFunction(translate_mat_mul_op)}, + {"MatrixDiag", CreatorFunction(translate_matrix_diag_op)}, + {"MaxPool", CreatorFunction(translate_max_pool_op)}, + {"MaxPoolV2", CreatorFunction(translate_max_pool_op)}, + {"MaxPool3D", CreatorFunction(translate_max_pool_op)}, + {"MirrorPad", CreatorFunction(translate_mirror_pad_op)}, + {"MutableHashTable", CreatorFunction(translate_hash_table_op)}, + {"MutableHashTableV2", CreatorFunction(translate_hash_table_op)}, + {"NonMaxSuppression", CreatorFunction(translate_non_max_suppression_op)}, + {"NonMaxSuppressionV2", CreatorFunction(translate_non_max_suppression_op)}, + {"NonMaxSuppressionV3", CreatorFunction(translate_non_max_suppression_op)}, + {"NonMaxSuppressionV4", CreatorFunction(translate_non_max_suppression_op)}, + {"NonMaxSuppressionV5", CreatorFunction(translate_non_max_suppression_op)}, + {"NoOp", CreatorFunction(translate_no_op)}, // do nothing + {"OneHot", CreatorFunction(translate_one_hot_op)}, + {"OneShotIterator", CreatorFunction(translate_iterator_op)}, + {"Pack", CreatorFunction(translate_pack_op)}, + {"Pad", CreatorFunction(translate_pad_op)}, + {"PadV2", CreatorFunction(translate_padv2_op)}, + {"QueueDequeue", CreatorFunction(translate_queue_dequeue_op)}, + {"QueueDequeueV2", CreatorFunction(translate_queue_dequeue_op)}, + {"QueueDequeueUpTo", CreatorFunction(translate_queue_dequeue_many_op)}, + {"QueueDequeueUpToV2", CreatorFunction(translate_queue_dequeue_many_op)}, + {"QueueDequeueMany", CreatorFunction(translate_queue_dequeue_many_op)}, + {"DynamicStitch", CreatorFunction(translate_parallel_dynamic_stitch_op)}, + {"ParallelDynamicStitch", CreatorFunction(translate_parallel_dynamic_stitch_op)}, + {"PartitionedCall", CreatorFunction(translate_partitioned_call_op)}, + {"Placeholder", CreatorFunction(translate_placeholder_op)}, + {"PlaceholderWithDefault", CreatorFunction(translate_placeholder_with_default_op)}, + {"PreventGradient", CreatorFunction(translate_identity_op)}, + {"Range", CreatorFunction(translate_range_op)}, + {"Rank", CreatorFunction(translate_rank_op)}, + {"RandomUniform", CreatorFunction(translate_random_uniform_op)}, + {"RandomUniformInt", CreatorFunction(translate_random_uniform_int_op)}, + {"Reciprocal", CreatorFunction(translate_reciprocal_op)}, + {"Relu6", CreatorFunction(translate_relu_6_op)}, + {"Reshape", CreatorFunction(translate_reshape_op)}, + {"Reverse", CreatorFunction(translate_reverse_op)}, + {"ReverseSequence", CreatorFunction(translate_reverse_sequence_op)}, + {"ReverseV2", CreatorFunction(translate_reverse_v2_op)}, + {"ResizeBilinear", CreatorFunction(translate_interpolate_op)}, + {"ResizeNearestNeighbor", CreatorFunction(translate_interpolate_op)}, + {"ResourceGather", CreatorFunction(translate_resource_gather_op)}, + {"Roll", CreatorFunction(translate_roll_op)}, + {"Round", CreatorFunction(translate_round_op)}, + {"Rsqrt", CreatorFunction(translate_rsqrt_op)}, + {"SaveV2", CreatorFunction(translate_no_op)}, + {"ScatterNd", CreatorFunction(translate_scatter_nd_op)}, + {"SegmentSum", CreatorFunction(translate_segment_sum_op)}, + {"SparseToDense", CreatorFunction(translate_sparse_to_dense_op)}, + {"Select", CreatorFunction(translate_select_op)}, + {"SelectV2", CreatorFunction(translate_select_v2_op)}, + {"Shape", CreatorFunction(translate_shape_op)}, + {"Size", CreatorFunction(translate_size_op)}, + {"Slice", CreatorFunction(translate_slice_op)}, + {"Snapshot", CreatorFunction(translate_identity_op)}, + {"Softmax", CreatorFunction(translate_softmax_op)}, + {"SpaceToDepth", CreatorFunction(translate_space_to_depth_op)}, + {"SparseReshape", CreatorFunction(translate_sparse_reshape_op)}, + {"Split", CreatorFunction(translate_split_op)}, + {"SplitV", CreatorFunction(translate_split_v_op)}, + {"StopGradient", CreatorFunction(translate_identity_op)}, + {"Sqrt", CreatorFunction(translate_sqrt_op)}, + {"Square", CreatorFunction(translate_square_op)}, + {"Squeeze", CreatorFunction(translate_squeeze_op)}, + {"SpaceToBatchND", CreatorFunction(translate_space_to_batch_nd_op)}, + {"StatefulPartitionedCall", CreatorFunction(translate_partitioned_call_op)}, + {"StatelessIf", CreatorFunction(translate_if_op)}, + {"StatelessWhile", CreatorFunction(translate_while_op)}, + {"StridedSlice", CreatorFunction(translate_strided_slice_op)}, + {"TensorListFromTensor", CreatorFunction(translate_tensor_list_from_tensor_op)}, + {"TensorListGetItem", CreatorFunction(translate_tensor_list_get_item_op)}, + {"TensorListPushBack", CreatorFunction(translate_tensor_list_push_back_op)}, + {"TensorListSetItem", CreatorFunction(translate_tensor_list_set_item_op)}, + {"TensorListStack", CreatorFunction(translate_tensor_list_stack_op)}, + {"TensorListReserve", CreatorFunction(translate_tensor_list_reserve_op)}, + {"Tile", CreatorFunction(translate_tile_op)}, + {"TopK", CreatorFunction(translate_top_k_op)}, + {"TopKV2", CreatorFunction(translate_top_k_v2_op)}, + {"Transpose", CreatorFunction(translate_transpose_op)}, + {"ReadVariableOp", CreatorFunction(translate_readvariable_op)}, + {"AssignVariableOp", CreatorFunction(translate_assignvariable_op)}, + {"VarIsInitializedOp", CreatorFunction(translate_varisinitialized_op)}, + {"VarHandleOp", CreatorFunction(translate_varhandle_op)}, + {"RestoreV2", CreatorFunction(translate_restorev2_op)}, + {"StaticRegexFullMatch", CreatorFunction(translate_staticregexfullmatch_op)}, + {"StringJoin", CreatorFunction(translate_stringjoin_op)}, + {"ShardedFilename", CreatorFunction(translate_identity_op)}, + {"MergeV2Checkpoints", CreatorFunction(translate_identity_op)}, + {"Unpack", CreatorFunction(translate_unpack_op)}, + {"While", CreatorFunction(translate_while_op)}, + {"Where", CreatorFunction(translate_where_op)}, + {"Xdivy", CreatorFunction(translate_x_div_y_op)}, + {"ZerosLike", CreatorFunction(translate_zeros_like_op)}, // Translators for internal operations - {"BlockLSTM", translate_block_lstm_op}, - {"GRUBlockCell", translate_gru_block_cell_op}, - {"SparseFillEmptyRows", translate_sparse_fill_empty_rows_op}, - {"SparseSegmentSum", translate_sparse_segment_sum_op}, - {"Unique", translate_unique_op}, + {"BlockLSTM", CreatorFunction(translate_block_lstm_op)}, + {"GRUBlockCell", CreatorFunction(translate_gru_block_cell_op)}, + {"SparseFillEmptyRows", CreatorFunction(translate_sparse_fill_empty_rows_op)}, + {"SparseSegmentSum", CreatorFunction(translate_sparse_segment_sum_op)}, + {"Unique", CreatorFunction(translate_unique_op)}, }; }; } // namespace op diff --git a/src/frontends/tensorflow/src/op_table.hpp b/src/frontends/tensorflow/src/op_table.hpp index cac20c1ebe10ad..6c0d5f9019deb7 100644 --- a/src/frontends/tensorflow/src/op_table.hpp +++ b/src/frontends/tensorflow/src/op_table.hpp @@ -10,7 +10,6 @@ #include "openvino/core/node_vector.hpp" #include "openvino/frontend/tensorflow/node_context.hpp" -#include "openvino_conversions.hpp" #include "utils.hpp" namespace ov { diff --git a/src/frontends/tensorflow/src/proto/any.proto b/src/frontends/tensorflow/src/proto/any.proto new file mode 100644 index 00000000000000..a52639dc72b3d1 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/any.proto @@ -0,0 +1,159 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// ==== +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/src/frontends/tensorflow/src/proto/meta_graph.proto b/src/frontends/tensorflow/src/proto/meta_graph.proto new file mode 100644 index 00000000000000..b6918fa853bf8c --- /dev/null +++ b/src/frontends/tensorflow/src/proto/meta_graph.proto @@ -0,0 +1,351 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2018-2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +import "any.proto"; +import "graph.proto"; +import "op_def.proto"; +import "tensor_shape.proto"; +import "types.proto"; +import "saved_object_graph.proto"; +import "saver.proto"; +import "struct.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "MetaGraphProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// Protocol buffer containing the following which are necessary to restart +// training, run inference. It can be used to serialize/de-serialize memory +// objects necessary for running computation in a graph when crossing the +// process boundary. It can be used for long term storage of graphs, +// cross-language execution of graphs, etc. +// MetaInfoDef +// GraphDef +// SaverDef +// CollectionDef +// TensorInfo +// SignatureDef +message MetaGraphDef { + // Meta information regarding the graph to be exported. To be used by users + // of this protocol buffer to encode information regarding their meta graph. + message MetaInfoDef { + // User specified Version string. Can be the name of the model and revision, + // steps this model has been trained to, etc. + string meta_graph_version = 1; + + // A copy of the OpDefs used by the producer of this graph_def. + // Descriptions and Ops not used in graph_def are stripped out. + OpList stripped_op_list = 2; + + // A serialized protobuf. Can be the time this meta graph is created, or + // modified, or name of the model. + google.protobuf.Any any_info = 3; + + // User supplied tag(s) on the meta_graph and included graph_def. + // + // MetaGraphDefs should be tagged with their capabilities or use-cases. + // Examples: "train", "serve", "gpu", "tpu", etc. + // These tags enable loaders to access the MetaGraph(s) appropriate for a + // specific use-case or runtime environment. + repeated string tags = 4; + + // The __version__ string of the tensorflow build used to write this graph. + // This will be populated by the framework, which will overwrite any user + // supplied value. + string tensorflow_version = 5; + + // The __git_version__ string of the tensorflow build used to write this + // graph. This will be populated by the framework, which will overwrite any + // user supplied value. + string tensorflow_git_version = 6; + + // A flag to denote whether default-valued attrs have been stripped from + // the nodes in this graph_def. + bool stripped_default_attrs = 7; + + // FunctionDef name to aliases mapping. + map function_aliases = 8; + } + MetaInfoDef meta_info_def = 1; + + // GraphDef. + GraphDef graph_def = 2; + + // SaverDef. + SaverDef saver_def = 3; + + // collection_def: Map from collection name to collections. + // See CollectionDef section for details. + map collection_def = 4; + + // signature_def: Map from user supplied key for a signature to a single + // SignatureDef. + map signature_def = 5; + + // Asset file def to be used with the defined graph. + repeated AssetFileDef asset_file_def = 6; + + // Extra information about the structure of functions and stateful objects. + SavedObjectGraph object_graph_def = 7; +} + +// CollectionDef should cover most collections. +// To add a user-defined collection, do one of the following: +// 1. For simple data types, such as string, int, float: +// tf.add_to_collection("your_collection_name", your_simple_value) +// strings will be stored as bytes_list. +// +// 2. For Protobuf types, there are three ways to add them: +// 1) tf.add_to_collection("your_collection_name", +// your_proto.SerializeToString()) +// +// collection_def { +// key: "user_defined_bytes_collection" +// value { +// bytes_list { +// value: "queue_name: \"test_queue\"\n" +// } +// } +// } +// +// or +// +// 2) tf.add_to_collection("your_collection_name", str(your_proto)) +// +// collection_def { +// key: "user_defined_string_collection" +// value { +// bytes_list { +// value: "\n\ntest_queue" +// } +// } +// } +// +// or +// +// 3) any_buf = any_pb2.Any() +// tf.add_to_collection("your_collection_name", +// any_buf.Pack(your_proto)) +// +// collection_def { +// key: "user_defined_any_collection" +// value { +// any_list { +// value { +// type_url: "type.googleapis.com/tensorflow.QueueRunnerDef" +// value: "\n\ntest_queue" +// } +// } +// } +// } +// +// 3. For Python objects, implement to_proto() and from_proto(), and register +// them in the following manner: +// ops.register_proto_function("your_collection_name", +// proto_type, +// to_proto=YourPythonObject.to_proto, +// from_proto=YourPythonObject.from_proto) +// These functions will be invoked to serialize and de-serialize the +// collection. For example, +// ops.register_proto_function(ops.GraphKeys.GLOBAL_VARIABLES, +// proto_type=variable_pb2.VariableDef, +// to_proto=Variable.to_proto, +// from_proto=Variable.from_proto) +message CollectionDef { + // NodeList is used for collecting nodes in graph. For example + // collection_def { + // key: "summaries" + // value { + // node_list { + // value: "input_producer/ScalarSummary:0" + // value: "shuffle_batch/ScalarSummary:0" + // value: "ImageSummary:0" + // } + // } + message NodeList { + repeated string value = 1; + } + + // BytesList is used for collecting strings and serialized protobufs. For + // example: + // collection_def { + // key: "trainable_variables" + // value { + // bytes_list { + // value: "\n\017conv1/weights:0\022\024conv1/weights/Assign + // \032\024conv1/weights/read:0" + // value: "\n\016conv1/biases:0\022\023conv1/biases/Assign\032 + // \023conv1/biases/read:0" + // } + // } + // } + message BytesList { + repeated bytes value = 1; + } + + // Int64List is used for collecting int, int64 and long values. + message Int64List { + repeated int64 value = 1 [packed = true]; + } + + // FloatList is used for collecting float values. + message FloatList { + repeated float value = 1 [packed = true]; + } + + // AnyList is used for collecting Any protos. + message AnyList { + repeated google.protobuf.Any value = 1; + } + + oneof kind { + NodeList node_list = 1; + BytesList bytes_list = 2; + Int64List int64_list = 3; + FloatList float_list = 4; + AnyList any_list = 5; + } +} + +// Information about a Tensor necessary for feeding or retrieval. +message TensorInfo { + // For sparse tensors, The COO encoding stores a triple of values, indices, + // and shape. + message CooSparse { + // The shape of the values Tensor is [?]. Its dtype must be the dtype of + // the SparseTensor as a whole, given in the enclosing TensorInfo. + string values_tensor_name = 1; + + // The indices Tensor must have dtype int64 and shape [?, ?]. + string indices_tensor_name = 2; + + // The dynamic logical shape represented by the SparseTensor is recorded in + // the Tensor referenced here. It must have dtype int64 and shape [?]. + string dense_shape_tensor_name = 3; + } + + // Generic encoding for composite tensors. + message CompositeTensor { + // The serialized TypeSpec for the composite tensor. + TypeSpecProto type_spec = 1; + + // A TensorInfo for each flattened component tensor. + repeated TensorInfo components = 2; + } + + oneof encoding { + // For dense `Tensor`s, the name of the tensor in the graph. + string name = 1; + // There are many possible encodings of sparse matrices + // (https://en.wikipedia.org/wiki/Sparse_matrix). Currently, TensorFlow + // uses only the COO encoding. This is supported and documented in the + // SparseTensor Python class. + CooSparse coo_sparse = 4; + // Generic encoding for CompositeTensors. + CompositeTensor composite_tensor = 5; + } + DataType dtype = 2; + // The static shape should be recorded here, to the extent that it can + // be known in advance. In the case of a SparseTensor, this field describes + // the logical shape of the represented tensor (aka dense_shape). + TensorShapeProto tensor_shape = 3; +} + +// SignatureDef defines the signature of a computation supported by a TensorFlow +// graph. +// +// For example, a model with two loss computations, sharing a single input, +// might have the following signature_def map, in a MetaGraphDef message. +// +// Note that across the two SignatureDefs "loss_A" and "loss_B", the input key, +// output key, and method_name are identical, and will be used by system(s) that +// implement or rely upon this particular loss method. The output tensor names +// differ, demonstrating how different outputs can exist for the same method. +// +// signature_def { +// key: "loss_A" +// value { +// inputs { +// key: "input" +// value { +// name: "input:0" +// dtype: DT_STRING +// tensor_shape: ... +// } +// } +// outputs { +// key: "loss_output" +// value { +// name: "loss_output_A:0" +// dtype: DT_FLOAT +// tensor_shape: ... +// } +// } +// method_name: "some/package/compute_loss" +// } +// ... +// } +// signature_def { +// key: "loss_B" +// value { +// inputs { +// key: "input" +// value { +// name: "input:0" +// dtype: DT_STRING +// tensor_shape: ... +// } +// } +// outputs { +// key: "loss_output" +// value { +// name: "loss_output_B:0" +// dtype: DT_FLOAT +// tensor_shape: ... +// } +// } +// method_name: "some/package/compute_loss" +// } +// ... +// } +message SignatureDef { + // Named input parameters. + map inputs = 1; + // Named output parameters. + map outputs = 2; + // Extensible method_name information enabling third-party users to mark a + // SignatureDef as supporting a particular method. This enables producers and + // consumers of SignatureDefs, e.g. a model definition library and a serving + // library to have a clear hand-off regarding the semantics of a computation. + // + // Note that multiple SignatureDefs in a single MetaGraphDef may have the same + // method_name. This is commonly used to support multi-headed computation, + // where a single graph computation may return multiple results. + string method_name = 3; +} + +// An asset file def for a single file or a set of sharded files with the same +// name. +message AssetFileDef { + // The tensor to bind the asset filename to. + TensorInfo tensor_info = 1; + // The filename within an assets directory. Note: does not include the path + // prefix, i.e. directories. For an asset at /tmp/path/vocab.txt, the filename + // would be "vocab.txt". + string filename = 2; +} diff --git a/src/frontends/tensorflow/src/proto/saved_model.proto b/src/frontends/tensorflow/src/proto/saved_model.proto new file mode 100644 index 00000000000000..0034fdfd46dcf8 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/saved_model.proto @@ -0,0 +1,35 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +import "meta_graph.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "SavedModelProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// SavedModel is the high level serialization format for TensorFlow Models. +// See [todo: doc links, similar to session_bundle] for more information. +message SavedModel { + // The schema version of the SavedModel instance. Used for versioning when + // making future changes to the specification/implementation. Initial value + // at release will be 1. + int64 saved_model_schema_version = 1; + + // One or more MetaGraphs. + repeated MetaGraphDef meta_graphs = 2; +} diff --git a/src/frontends/tensorflow/src/proto/saved_object_graph.proto b/src/frontends/tensorflow/src/proto/saved_object_graph.proto new file mode 100644 index 00000000000000..671441075c3628 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/saved_object_graph.proto @@ -0,0 +1,263 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +import "any.proto"; +import "tensor_shape.proto"; +import "types.proto"; +import "variable.proto"; +import "versions.proto"; +import "struct.proto"; +import "trackable_object_graph.proto"; + +option cc_enable_arenas = true; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It +// describes the directed graph of Python objects (or equivalent in other +// languages) that make up a model, with nodes[0] at the root. + +// SavedObjectGraph shares some structure with TrackableObjectGraph, but +// SavedObjectGraph belongs to the MetaGraph and contains pointers to functions +// and type information, while TrackableObjectGraph lives in the checkpoint +// and contains pointers only to variable values. + +message SavedObjectGraph { + // Flattened list of objects in the object graph. + // + // The position of the object in this list indicates its id. + // Nodes[0] is considered the root node. + repeated SavedObject nodes = 1; + + // Information about captures and output structures in concrete functions. + // Referenced from SavedBareConcreteFunction and SavedFunction. + map concrete_functions = 2; +} + +message SavedObject { + // Objects which this object depends on: named edges in the dependency + // graph. + // + // Note: All kinds of SavedObject may have children, except + // "constant" and "captured_tensor". + repeated TrackableObjectGraph.TrackableObject.ObjectReference children = 1; + + // Ordered list of dependencies that must be loaded before this object. + // SavedModel loads with the bottom-up approach, by first creating all objects + // (in the order defined by the dependencies), then connecting the edges. + repeated TrackableObjectGraph.TrackableObject.ObjectReference dependencies = + 15; + + // Removed when forking SavedObject from TrackableObjectGraph. + reserved "attributes"; + reserved 2; + + // Slot variables owned by this object. This describes the three-way + // (optimizer, variable, slot variable) relationship; none of the three + // depend on the others directly. + // + // Note: currently only valid if kind == "user_object". + repeated TrackableObjectGraph.TrackableObject.SlotVariableReference + slot_variables = 3; + + oneof kind { + SavedUserObject user_object = 4; + SavedAsset asset = 5; + SavedFunction function = 6; + SavedVariable variable = 7; + SavedBareConcreteFunction bare_concrete_function = 8; + SavedConstant constant = 9; + SavedResource resource = 10; + CapturedTensor captured_tensor = 12; + } + + // Stores the functions used to save and restore this object. At most one of + // `saveable_objects` or `registered_saver` is defined for each SavedObject. + // See the comment below for the difference between SaveableObject and + // registered savers. + map saveable_objects = 11; + + // The fields below are filled when the user serializes a registered Trackable + // class or an object with a registered saver function. + // + // Registered classes may save additional metadata and supersede the + // default loading process where nodes are recreated from the proto. + // If the registered class cannot be found, then the object will load as one + // one of the default trackable objects: Autotrackable (a class similar to + // tf.Module), tf.function, or tf.Variable. + // + // Unlike SaveableObjects, which store the functions for saving and restoring + // from tensors, registered savers allow Trackables to write checkpoint shards + // directly (e.g. for performance or coordination reasons). + // *All registered savers must be available when loading the SavedModel.* + + // The name of the registered class of the form "{package}.{class_name}". + // This field is used to search for the registered class at loading time. + string registered_name = 13; + // The user-generated proto storing metadata for this object, to be passed to + // the registered classes's _deserialize_from_proto method when this object is + // loaded from the SavedModel. + google.protobuf.Any serialized_user_proto = 14; + + // String name of the registered saver. At most one of `saveable_objects` or + // `registered_saver` is defined for each SavedObject. + string registered_saver = 16; +} + +// A SavedUserObject is an object (in the object-oriented language of the +// TensorFlow program) of some user- or framework-defined class other than +// those handled specifically by the other kinds of SavedObjects. +// +// This object cannot be evaluated as a tensor, and therefore cannot be bound +// to an input of a function. +message SavedUserObject { + // Corresponds to a registration of the type to use in the loading program. + string identifier = 1; + // Version information from the producer of this SavedUserObject. + VersionDef version = 2; + // Metadata for deserializing this object. + // + // Deprecated! At the time of deprecation, Keras was the only user of this + // field, and its saving and loading code will be updated shortly. + // Please save your application-specific metadata to a separate file. + string metadata = 3 [deprecated = true]; +} + +// A SavedAsset points to an asset in the MetaGraph. +// +// When bound to a function this object evaluates to a tensor with the absolute +// filename. Users should not depend on a particular part of the filename to +// remain stable (e.g. basename could be changed). +message SavedAsset { + // Index into `MetaGraphDef.asset_file_def[]` that describes the Asset. + // + // Only the field `AssetFileDef.filename` is used. Other fields, such as + // `AssetFileDef.tensor_info`, MUST be ignored. + int32 asset_file_def_index = 1; +} + +// A function with multiple signatures, possibly with non-Tensor arguments. +message SavedFunction { + repeated string concrete_functions = 1; + FunctionSpec function_spec = 2; +} + +message CapturedTensor { + // Name of captured tensor + string name = 1; + + // Name of concrete function which contains the computed graph tensor. + string concrete_function = 2; +} + +// Stores low-level information about a concrete function. Referenced in either +// a SavedFunction or a SavedBareConcreteFunction. +message SavedConcreteFunction { + repeated int32 bound_inputs = 2; + + // Input in canonicalized form that was received to create this concrete + // function. + StructuredValue canonicalized_input_signature = 3; + // Output that was the return value of this function after replacing all + // Tensors with TensorSpecs. This can be an arbitrary nested function and will + // be used to reconstruct the full structure from pure tensors. + StructuredValue output_signature = 4; +} + +message SavedBareConcreteFunction { + // Identifies a SavedConcreteFunction. + string concrete_function_name = 1; + + // A sequence of unique strings, one per Tensor argument. + repeated string argument_keywords = 2; + // The prefix of `argument_keywords` which may be identified by position. + int64 allowed_positional_arguments = 3; + // The spec of the function that this ConcreteFunction is traced from. This + // allows the ConcreteFunction to be called with nest structure inputs. This + // field may not be populated. If this field is absent, the concrete function + // can only be called with flat inputs. + // TODO(b/169361281): support calling saved ConcreteFunction with structured + // inputs in C++ SavedModel API. + FunctionSpec function_spec = 4; +} + +message SavedConstant { + // An Operation name for a ConstantOp in this SavedObjectGraph's MetaGraph. + string operation = 1; +} + +// Represents a Variable that is initialized by loading the contents from the +// checkpoint. +message SavedVariable { + DataType dtype = 1; + TensorShapeProto shape = 2; + bool trainable = 3; + VariableSynchronization synchronization = 4; + VariableAggregation aggregation = 5; + string name = 6; + string device = 7; + // List of component variables for a distributed variable. + // + // When this field is non-empty, the SavedVariable will be assumed + // to be a distributed variable defined by the components listed here. + // + // This is only supported by experimental loaders at the moment. + repeated SavedVariable experimental_distributed_variable_components = 8; +} + +// Represents `FunctionSpec` used in `Function`. This represents a +// function that has been wrapped as a TensorFlow `Function`. +message FunctionSpec { + // Full arg spec from inspect.getfullargspec(). + StructuredValue fullargspec = 1; + // Whether this represents a class method. + bool is_method = 2; + // The input signature, if specified. + StructuredValue input_signature = 5; + + // Whether the function should be compiled by XLA. + // + // The public interface to `tf.function` uses an optional boolean to + // represent three distinct states for this field. Unfortunately, proto3 + // removes the ability to explicitly check for the presence or absence of a + // field, so we instead map to an enum. + // + // See `tf.function` for details. + enum JitCompile { + DEFAULT = 0; + ON = 1; + OFF = 2; + } + JitCompile jit_compile = 6; + + reserved 3, 4; +} + +// A SavedResource represents a TF object that holds state during its lifetime. +// An object of this type can have a reference to a: +// create_resource() and an initialize() function. +message SavedResource { + // A device specification indicating a required placement for the resource + // creation function, e.g. "CPU". An empty string allows the user to select a + // device. + string device = 1; +} + +message SaveableObject { + // Node ids of concrete functions for saving and loading from a checkpoint. + // These functions save and restore directly from tensors. + int32 save_function = 2; + int32 restore_function = 3; +} diff --git a/src/frontends/tensorflow/src/proto/saved_tensor_slice.proto b/src/frontends/tensorflow/src/proto/saved_tensor_slice.proto new file mode 100644 index 00000000000000..4645b2bdca9b89 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/saved_tensor_slice.proto @@ -0,0 +1,96 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2018-2023 Intel Corporation +// Protocol buffers for saved tensor slices. It's used for the brain tensor +// ops checkpoints and the V3 checkpoints in dist_belief. + +// A checkpoint file is an sstable. The value for each record is a serialized +// SavedTensorSlices message (defined below). +// +// Each checkpoint file has a record with the empty key (""), which corresponds +// to a SavedTensorSlices message that contains a "meta", that serves as a +// table of contents on all the tensor slices saved in this file. Since the key +// is "", it's always the first record in each file. +// +// Each of the rest of the records in a checkpoint stores the raw data of a +// particular tensor slice, in SavedSlice format. The corresponding key is an +// ordered code that encodes the name of the tensor and the slice +// information. The name is also stored in the SaveSlice message for ease of +// debugging and manual examination. +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; +option cc_enable_arenas = true; +option java_outer_classname = "SavedTensorSliceProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.util"; + +import "tensor_shape.proto"; +import "tensor_slice.proto"; +import "tensor.proto"; +import "types.proto"; +import "versions.proto"; + +// Metadata describing the set of slices of the same tensor saved in a +// checkpoint file. +message SavedSliceMeta { + // Name of the tensor. + string name = 1; + + // Shape of the tensor + TensorShapeProto shape = 2; + + // Type of the tensor + DataType type = 3; + + // Explicit list of slices saved in the checkpoint file. + repeated TensorSliceProto slice = 4; +}; + +// Metadata describing the set of tensor slices saved in a checkpoint file. +// It is always stored at the beginning of each checkpoint file. +message SavedTensorSliceMeta { + // Each SavedSliceMeta describes the slices for one tensor. + repeated SavedSliceMeta tensor = 1; + + // Compatibility version of this checkpoint. See core/public/version.h + // for version history. + VersionDef versions = 2; +}; + +// Saved tensor slice: it stores the name of the tensors, the slice, and the +// raw data. +message SavedSlice { + // Name of the tensor that this slice belongs to. This must be identical to + // the name used to encode the key for this record. + string name = 1; + + // Extent of the slice. Must have one entry for each of the dimension of the + // tensor that this slice belongs to. + TensorSliceProto slice = 2; + + // The raw data of the slice is stored as a TensorProto. Only raw data are + // stored (we don't fill in fields such as dtype or tensor_shape). + TensorProto data = 3; +}; + +// Each record in a v3 checkpoint file is a serialized SavedTensorSlices +// message. +message SavedTensorSlices { + // This is only present at the first item of each checkpoint file and serves + // as a table of contents, listing all the tensor slices saved in this file. + SavedTensorSliceMeta meta = 1; + + // This exists in all but the first item of each checkpoint file. + SavedSlice data = 2; +}; diff --git a/src/frontends/tensorflow/src/proto/saver.proto b/src/frontends/tensorflow/src/proto/saver.proto new file mode 100644 index 00000000000000..7834f473e4ccdf --- /dev/null +++ b/src/frontends/tensorflow/src/proto/saver.proto @@ -0,0 +1,60 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +option cc_enable_arenas = true; +option java_outer_classname = "SaverProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.util"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// Protocol buffer representing the configuration of a Saver. +message SaverDef { + // The name of the tensor in which to specify the filename when saving or + // restoring a model checkpoint. + string filename_tensor_name = 1; + + // The operation to run when saving a model checkpoint. + string save_tensor_name = 2; + + // The operation to run when restoring a model checkpoint. + string restore_op_name = 3; + + // Maximum number of checkpoints to keep. If 0, no checkpoints are deleted. + int32 max_to_keep = 4; + + // Shard the save files, one per device that has Variable nodes. + bool sharded = 5; + + // How often to keep an additional checkpoint. If not specified, only the last + // "max_to_keep" checkpoints are kept; if specified, in addition to keeping + // the last "max_to_keep" checkpoints, an additional checkpoint will be kept + // for every n hours of training. + float keep_checkpoint_every_n_hours = 6; + + // A version number that identifies a different on-disk checkpoint format. + // Usually, each subclass of BaseSaverBuilder works with a particular + // version/format. However, it is possible that the same builder may be + // upgraded to support a newer checkpoint format in the future. + enum CheckpointFormatVersion { + // Internal legacy format. + LEGACY = 0; + // Deprecated format: tf.Saver() which works with tensorflow::table::Table. + V1 = 1; + // Current format: more efficient. + V2 = 2; + } + CheckpointFormatVersion version = 7; +} diff --git a/src/frontends/tensorflow/src/proto/struct.proto b/src/frontends/tensorflow/src/proto/struct.proto new file mode 100644 index 00000000000000..d03201b685ac79 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/struct.proto @@ -0,0 +1,172 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +import "tensor.proto"; +import "tensor_shape.proto"; +import "types.proto"; + +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// `StructuredValue` represents a dynamically typed value representing various +// data structures that are inspired by Python data structures typically used in +// TensorFlow functions as inputs and outputs. +// +// For example when saving a Layer there may be a `training` argument. If the +// user passes a boolean True/False, that switches between two concrete +// TensorFlow functions. In order to switch between them in the same way after +// loading the SavedModel, we need to represent "True" and "False". +// +// A more advanced example might be a function which takes a list of +// dictionaries mapping from strings to Tensors. In order to map from +// user-specified arguments `[{"a": tf.constant(1.)}, {"q": tf.constant(3.)}]` +// after load to the right saved TensorFlow function, we need to represent the +// nested structure and the strings, recording that we have a trace for anything +// matching `[{"a": tf.TensorSpec(None, tf.float32)}, {"q": tf.TensorSpec([], +// tf.float64)}]` as an example. +// +// Likewise functions may return nested structures of Tensors, for example +// returning a dictionary mapping from strings to Tensors. In order for the +// loaded function to return the same structure we need to serialize it. +// +// This is an ergonomic aid for working with loaded SavedModels, not a promise +// to serialize all possible function signatures. For example we do not expect +// to pickle generic Python objects, and ideally we'd stay language-agnostic. +message StructuredValue { + // The kind of value. + oneof kind { + // Represents None. + NoneValue none_value = 1; + + // Represents a double-precision floating-point value (a Python `float`). + double float64_value = 11; + // Represents a signed integer value, limited to 64 bits. + // Larger values from Python's arbitrary-precision integers are unsupported. + sint64 int64_value = 12; + // Represents a string of Unicode characters stored in a Python `str`. + // In Python 3, this is exactly what type `str` is. + // In Python 2, this is the UTF-8 encoding of the characters. + // For strings with ASCII characters only (as often used in TensorFlow code) + // there is effectively no difference between the language versions. + // The obsolescent `unicode` type of Python 2 is not supported here. + string string_value = 13; + // Represents a boolean value. + bool bool_value = 14; + + // Represents a TensorShape. + tensorflow.TensorShapeProto tensor_shape_value = 31; + // Represents an enum value for dtype. + tensorflow.DataType tensor_dtype_value = 32; + // Represents a value for tf.TensorSpec. + TensorSpecProto tensor_spec_value = 33; + // Represents a value for tf.TypeSpec. + TypeSpecProto type_spec_value = 34; + // Represents a value for tf.BoundedTensorSpec. + BoundedTensorSpecProto bounded_tensor_spec_value = 35; + + // Represents a list of `Value`. + ListValue list_value = 51; + // Represents a tuple of `Value`. + TupleValue tuple_value = 52; + // Represents a dict `Value`. + DictValue dict_value = 53; + // Represents Python's namedtuple. + NamedTupleValue named_tuple_value = 54; + } +} + +// Represents None. +message NoneValue {} + +// Represents a Python list. +message ListValue { + repeated StructuredValue values = 1; +} + +// Represents a Python tuple. +message TupleValue { + repeated StructuredValue values = 1; +} + +// Represents a Python dict keyed by `str`. +// The comment on Unicode from Value.string_value applies analogously. +message DictValue { + map fields = 1; +} + +// Represents a (key, value) pair. +message PairValue { + string key = 1; + StructuredValue value = 2; +} + +// Represents Python's namedtuple. +message NamedTupleValue { + string name = 1; + repeated PairValue values = 2; +} + +// A protobuf to represent tf.TensorSpec. +message TensorSpecProto { + string name = 1; + tensorflow.TensorShapeProto shape = 2; + tensorflow.DataType dtype = 3; +} + +// A protobuf to represent tf.BoundedTensorSpec. +message BoundedTensorSpecProto { + string name = 1; + tensorflow.TensorShapeProto shape = 2; + tensorflow.DataType dtype = 3; + tensorflow.TensorProto minimum = 4; + tensorflow.TensorProto maximum = 5; +} + +// Represents a tf.TypeSpec +message TypeSpecProto { + enum TypeSpecClass { + UNKNOWN = 0; + SPARSE_TENSOR_SPEC = 1; // tf.SparseTensorSpec + INDEXED_SLICES_SPEC = 2; // tf.IndexedSlicesSpec + RAGGED_TENSOR_SPEC = 3; // tf.RaggedTensorSpec + TENSOR_ARRAY_SPEC = 4; // tf.TensorArraySpec + DATA_DATASET_SPEC = 5; // tf.data.DatasetSpec + DATA_ITERATOR_SPEC = 6; // IteratorSpec from data/ops/iterator_ops.py + OPTIONAL_SPEC = 7; // tf.OptionalSpec + PER_REPLICA_SPEC = 8; // PerReplicaSpec from distribute/values.py + VARIABLE_SPEC = 9; // tf.VariableSpec + ROW_PARTITION_SPEC = 10; // RowPartitionSpec from ragged/row_partition.py + reserved 11; + REGISTERED_TYPE_SPEC = 12; // The type registered as type_spec_class_name. + EXTENSION_TYPE_SPEC = 13; // Subclasses of tf.ExtensionType + } + TypeSpecClass type_spec_class = 1; + + // The value returned by TypeSpec._serialize(). + StructuredValue type_state = 2; + + // The name of the TypeSpec class. + // * If type_spec_class == REGISTERED_TYPE_SPEC, the TypeSpec class is + // the one registered under this name. For types registered outside + // core TensorFlow by an add-on library, that library must be loaded + // before this value can be deserialized by nested_structure_coder. + // * If type_spec_class specifies a particular TypeSpec class, this field is + // redundant with the type_spec_class enum, and is only used for error + // reporting in older binaries that do not know the tupe_spec_class enum. + string type_spec_class_name = 3; + + // The number of flat tensor components required by this TypeSpec. + int32 num_flat_components = 4; +} diff --git a/src/frontends/tensorflow/src/proto/tensor_bundle.proto b/src/frontends/tensorflow/src/proto/tensor_bundle.proto new file mode 100644 index 00000000000000..43fea749b42172 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/tensor_bundle.proto @@ -0,0 +1,78 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +import "tensor_shape.proto"; +import "tensor_slice.proto"; +import "types.proto"; +import "versions.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "TensorBundleProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.util"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// Protos used in the tensor bundle module (tf/core/util/tensor_bundle/). + +// Special header that is associated with a bundle. +// +// TODO(zongheng,zhifengc): maybe in the future, we can add information about +// which binary produced this checkpoint, timestamp, etc. Sometime, these can be +// valuable debugging information. And if needed, these can be used as defensive +// information ensuring reader (binary version) of the checkpoint and the writer +// (binary version) must match within certain range, etc. +message BundleHeaderProto { + // Number of data files in the bundle. + int32 num_shards = 1; + + // An enum indicating the endianness of the platform that produced this + // bundle. A bundle can only be read by a platform with matching endianness. + // Defaults to LITTLE, as most modern platforms are little-endian. + // + // Affects the binary tensor data bytes only, not the metadata in protobufs. + enum Endianness { + LITTLE = 0; + BIG = 1; + } + Endianness endianness = 2; + + // Versioning of the tensor bundle format. + VersionDef version = 3; +} + +// Describes the metadata related to a checkpointed tensor. +message BundleEntryProto { + // The tensor dtype and shape. + DataType dtype = 1; + TensorShapeProto shape = 2; + // The binary content of the tensor lies in: + // File "shard_id": bytes [offset, offset + size). + int32 shard_id = 3; + int64 offset = 4; + int64 size = 5; + + // The CRC32C checksum of the tensor bytes. + fixed32 crc32c = 6; + + // Iff present, this entry represents a partitioned tensor. The previous + // fields are interpreted as follows: + // + // "dtype", "shape": describe the full tensor. + // "shard_id", "offset", "size", "crc32c": all IGNORED. + // These information for each slice can be looked up in their own + // BundleEntryProto, keyed by each "slice_name". + repeated TensorSliceProto slices = 7; +} diff --git a/src/frontends/tensorflow/src/proto/trackable_object_graph.proto b/src/frontends/tensorflow/src/proto/trackable_object_graph.proto new file mode 100644 index 00000000000000..f4a8e4da34f129 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/trackable_object_graph.proto @@ -0,0 +1,92 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/ +// Modification Copyright (C) 2023 Intel Corporation + +syntax = "proto3"; + +package tensorflow; + +import "wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; + +// A TensorBundle addition which saves extra information about the objects which +// own variables, allowing for more robust checkpoint loading into modified +// programs. + +message TrackableObjectGraph { + message TrackableObject { + message ObjectReference { + // An index into `TrackableObjectGraph.nodes`, indicating the object + // being referenced. + int32 node_id = 1; + // A user-provided name for the edge. + string local_name = 2; + } + + message SerializedTensor { + // A name for the Tensor. Simple variables have only one + // `SerializedTensor` named "VARIABLE_VALUE" by convention. This value may + // be restored on object creation as an optimization. + string name = 1; + // The full name of the variable/tensor, if applicable. Used to allow + // name-based loading of checkpoints which were saved using an + // object-based API. Should match the checkpoint key which would have been + // assigned by tf.train.Saver. + string full_name = 2; + // The generated name of the Tensor in the checkpoint. + string checkpoint_key = 3; + // Deprecated bool field for optional restore. This field has never been + // set to True. + reserved "optional_restore"; + reserved 4; + } + + message SlotVariableReference { + // An index into `TrackableObjectGraph.nodes`, indicating the + // variable object this slot was created for. + int32 original_variable_node_id = 1; + // The name of the slot (e.g. "m"/"v"). + string slot_name = 2; + // An index into `TrackableObjectGraph.nodes`, indicating the + // `Object` with the value of the slot variable. + int32 slot_variable_node_id = 3; + } + + // Objects which this object depends on. + repeated ObjectReference children = 1; + // Serialized data specific to this object. + repeated SerializedTensor attributes = 2; + // Slot variables owned by this object. + repeated SlotVariableReference slot_variables = 3; + + // The registered saver used to save this object. If this saver is not + // present when loading the checkpoint, then loading will fail. + RegisteredSaver registered_saver = 4; + + // Whether this object has checkpoint values or descendants with checkpoint + // values. This is computed at save time to avoid traversing the entire + // object graph proto when restoring (which also has to traverse the live + // object graph). + google.protobuf.BoolValue has_checkpoint_values = 5; + } + + repeated TrackableObject nodes = 1; +} + +message RegisteredSaver { + // The name of the registered saver/restore function. + string name = 1; + + // Unique auto-generated name of the object. + string object_name = 2; +} diff --git a/src/frontends/tensorflow/src/proto/wrappers.proto b/src/frontends/tensorflow/src/proto/wrappers.proto new file mode 100644 index 00000000000000..228a92f0ef58f2 --- /dev/null +++ b/src/frontends/tensorflow/src/proto/wrappers.proto @@ -0,0 +1,124 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// Modification Copyright (C) 2023 Intel Corporation + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/src/frontends/tensorflow/src/saved_model.cpp b/src/frontends/tensorflow/src/saved_model.cpp new file mode 100644 index 00000000000000..797c8c2985a52e --- /dev/null +++ b/src/frontends/tensorflow/src/saved_model.cpp @@ -0,0 +1,482 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include + +#include "graph_iterator_saved_model.hpp" +#include "openvino/core/type/element_type.hpp" +#include "tensor_bundle.pb.h" +#include "trackable_object_graph.pb.h" + +#ifdef ENABLE_SNAPPY_COMPRESSION +# include "snappy.h" +#endif + +namespace ov { +namespace frontend { +namespace tensorflow { + +template +static T smReadFixed(const char* ptr) { + T result = 0; + for (uint8_t i = 0; i < sizeof(T); ++i) { + result |= ptr[i] << (i * 8); + } + return result; +} + +template +static T smUnpack(char*& ptr, const char* ptr_end) { + T result = 0; + for (uint8_t i = 0; i < sizeof(T) * 7 && ptr < ptr_end; i += 7) { + T byte = *(ptr++); + if (byte & 0x80) { + result |= ((byte & 0x7F) << i); + } else { + result |= byte << i; + return result; + } + } + return 0; +} + +struct VIBlock { + uint64_t m_size; + uint64_t m_offset; + + void read(char*& ptr, const char* ptr_end) { + m_offset = smUnpack(ptr, ptr_end); + m_size = smUnpack(ptr, ptr_end); + } +}; + +struct VIFooter { + VIBlock m_metaIndex; + VIBlock m_index; + + void read(char*& ptr, const char* ptr_end) { + m_index.read(ptr, ptr_end); + m_metaIndex.read(ptr, ptr_end); + } + + void read(std::ifstream& fs) { + fs.seekg(0, std::ios::end); + size_t size = fs.tellg(); + char footerData[48] = {}, *ptr = &footerData[0]; + fs.seekg(size - sizeof(footerData)); + fs.read(ptr, sizeof(footerData)); + + // https://github.com/tensorflow/tensorflow/blob/9659b7bdca80a8ef8240eb021d4da089034eeb00/tensorflow/tsl/lib/io/format.cc#L59 + ptr += sizeof(footerData) - 8; + uint32_t magic_lo = *reinterpret_cast(ptr); + uint32_t magic_hi = *reinterpret_cast(ptr + 4); + uint64_t magic_no = (static_cast(magic_hi) << 32) | static_cast(magic_lo); + + FRONT_END_GENERAL_CHECK(magic_no == 0xdb4775248b80fb57ull, "Wrong index file, magic number mismatch detected"); + + ptr = &footerData[0]; + m_metaIndex.read(ptr, ptr + sizeof(footerData)); + m_index.read(ptr, ptr + sizeof(footerData)); + } +}; + +void SavedModelVariablesIndex::read_variables_index_block(std::ifstream& fs, + const VIBlock& index, + std::vector& data, + uint32_t& offset, + uint32_t& offset_end) { + size_t block_size = index.m_size; + data.clear(); + data.resize(block_size + 5 /*kBlockTrailerSize*/); + fs.seekg(index.m_offset, std::ios::beg); + fs.read(data.data(), data.size()); +#ifndef ENABLE_SNAPPY_COMPRESSION + FRONT_END_GENERAL_CHECK(data[block_size] == 0, "Compressed files aren't supported"); +#else + FRONT_END_GENERAL_CHECK(data[block_size] == 0 || data[block_size] == 1, "Compression method isn't supported"); + if (data[block_size] == 1) { + size_t uncompressed_length = 0; + FRONT_END_GENERAL_CHECK(snappy::GetUncompressedLength(data.data(), data.size(), &uncompressed_length), + "Cannot retrieve uncompressed block length"); + std::string uncompressed_string; + uncompressed_string.reserve(uncompressed_length); + snappy::Uncompress(data.data(), data.size(), &uncompressed_string); + data.resize(uncompressed_length); + std::copy(uncompressed_string.begin(), uncompressed_string.end(), data.begin()); + block_size = uncompressed_length; + } +#endif + uint32_t numRestarts = smReadFixed(data.data() + block_size - sizeof(uint32_t)); + size_t maxRestarts = (block_size - sizeof(uint32_t)) / sizeof(uint32_t); + FRONT_END_GENERAL_CHECK(maxRestarts >= numRestarts, "Wrong restarts value"); + offset_end = static_cast(block_size) - ((numRestarts + 1) * sizeof(uint32_t)); + offset = smReadFixed(data.data() + offset_end); +} + +void SavedModelVariablesIndex::read_variables_index_pair(char*& ptr, + const char* ptr_end, + std::string& key, + char*& value, + uint32_t& val_length) { + uint32_t shared, nonShared; + shared = smUnpack(ptr, ptr_end); + nonShared = smUnpack(ptr, ptr_end); + val_length = smUnpack(ptr, ptr_end); + + // Key inherits last part of string (shared-size bytes) and appends new string + // shared_part_key1 //resize(0) + append(shared_part_key1) + // ............key2 //resize(12) + append(key2) + // ............key3 //resize(12) + append(key3) + // new_shared_key4 //resize(0) + append(new_shared_key4) + // ...........key5 //resize(11) + append(key5) + key.resize(shared); + key.append(ptr, nonShared); + + value = ptr + nonShared; + ptr = value + val_length; +} + +void SavedModelVariablesIndex::read_variables_index(std::ifstream& fs, + std::map>& varIndex) { + VIFooter footer; + + footer.read(fs); + + std::vector secondLevel; + std::vector blockData; + + uint32_t offset = 0, offset_end = 0; + + read_variables_index_block(fs, footer.m_index, blockData, offset, offset_end); + char *ptr = blockData.data() + offset, *ptr_end = blockData.data() + offset_end, *value = nullptr; + std::string key = ""; + uint32_t valLength; + + while (ptr < ptr_end) { + read_variables_index_pair(ptr, ptr_end, key, value, valLength); + + VIBlock valBlock; + valBlock.read(value, value + valLength); + secondLevel.push_back(valBlock); + ptr = value + valLength; + } + + for (auto& block : secondLevel) { + read_variables_index_block(fs, block, blockData, offset, offset_end); + + key = ""; + ptr = blockData.data() + offset; + ptr_end = blockData.data() + offset_end; + while (ptr < ptr_end) { + read_variables_index_pair(ptr, ptr_end, key, value, valLength); + varIndex[key] = std::vector(value, value + valLength); + } + } +} + +void SavedModelVariablesIndex::read_bundle_header() { + auto item = m_variables_index.find(""); + FRONT_END_GENERAL_CHECK(item != m_variables_index.end(), "Bundle Header isn't found in index"); + + ::tensorflow::BundleHeaderProto bundleHeader; + FRONT_END_GENERAL_CHECK(bundleHeader.ParseFromString(item->second.data()), + "Bundle Header: Cannot parse Bundle Header"); + FRONT_END_GENERAL_CHECK(bundleHeader.version().producer() == 1, "Bundle Header: Unsupported producer version"); + FRONT_END_GENERAL_CHECK(bundleHeader.version().min_consumer() == 0, "Bundle Header: Unsupported consumer version"); + FRONT_END_GENERAL_CHECK(bundleHeader.endianness() == 0, "Bundle Header: BIG endian isn't supported"); + + m_total_shards = bundleHeader.num_shards(); +} + +void SavedModelVariablesIndex::read_checkpointable_object_graph() { + m_variables_map.clear(); + + auto item = m_variables_index.find("_CHECKPOINTABLE_OBJECT_GRAPH"); + FRONT_END_GENERAL_CHECK(item != m_variables_index.end(), "Checkpointable Object Graph isn't found in index"); + + ::tensorflow::BundleEntryProto entry; + FRONT_END_GENERAL_CHECK(entry.ParseFromArray(item->second.data(), static_cast(item->second.size())), + "CMO: Cannot parse Bundle Entry"); + + FRONT_END_GENERAL_CHECK(entry.slices().empty(), "CMO: Slices are not supported"); + + auto shard = m_data_files.find(entry.shard_id()); + FRONT_END_GENERAL_CHECK(shard != m_data_files.end(), "CMO: data files isn't found"); + + std::vector data(entry.size()); + ::tensorflow::TrackableObjectGraph tog; + + // TODO: have to understand this offset + // It looks like reinterpret_cast artifact + // https://github.com/tensorflow/tensorflow/blob/d90f1947ebcf510b23c238f43c2191e5b3817cb3/tensorflow/cc/experimental/libexport/load.cc#L70 + int chg = 6; + shard->second->seekg(entry.offset() + chg); + shard->second->read(data.data(), entry.size() - chg); + + // Might be need to remove this verification: + // https://github.com/tensorflow/tensorflow/blob/d90f1947ebcf510b23c238f43c2191e5b3817cb3/tensorflow/cc/experimental/libexport/load.cc#L73 + // FRONT_END_GENERAL_CHECK(tog.ParseFromArray(data.data(), static_cast(data.size()) - chg), "CMO: Trackable + // Object Graph couldn't be read"); + + tog.ParseFromArray(data.data(), static_cast(data.size()) - chg); + + for (const auto& node : tog.nodes()) { + for (const auto& attr : node.attributes()) { + m_variables_map[attr.full_name()] = attr.checkpoint_key(); + } + } +} + +bool GraphIteratorSavedModel::is_valid_signature(const ::tensorflow::SignatureDef& signature) const { + const std::map<::tensorflow::DataType, ov::element::Type> types{ + {::tensorflow::DataType::DT_BOOL, ov::element::boolean}, + {::tensorflow::DataType::DT_INT16, ov::element::i16}, + {::tensorflow::DataType::DT_INT32, ov::element::i32}, + {::tensorflow::DataType::DT_INT64, ov::element::i64}, + {::tensorflow::DataType::DT_HALF, ov::element::f16}, + {::tensorflow::DataType::DT_FLOAT, ov::element::f32}, + {::tensorflow::DataType::DT_DOUBLE, ov::element::f64}, + {::tensorflow::DataType::DT_UINT8, ov::element::u8}, + {::tensorflow::DataType::DT_INT8, ov::element::i8}, + {::tensorflow::DataType::DT_BFLOAT16, ov::element::bf16}, + {::tensorflow::DataType::DT_STRING, ov::element::undefined}}; + + for (const auto& it : signature.inputs()) { + if (it.second.name().empty() || types.find(it.second.dtype()) == types.end()) + return false; + } + for (const auto& it : signature.outputs()) { + if (it.second.name().empty() || types.find(it.second.dtype()) == types.end()) + return false; + } + return true; +} + +bool SavedModelVariablesIndex::read_variables(std::ifstream& vi_stream, const std::string& path) { + m_variables_index.clear(); + read_variables_index(vi_stream, m_variables_index); + read_bundle_header(); + + std::vector suffix(20); + for (int32_t shard = 0; shard < m_total_shards; ++shard) { + std::snprintf(suffix.data(), suffix.size(), "data-%05d-of-%05d", shard, m_total_shards); + std::string fullPath = ov::util::path_join({path, "variables", std::string("variables.") + suffix.data()}); + m_data_files[shard] = + std::shared_ptr(new std::ifstream(fullPath, std::ifstream::in | std::ifstream::binary)); + FRONT_END_GENERAL_CHECK(m_data_files[shard]->is_open(), "Saved Model's variable index file does not exist"); + } + + read_checkpointable_object_graph(); + return true; +} + +#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) +bool SavedModelVariablesIndex::read_variables(std::ifstream& vi_stream, const std::wstring& path) { + m_variables_index.clear(); + read_variables_index(vi_stream, m_variables_index); + read_bundle_header(); + + std::vector suffix(20); + for (int32_t shard = 0; shard < m_total_shards; ++shard) { + swprintf_s(suffix.data(), suffix.size(), L"data-%05d-of-%05d", shard, m_total_shards); + std::wstring fullPath = + ov::util::path_join_w({path, L"variables", std::wstring(L"variables.") + suffix.data()}); + m_data_files[shard] = + std::shared_ptr(new std::ifstream(fullPath, std::ifstream::in | std::ifstream::binary)); + FRONT_END_GENERAL_CHECK(m_data_files[shard]->is_open(), "Saved Model's variable index file does not exist"); + } + + read_checkpointable_object_graph(); + return true; +} +#endif + +struct PtrNode { + const ::tensorflow::NodeDef* node; + std::vector inputs; + std::vector outputs; + + PtrNode() : node(nullptr), inputs(), outputs() {} + + PtrNode(const ::tensorflow::NodeDef& src_node, const std::map& node_dictionary) { + node = &src_node; + std::vector parsedName; + for (const auto& input_name : node->input()) { + parse_node_name(input_name, parsedName); + + auto input_node = node_dictionary.find(parsedName[0]); + if (input_node == node_dictionary.end()) { + continue; + } + + input_node->second->outputs.push_back(this); + inputs.push_back(input_node->second); + } + } + + void find_parent_by_op(const std::string& op, std::vector& result) const { + for (auto input : inputs) { + if (input->op() == op) { + result.push_back(input); + } + input->find_parent_by_op(op, result); + } + } + + static void parse_node_name(const std::string& name, std::vector& result) { + result.clear(); + size_t left_pos = name.find_first_of('^'), right_pos = name.find(':'); + if (left_pos != std::string::npos && left_pos < right_pos) { + ++left_pos; + } else { + left_pos = 0; + } + while (right_pos != std::string::npos && right_pos > left_pos) { + result.push_back(name.substr(left_pos, right_pos - left_pos)); + left_pos = right_pos + 1; + right_pos = name.find(':', left_pos); + } + result.push_back(name.substr(left_pos, name.length() - left_pos)); + } + + const std::string& op() const { + return node->op(); + } +}; + +static void read_stateful_partitioned_call(const std::shared_ptr<::tensorflow::GraphDef> graph_def, + const ::tensorflow::NodeDef& partCall, + std::map& node_dictionary) { + FRONT_END_GENERAL_CHECK(partCall.op() == "StatefulPartitionedCall", "Passed node isn't StatefulPartitionedCall"); + + std::string func_name = partCall.attr().at("f").func().name(); + + const ::tensorflow::FunctionDef* func_def = nullptr; + for (const auto& func : graph_def->library().function()) { + if (func.signature().name() == func_name) { + func_def = &func; + break; + } + } + + FRONT_END_GENERAL_CHECK(func_def, "Function isn't found in the library"); + FRONT_END_GENERAL_CHECK(graph_def->has_library(), "GraphDef contains functions, but doesn't have the library"); + + std::map nodes; + + // Filling temporary input nodes for exact function + for (int i = 0; i < func_def->signature().input_arg_size(); ++i) { + const auto& input_arg = func_def->signature().input_arg(i).name(); + const auto& parent_input = partCall.input(i); + auto input_node = node_dictionary.find(parent_input); + if (input_node != node_dictionary.end()) { + nodes[input_arg] = input_node->second; + } + } + + // Parsing nodes and inline partitioned calls + for (const auto& node : func_def->node_def()) { + nodes[node.name()] = new PtrNode(node, nodes); + + if (node.op() == "StatefulPartitionedCall") { + read_stateful_partitioned_call(graph_def, node, nodes); + } + } + + // Removing temporary input nodes + for (int i = 0; i < func_def->signature().input_arg_size(); ++i) { + const auto& input_arg = func_def->signature().input_arg(i).name(); + auto input_node = nodes.find(input_arg); + if (input_node != nodes.end()) { + nodes.erase(input_node); + } + } + + // Moving nodes to the global dictionary + for (const auto& node : nodes) { + std::string global_name = partCall.name() + "/" + node.first; + node_dictionary[global_name] = node.second; + } +} + +void GraphIteratorSavedModel::map_assignvariable(const std::shared_ptr<::tensorflow::GraphDef> graph_def, + std::map& variables_map) const { + std::map nodes; + + for (const auto& node : graph_def->node()) { + nodes[node.name()] = new PtrNode(node, nodes); + + if (node.op() == "StatefulPartitionedCall") { + read_stateful_partitioned_call(graph_def, node, nodes); + } + } + + for (const auto& node : nodes) { + if (node.second->op() != "AssignVariableOp") { + continue; + } + + // TODO: assets reading + + std::vector restorev2_nodes; + std::vector varhandle_nodes; + + node.second->find_parent_by_op("RestoreV2", restorev2_nodes); + node.second->find_parent_by_op("VarHandleOp", varhandle_nodes); + + FRONT_END_GENERAL_CHECK(restorev2_nodes.size() == 1, "Found unexpected amount of RestoreV2 nodes"); + FRONT_END_GENERAL_CHECK(varhandle_nodes.size() == 1, "Found unexpected amount of VarHandleOp nodes"); + + std::vector restore_output; + // Expected path is: RestoreV2 -(output_index)-(0)-> Identity -(0)-(1)-> AssignVariableOp + PtrNode::parse_node_name(node.second->inputs[1]->node->input(0), restore_output); + + int output_index = std::atoi(restore_output[restore_output.size() - 1].c_str()); + + // Expected path is: Const(tensor_names) -(0)-(1)-> RestoreV2 + const auto& variable_name = + restorev2_nodes[0]->inputs[1]->node->attr().at("value").tensor().string_val(output_index); + + variables_map[varhandle_nodes[0]->node->name()] = variable_name; + } + + nodes.clear(); +} + +bool GraphIteratorSavedModel::is_supported(const std::string& path) { + return ov::util::directory_exists(path) && ov::util::file_exists(ov::util::path_join({path, "saved_model.pb"})); +} + +#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) +bool GraphIteratorSavedModel::is_supported(const std::wstring& path) { + return ov::util::directory_exists(path) && ov::util::file_exists(ov::util::path_join_w({path, L"saved_model.pb"})); +} +#endif + +template <> +std::basic_string get_saved_model_name() { + return "/saved_model.pb"; +} +template <> +std::basic_string get_variables_index_name() { + return "/variables/variables.index"; +} + +#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) +template <> +std::basic_string get_saved_model_name() { + return L"/saved_model.pb"; +} +template <> +std::basic_string get_variables_index_name() { + return L"/variables/variables.index"; +} +#endif + +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow/src/translate_session.cpp b/src/frontends/tensorflow/src/translate_session.cpp index 4f1979851dd8d8..571a92f99676e3 100644 --- a/src/frontends/tensorflow/src/translate_session.cpp +++ b/src/frontends/tensorflow/src/translate_session.cpp @@ -5,7 +5,9 @@ #include "translate_session.hpp" #include "input_model.hpp" +#include "openvino/op/util/framework_node.hpp" #include "openvino/opsets/opset10.hpp" +#include "openvino/opsets/opset8.hpp" #include "tf_framework_node.hpp" #include "utils.hpp" @@ -36,6 +38,59 @@ std::vector reorder_ops_by_names(const std::vector& names, const } return resulted_ops; }; + +/// \brief Adds known input names from Saved Model file format +/// \param[in] node Node which should be updated +/// \param[in] saved_model_names Map of names from saved model +/// \returns True if node was updated, false otherwise +static bool apply_saved_model_names(std::shared_ptr node, + const std::shared_ptr>& saved_model_names) { + for (size_t i = 0; i < node->get_output_size(); ++i) { + const auto& node_names = node->get_output_tensor(i).get_names(); + for (const auto& name : node_names) { + const auto& saved_model_name = saved_model_names->find(name); + if (saved_model_name != saved_model_names->end()) { + node->set_friendly_name(saved_model_name->second); + return true; + } + } + } + return false; +} + +// it creates framework node and saves exception message in the node attribute +ov::OutputVector create_fw_node_with_exception(const std::shared_ptr& decoder, + const ov::OutputVector& inputs, + size_t num_outputs, + const std::string& operation_name, + const std::string& exception_message) { + ov::op::util::FrameworkNodeAttrs attrs; + attrs[FrameworkNode::failed_conversion_key] = exception_message; + auto fw_node = std::make_shared(decoder, inputs, num_outputs); + fw_node->set_attrs(attrs); + set_node_name(operation_name, fw_node); + return fw_node->outputs(); +} + +size_t get_flat_index_by_name_and_id(const ov::frontend::NamedOutputVector& outputs, + const std::string& name, + size_t idx) { + // Assume that if at least one output port has name, then all the ports should have names + if (!outputs.empty() && !outputs.front().name.empty()) { + // Producer has names in ports + auto it = std::find_if(outputs.begin(), outputs.end(), [&](const ov::frontend::NamedOutput& x) { + return name == x.name; + }); + FRONT_END_GENERAL_CHECK(outputs.end() - it > ptrdiff_t(idx), + "There is no output port specified by name and index"); + FRONT_END_GENERAL_CHECK(it[idx].name == name, + "There is no output port with specified index in a group with specified name"); + return it - outputs.begin() + idx; + } else { + // There are no named ports in the producer node, so reference by name wouldn't work + return idx; + } +} } // namespace TranslateSession::TranslateSession(const ov::frontend::InputModel::Ptr& input_model, @@ -66,7 +121,16 @@ void TranslateSession::inject_body_model(std::shared_ptr body_model, "inputs and arguments to the function " + operation_type + " do not match."); for (size_t param_ind = 0; param_ind < body_parameters.size(); ++param_ind) { + auto orig_type = body_parameters[param_ind]->get_element_type(); body_parameters[param_ind]->output(0).replace(ov_inputs[param_ind]); + if (auto ext_parameter = as_type_ptr(ov_inputs[param_ind].get_node_shared_ptr())) { + // save type of a Parameter as converted in the body + // this is important if the external conversion extension is applied to body graph node + // with setting its own type + if (orig_type != element::dynamic) { + ext_parameter->set_element_type(orig_type); + } + } } for (const auto& result_node : body_model->get_results()) { ov_outputs.push_back(result_node->input_value(0)); @@ -75,7 +139,6 @@ void TranslateSession::inject_body_model(std::shared_ptr body_model, void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& input_model, std::shared_ptr& ov_model) { - DecoderBase::OpTypeByName op_type_by_name; OpMap ng_op_map; ov::ParameterVector params; ov::ResultVector results; @@ -85,6 +148,8 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu const auto& model_inputs = model_tf->get_inputs(); const auto& model_outputs = model_tf->get_outputs(); const auto& model_frozen_inputs = model_tf->get_tensor_values(); + const auto& saved_model_inputs = model_tf->get_saved_model_input_names(); + const auto& saved_model_outputs = model_tf->get_saved_model_output_names(); // fill ng_op_map with Constant outputs for frozen inputs for (const auto& frozen_input : model_frozen_inputs) { @@ -114,15 +179,19 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu auto param = std::make_shared(input_type, input_shape); set_node_name(input_name, param); + if (saved_model_inputs.get() && saved_model_inputs->size() > 0) { + if (!apply_saved_model_names(param, saved_model_inputs)) { + param->get_output_tensor(0).add_names({"saved_model_unused"}); + } + } params.push_back(param); - ng_op_map[input_name] = {param}; + ng_op_map[input_name] = {NamedOutput(param)}; } // create the OV ops from TensorFlow ops for (const auto& operation_place : operation_places) { auto operation_decoder = operation_place->get_decoder(); auto operation_name = operation_place->get_names()[0]; - op_type_by_name[operation_name] = operation_decoder->get_op_type(); // output for parameter nodes has been already generated if (ng_op_map.count(operation_name)) { continue; @@ -144,7 +213,12 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu std::string producer_name; size_t producer_port_idx; try { - operation_decoder->get_input_node(input_port_idx, producer_name, producer_port_idx, op_type_by_name); + std::string producer_port_name; + operation_decoder->get_input_node(input_port_idx, producer_name, producer_port_name, producer_port_idx); + if (!producer_port_name.empty()) { + producer_port_idx = + get_flat_index_by_name_and_id(ng_op_map[producer_name], producer_port_name, producer_port_idx); + } } catch (const std::exception&) { FRONT_END_THROW("[ ERROR ] Exception happened when preparing input " + std::to_string(input_port_idx) + " for op '" + operation_decoder->get_op_name() + "', expected input name: '" + @@ -169,28 +243,28 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu const auto& input_outputs_vector = ng_op_map.at(std::to_string(input_port_idx) + ":" + operation_name); FRONT_END_GENERAL_CHECK(input_outputs_vector.size() == 1, "Input created with pruning must have one output"); - ov_inputs.push_back(input_outputs_vector.at(0)); + ov_inputs.push_back(input_outputs_vector.at(0).port); } else if (ng_op_map.count(producer_name + ":" + std::to_string(producer_port_idx))) { const auto& input_outputs_vector = ng_op_map.at(producer_name + ":" + std::to_string(producer_port_idx)); FRONT_END_GENERAL_CHECK(input_outputs_vector.size() == 1, "Input created with pruning must have one output"); - ov_inputs.push_back(input_outputs_vector.at(0)); + ov_inputs.push_back(input_outputs_vector.at(0).port); } else if (ng_op_map.count(producer_name)) { const auto& input_outputs_vector = ng_op_map.at(producer_name); if (input_outputs_vector.size() <= producer_port_idx) { - auto producer_node = input_outputs_vector[0].get_node_shared_ptr(); + auto producer_node = input_outputs_vector[0].port.get_node_shared_ptr(); if (std::dynamic_pointer_cast(producer_node)) { // FrameworkNode node does not know in advance how many output ports will be used // so we can increase number of outputs by demand producer_node->set_output_type(producer_port_idx, element::dynamic, PartialShape::dynamic()); // update output vector in node map - ng_op_map[producer_name] = producer_node->outputs(); + ng_op_map[producer_name] = named_from_indexed(producer_node->outputs()); } } FRONT_END_GENERAL_CHECK(input_outputs_vector.size() > producer_port_idx, "Input created with pruning must have one output"); - ov_inputs.push_back(input_outputs_vector.at(producer_port_idx)); + ov_inputs.push_back(input_outputs_vector.at(producer_port_idx).port); } else { FRONT_END_GENERAL_CHECK(false, "No input is found for node \"" + operation_name + "\" by port " + @@ -199,29 +273,39 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu } // generate OV node output vector for the current operation node - ov::OutputVector ov_outputs; + NamedOutputVector ov_outputs; auto operation_type = operation_decoder->get_op_type(); if (m_translator_map->count(operation_type)) { try { auto translator = m_translator_map->at(operation_decoder->get_op_type()); NodeContext node_context(operation_decoder, ov_inputs, this); ov_outputs = translator(node_context); - } catch (const std::exception&) { - // continue translation by replacing with FrameworkNode - // in case of any failures in translators due to their limitation - auto fw_node = std::make_shared(operation_decoder, - ov_inputs, - operation_place->get_output_ports().size()); - set_node_name(operation_name, fw_node); - ov_outputs = fw_node->outputs(); + } catch (const std::exception& ex) { + // save the root-cause of the translation failure + const auto fw_outs = create_fw_node_with_exception(operation_decoder, + ov_inputs, + operation_place->get_output_ports().size(), + operation_name, + ex.what()); + ov_outputs = named_from_indexed(fw_outs); + } catch (...) { + // save unknown exception type + const auto fw_outs = create_fw_node_with_exception(operation_decoder, + ov_inputs, + operation_place->get_output_ports().size(), + operation_name, + "Unknown exception type"); + ov_outputs = named_from_indexed(fw_outs); } } else if (auto body_ov_model = get_body_ov_model(operation_type)) { - inject_body_model(body_ov_model, operation_type, ov_inputs, ov_outputs); + OutputVector indexed_ov_outputs; + inject_body_model(body_ov_model, operation_type, ov_inputs, indexed_ov_outputs); // set output tensor names - for (size_t idx = 0; idx < ov_outputs.size(); ++idx) { - ov_outputs[idx].get_tensor().set_names({operation_name + ":" + std::to_string(idx)}); + for (size_t idx = 0; idx < indexed_ov_outputs.size(); ++idx) { + indexed_ov_outputs[idx].get_tensor().set_names({operation_name + ":" + std::to_string(idx)}); } + ov_outputs = named_from_indexed(indexed_ov_outputs); } else { // continue translation by replacing with FrameworkNode // for example, it helps auto-pruning to be triggered on later nodes @@ -229,16 +313,16 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu ov_inputs, operation_place->get_output_ports().size()); set_node_name(operation_name, fw_node); - ov_outputs = fw_node->outputs(); + ov_outputs = named_from_indexed(fw_node->outputs()); } // register OV node outputs in the map for new operation node for (const auto& output : ov_outputs) { - if (auto result = as_type_ptr(output.get_node_shared_ptr())) { + if (auto result = as_type_ptr(output.port.get_node_shared_ptr())) { // do not add RetVal type operation to ng_op_map results.push_back(result); } else { - auto param = as_type_ptr(output.get_node_shared_ptr()); + auto param = as_type_ptr(output.port.get_node_shared_ptr()); // avoid duplicating Parameter nodes if they are already in the Parameters vector if (param && std::find(params.begin(), params.end(), param) == params.end()) { params.push_back(param); @@ -262,15 +346,35 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu port_type); if (port_type == "none") { - for (const auto& node_output : ng_op_map[operation_name]) { + for (const auto& node_output : indexed_from_named(ng_op_map[operation_name])) { auto result_node = std::make_shared(node_output); - // to be aligned with Legacy Frontend we set a name along with output port index - // though, the Result name is not used in the OV API 2.0 but it is checked in MO args tests - result_node->set_friendly_name(model_output_name + ":0"); - results.push_back(result_node); + // Customize output name in case we have mapping from Saved Model format + if (saved_model_outputs.get() && saved_model_outputs->size() > 0) { + bool isUsed = true; + for (const auto& name : model_output_tensor_place->get_names()) { + auto saved_model_name = saved_model_outputs->find(name); + if (saved_model_name == saved_model_outputs->end()) { + saved_model_name = saved_model_outputs->find(name + ":0"); + } + if (saved_model_name != saved_model_outputs->end()) { + result_node->set_friendly_name(saved_model_name->second); + results.push_back(result_node); + isUsed = false; + break; + } + if (!isUsed) { + result_node->get_input_tensor(0).add_names({"saved_model_unused"}); + } + } + } else { + // to be aligned with Legacy Frontend we set a name along with output port index + // though, the Result name is not used in the OV API 2.0 but it is checked in MO args tests + result_node->set_friendly_name(model_output_name + ":0"); + results.push_back(result_node); + } } } else if (port_type == "out") { - const auto& node_outputs = ng_op_map[operation_name]; + const auto& node_outputs = indexed_from_named(ng_op_map[operation_name]); FRONT_END_GENERAL_CHECK(node_outputs.size() > port_index, "Output port with index " + std::to_string(port_index) + " of " + operation_name + "node specified as custom output does not exist"); @@ -291,9 +395,15 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu // get to know a producer node and by which its output port data is generated std::string producer_name; + std::string producer_port_name; size_t producer_port_idx; try { - operation_decoder->get_input_node(port_index, producer_name, producer_port_idx, op_type_by_name); + operation_decoder->get_input_node(port_index, producer_name, producer_port_name, producer_port_idx); + if (!producer_port_name.empty()) { + producer_port_idx = get_flat_index_by_name_and_id(ng_op_map[producer_name], + producer_port_name, + producer_port_idx); + } } catch (const std::exception&) { FRONT_END_THROW("[ ERROR ] Exception happened when preparing input " + std::to_string(port_index) + " for op '" + operation_decoder->get_op_name() + "', expected input name: '" + @@ -302,7 +412,7 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu } // add Result node for this producer output port - const auto& node_outputs = ng_op_map[producer_name]; + const auto& node_outputs = indexed_from_named(ng_op_map[producer_name]); FRONT_END_GENERAL_CHECK(node_outputs.size() > producer_port_idx, "Output port with index " + std::to_string(producer_port_idx) + " of " + producer_name + "node specified as custom output does not exist"); @@ -321,7 +431,7 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu if (results.empty()) { for (const auto& node_output_vector : ng_op_map) { for (size_t output_ind = 0; output_ind < node_output_vector.second.size(); ++output_ind) { - auto output = node_output_vector.second[output_ind]; + auto output = node_output_vector.second[output_ind].port; if (output.get_target_inputs().empty() && !std::dynamic_pointer_cast(output.get_node_shared_ptr())) { auto model_output_name = diff --git a/src/frontends/tensorflow/src/translate_session.hpp b/src/frontends/tensorflow/src/translate_session.hpp index 86f5d5cd973e35..9da2150eb31d65 100644 --- a/src/frontends/tensorflow/src/translate_session.hpp +++ b/src/frontends/tensorflow/src/translate_session.hpp @@ -31,6 +31,10 @@ class TranslateSession { std::shared_ptr get_body_ov_model(const std::string& body_graph_name); + ov::frontend::InputModel::Ptr get_input_model(void) const { + return m_input_model; + } + private: const ov::frontend::InputModel::Ptr m_input_model; const std::shared_ptr m_translator_map; diff --git a/src/frontends/tensorflow/tests/CMakeLists.txt b/src/frontends/tensorflow/tests/CMakeLists.txt index d9f477b1b00ab7..89552e8deae6f7 100644 --- a/src/frontends/tensorflow/tests/CMakeLists.txt +++ b/src/frontends/tensorflow/tests/CMakeLists.txt @@ -4,12 +4,17 @@ set(TARGET_NAME "ov_tensorflow_frontend_tests") +list(APPEND TF_TESTS_DEPENDENCIES tensorflow_test_models tensorflow_fe_standalone_build_test openvino_intel_cpu_plugin) +if (NOT ENABLE_INTEL_CPU) + list(REMOVE_ITEM TF_TESTS_DEPENDENCIES openvino_intel_cpu_plugin) + set(EXCLUDED_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/compilation.cpp) +endif() + ov_add_test_target( NAME ${TARGET_NAME} ROOT ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDENCIES - tensorflow_test_models - tensorflow_fe_standalone_build_test + EXCLUDED_SOURCE_PATHS ${EXCLUDED_TESTS} + DEPENDENCIES ${TF_TESTS_DEPENDENCIES} LINK_LIBRARIES gtest_main_manifest frontend_shared_test_classes diff --git a/src/frontends/tensorflow/tests/compilation.cpp b/src/frontends/tensorflow/tests/compilation.cpp new file mode 100644 index 00000000000000..6fc06b7661fc93 --- /dev/null +++ b/src/frontends/tensorflow/tests/compilation.cpp @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include "gtest/gtest.h" +#include "tf_utils.hpp" +#include "utils.hpp" + +namespace { +std::shared_ptr convert_model(const std::string& model_path) { + ov::frontend::FrontEndManager fem; + auto front_end = fem.load_by_framework(TF_FE); + if (!front_end) { + throw "TensorFlow Frontend is not initialized"; + } + auto model_filename = FrontEndTestUtils::make_model_path(std::string(TEST_TENSORFLOW_MODELS_DIRNAME) + model_path); + auto input_model = front_end->load(model_filename); + if (!input_model) { + throw "Input model is not read"; + } + auto model = front_end->convert(input_model); + if (!model) { + throw "Model is not converted"; + } + + return model; +} +} // namespace + +class CompileModelsTests : public ::testing::Test {}; + +TEST_F(CompileModelsTests, NgramCompilation) { + ov::Core core; + auto model = convert_model("model_ngram/model_ngram.pbtxt"); + ov::CompiledModel compiled_model = core.compile_model(model, "CPU"); + const auto runtime_model = compiled_model.get_runtime_model(); + + EXPECT_EQ(runtime_model->get_ordered_ops().size(), 4); + EXPECT_EQ(runtime_model->get_parameters().size(), 2); + EXPECT_EQ(runtime_model->get_results().size(), 1); +} \ No newline at end of file diff --git a/src/frontends/tensorflow/tests/conversion.cpp b/src/frontends/tensorflow/tests/conversion.cpp index ef719eb612bcc0..c7754e08275a9b 100644 --- a/src/frontends/tensorflow/tests/conversion.cpp +++ b/src/frontends/tensorflow/tests/conversion.cpp @@ -19,11 +19,13 @@ class TensorflowFrontendWrapper : public ov::frontend::tensorflow::FrontEnd { ov::frontend::tensorflow::FrontEnd::add_extension(extension); if (auto conv_ext = std::dynamic_pointer_cast(extension)) { - EXPECT_NE(std::find(m_conversion_extensions.begin(), m_conversion_extensions.end(), conv_ext), - m_conversion_extensions.end()) - << "ConversionExtension is not registered."; - EXPECT_NE(m_op_translators.find(conv_ext->get_op_type()), m_op_translators.end()) - << conv_ext->get_op_type() << " translator is not registered."; + if (conv_ext->get_converter() || conv_ext->get_converter_named_and_indexed()) { + EXPECT_NE(std::find(m_conversion_extensions.begin(), m_conversion_extensions.end(), conv_ext), + m_conversion_extensions.end()) + << "ConversionExtension is not registered."; + EXPECT_NE(m_op_translators.find(conv_ext->get_op_type()), m_op_translators.end()) + << conv_ext->get_op_type() << " translator is not registered."; + } } else if (auto telemetry = std::dynamic_pointer_cast(extension)) { EXPECT_EQ(m_telemetry, telemetry) << "TelemetryExtension is not registered."; } else if (auto transformation = std::dynamic_pointer_cast(extension)) { diff --git a/src/frontends/tensorflow/tests/convert_model.cpp b/src/frontends/tensorflow/tests/convert_model.cpp index ddf0b4fe4ef951..71c028a6ae73cf 100644 --- a/src/frontends/tensorflow/tests/convert_model.cpp +++ b/src/frontends/tensorflow/tests/convert_model.cpp @@ -13,9 +13,9 @@ using TFConvertModelTest = FrontEndConvertModelTest; static const std::vector models{ std::string("2in_2out/2in_2out.pb"), - std::string("forward_edge_model/forward_edge_model.pb"), - std::string("forward_edge_model2/forward_edge_model2.pb"), - std::string("concat_with_non_constant_axis/concat_with_non_constant_axis.pb")}; + std::string("forward_edge_model/forward_edge_model.pbtxt"), + std::string("forward_edge_model2/forward_edge_model2.pbtxt"), + std::string("concat_with_non_constant_axis/concat_with_non_constant_axis.pbtxt")}; INSTANTIATE_TEST_SUITE_P(TFConvertModelTest, FrontEndConvertModelTest, diff --git a/src/frontends/tensorflow/tests/convert_tricky_models.cpp b/src/frontends/tensorflow/tests/convert_tricky_models.cpp index 6dd2a5a510b325..091508fd1c68b9 100644 --- a/src/frontends/tensorflow/tests/convert_tricky_models.cpp +++ b/src/frontends/tensorflow/tests/convert_tricky_models.cpp @@ -3,6 +3,7 @@ // #include +#include #include #include #include @@ -20,12 +21,15 @@ using namespace ov::opset10; using namespace ov::frontend; namespace { -shared_ptr convert_model(const string& model_path) { +shared_ptr convert_model(const string& model_path, const ConversionExtension::Ptr& conv_ext = nullptr) { FrontEndManager fem; auto front_end = fem.load_by_framework(TF_FE); if (!front_end) { throw "TensorFlow Frontend is not initialized"; } + if (conv_ext) { + front_end->add_extension(conv_ext); + } auto model_filename = FrontEndTestUtils::make_model_path(string(TEST_TENSORFLOW_MODELS_DIRNAME) + model_path); auto input_model = front_end->load(model_filename); if (!input_model) { @@ -38,12 +42,42 @@ shared_ptr convert_model(const string& model_path) { return model; } + +NamedOutputVector fake_translator_ragged_tensor_to_sparse(const NodeContext& node) { + // NOTE: pay attention that this is a fake translator for RaggedTensorToSparse + // only serves for testing purposes + FRONT_END_GENERAL_CHECK(node.get_input_size() > 1, "RaggedTensorToSparse expects at least two inputs."); + auto node_name = node.get_name(); + auto row_splits = node.get_input(0); + auto strings = node.get_input(1); + + // Override type of input tensor if this is a Parameter + if (auto parameter = as_type_ptr(strings.get_node_shared_ptr())) { + parameter->set_partial_shape(ov::PartialShape{Dimension()}); + parameter->set_element_type(ov::element::u8); + parameter->validate_and_infer_types(); + } + + row_splits = make_shared(row_splits, strings); + auto const_one = make_shared(row_splits.get_element_type(), Shape{}, 1); + Output mul = make_shared(row_splits, const_one); + auto const_two = make_shared(ov::element::u8, Shape{}, 2); + Output add = make_shared(strings, const_two); + auto const_three = make_shared(ov::element::u8, Shape{}, 3); + Output sub = make_shared(strings, const_three); + + mul.get_tensor().add_names({node_name + ":0"}); + add.get_tensor().add_names({node_name + ":1"}); + sub.get_tensor().add_names({node_name + ":2"}); + + return {{"sparse_indices", mul}, {"sparse_values", add}, {"sparse_dense_shape", sub}}; +} } // namespace TEST(FrontEndConvertTrickyModels, undefined_input_shape) { shared_ptr model; try { - model = convert_model("undefined_input_shape/undefined_input_shape.pb"); + model = convert_model("undefined_input_shape/undefined_input_shape.pbtxt"); } catch (std::exception& ex) { ASSERT_TRUE(false) << ex.what(); } @@ -62,7 +96,7 @@ TEST(FrontEndConvertTrickyModels, undefined_input_shape) { TEST(FrontEndConvertTrickyModels, simple_wide_and_deep) { shared_ptr model; try { - model = convert_model("simple_wide_and_deep/simple_wide_and_deep.pb"); + model = convert_model("simple_wide_and_deep/simple_wide_and_deep.pbtxt"); } catch (std::exception& ex) { ASSERT_TRUE(false) << ex.what(); } @@ -80,7 +114,7 @@ TEST(FrontEndConvertTrickyModels, simple_wide_and_deep) { TEST(FrontEndConvertTrickyModels, model_with_output_shapes) { shared_ptr model; try { - model = convert_model("model_with_output_shapes_attr/model_with_output_shapes_attr.pb"); + model = convert_model("model_with_output_shapes_attr/model_with_output_shapes_attr.pbtxt"); } catch (std::exception& ex) { ASSERT_TRUE(false) << ex.what(); } @@ -96,7 +130,7 @@ TEST(FrontEndConvertTrickyModels, model_with_output_shapes) { TEST_F(TransformationTestsF, AssertAndStringTensors) { { - model = convert_model("string_tensors_model/string_tensors_model.pb"); + model = convert_model("string_tensors_model/string_tensors_model.pbtxt"); // TODO: investigate - why we have redundant nodes after the conversion manager.register_pass(false); } @@ -111,13 +145,13 @@ TEST_F(TransformationTestsF, AssertAndStringTensors) { } TEST_F(TransformationTestsF, UnsortedNodes) { - { model = convert_model("forward_edge_model_unsorted/forward_edge_model_unsorted.pb"); } - { model_ref = convert_model("forward_edge_model/forward_edge_model.pb"); } + { model = convert_model("forward_edge_model_unsorted/forward_edge_model_unsorted.pbtxt"); } + { model_ref = convert_model("forward_edge_model/forward_edge_model.pbtxt"); } } TEST_F(TransformationTestsF, ModelWithSwishF32BodyGraph) { { - model = convert_model("swish_f32/swish_f32.pb"); + model = convert_model("swish_f32/swish_f32.pbtxt"); // need to call shape inference since body graphs can be injected with undefined shapes model->validate_nodes_and_infer_types(); } @@ -135,7 +169,7 @@ TEST_F(TransformationTestsF, ModelWithSwishF32BodyGraph) { TEST_F(TransformationTestsF, PartitionedCall) { { - model = convert_model("partitioned_call/partitioned_call.pb"); + model = convert_model("partitioned_call/partitioned_call.pbtxt"); // need to call shape inference since body graphs can be injected with undefined shapes model->validate_nodes_and_infer_types(); } @@ -151,7 +185,7 @@ TEST_F(TransformationTestsF, PartitionedCall) { } TEST_F(TransformationTestsF, ModelWithIf) { - { model = convert_model("model_with_if/model_with_if.pb"); } + { model = convert_model("model_with_if/model_with_if.pbtxt"); } { // create then branch body graph auto then_x = make_shared(i32, Shape{2}); @@ -185,7 +219,7 @@ TEST_F(TransformationTestsF, ModelWithIf) { TEST_F(TransformationTestsF, InjectedBodyAndIf) { { - model = convert_model("injected_body_and_if/injected_body_and_if.pb"); + model = convert_model("injected_body_and_if/injected_body_and_if.pbtxt"); // need to call shape inference since body graphs can be injected with undefined shapes model->validate_nodes_and_infer_types(); } @@ -224,7 +258,7 @@ TEST_F(TransformationTestsF, InjectedBodyAndIf) { TEST_F(TransformationTestsF, ModelWithDilatedGroupConvolution) { { - model = convert_model("dilated_gconv_model/dilated_gconv_model.pb"); + model = convert_model("dilated_gconv_model/dilated_gconv_model.pbtxt"); // need to call MOC to fuse BatchToSpace/SpaceToBatch with GroupConvolution manager.register_pass(false); } @@ -248,7 +282,7 @@ TEST_F(TransformationTestsF, ModelWithDilatedGroupConvolution) { TEST_F(TransformationTestsF, ModelWithSaveV2) { { - model = convert_model("model_savev2/model_savev2.pb"); + model = convert_model("model_savev2/model_savev2.pbtxt"); // need to call shape inference since body graphs can be injected with undefined shapes model->validate_nodes_and_infer_types(); } @@ -263,7 +297,7 @@ TEST_F(TransformationTestsF, ModelWithSaveV2) { } TEST_F(TransformationTestsF, ModelWithConstResultSubgraphs) { - { model = convert_model("model_with_const_result/model_with_const_result.pb"); } + { model = convert_model("model_with_const_result/model_with_const_result.pbtxt"); } { // create a reference graph auto x = make_shared(element::f32, PartialShape{Dimension::dynamic(), 60, 60, 1}); @@ -286,7 +320,7 @@ TEST_F(TransformationTestsF, ModelWithConstResultSubgraphs) { } TEST_F(TransformationTestsF, ModelWithIteratorGetNext) { - { model = convert_model("model_with_iterator_get_next/model_with_iterator_get_next.pb"); } + { model = convert_model("model_with_iterator_get_next/model_with_iterator_get_next.pbtxt"); } { // create a reference graph auto x = make_shared(element::f32, Shape{2, 3}); @@ -298,7 +332,7 @@ TEST_F(TransformationTestsF, ModelWithIteratorGetNext) { } TEST_F(TransformationTestsF, ModelWithQueueOperations) { - { model = convert_model("model_with_queue_ops/model_with_queue_ops.pb"); } + { model = convert_model("model_with_queue_ops/model_with_queue_ops.pbtxt"); } { // create a reference graph auto x = make_shared(element::f32, PartialShape{Dimension::dynamic(), 160, 160, 3}); @@ -310,7 +344,7 @@ TEST_F(TransformationTestsF, ModelWithQueueOperations) { } TEST_F(TransformationTestsF, ModelWithQueueOperations2) { - { model = convert_model("model_with_queue_ops2/model_with_queue_ops2.pb"); } + { model = convert_model("model_with_queue_ops2/model_with_queue_ops2.pbtxt"); } { // create a reference graph auto x = make_shared(element::f32, PartialShape{1, Dimension::dynamic(), Dimension::dynamic(), 3}); @@ -324,7 +358,7 @@ TEST_F(TransformationTestsF, ModelWithQueueOperations2) { } TEST_F(TransformationTestsF, ModelWithLookupTableOperations) { - { model = convert_model("model_with_lookup_table/model_with_lookup_table.pb"); } + { model = convert_model("model_with_lookup_table/model_with_lookup_table.pbtxt"); } { // create a reference graph auto x = make_shared(element::f32, Shape{2}); @@ -346,8 +380,9 @@ TEST_F(TransformationTestsF, ModelWithIteratorGetNextAndUnsupportedOp) { model_ref = make_shared(OutputVector{add}, ParameterVector{x, y}); } } + TEST_F(TransformationTestsF, ModelWithMultioutputBodyGraphNode) { - { model = convert_model("partitioned_call2/partitioned_call2.pb"); } + { model = convert_model("partitioned_call2/partitioned_call2.pbtxt"); } { auto x = make_shared(i32, Shape{5}); auto y = make_shared(i32, Shape{5}); @@ -371,8 +406,98 @@ TEST_F(TransformationTestsF, ModelWithEmptyTensorListAndPushBack) { auto x_unsqueeze_flatten = make_shared(x_flatten, zero_const); auto empty_const = make_shared(f32, Shape{0, 30}, vector{}); auto list_push_back = make_shared(OutputVector{empty_const, x_unsqueeze_flatten}, 0); - auto recover_item_shape = make_shared(i32, Shape{4}, vector{1, 2, 3, 5}); + auto list_push_back_shape = make_shared(list_push_back, element::i32); + auto start = make_shared(i32, Shape{1}, 0); + auto stop = make_shared(i32, Shape{1}, 1); + auto step = make_shared(i32, Shape{1}, 1); + auto batch = make_shared(list_push_back_shape, start, stop, step); + auto shape_without_batch = make_shared(i32, Shape{3}, vector{2, 3, 5}); + auto recover_item_shape = make_shared(OutputVector{batch, shape_without_batch}, 0); auto recover_item = make_shared(list_push_back, recover_item_shape, false); model_ref = make_shared(OutputVector{recover_item}, ParameterVector{x}); } } + +TEST_F(TransformationTestsF, ModelWithAssertNode) { + { model = convert_model("model_with_assert/model_with_assert.pb"); } + { + auto x = make_shared(i32, PartialShape{Dimension::dynamic()}); + auto y = make_shared(i32, PartialShape{Dimension::dynamic()}); + auto add = make_shared(x, y); + model_ref = make_shared(OutputVector{add}, ParameterVector{x, y}); + } +} + +TEST_F(TransformationTestsF, PartitionedCallWithUnique) { + // This test aims to test named output ports for Unique operation + { model = convert_model("partitioned_call_with_unique/partitioned_call_with_unique.pb"); } + { + auto x = make_shared(f32, Shape{5}); + auto relu = make_shared(x); + auto unique = make_shared(relu, false, i32); + auto const_one = make_shared(i32, Shape{}, 1); + auto add = make_shared(unique->output(2), const_one); + auto sigmoid = make_shared(unique->output(0)); + model_ref = make_shared(OutputVector{sigmoid, add}, ParameterVector{x}); + } +} + +TEST_F(TransformationTestsF, RaggedTensorToSparse) { + // This test aims to test named output ports for RaggedTensorToSparse operation + // also, it tests propagation of custom type (specified in the extension) to Parameter node in the parent graph + { + // create FAKE conversion extension for RaggedTensorToSparse + auto conv_ext = std::make_shared("RaggedTensorToSparse", + fake_translator_ragged_tensor_to_sparse); + model = convert_model("ragged_tensor_to_sparse/ragged_tensor_to_sparse.pb", conv_ext); + } + { + auto strings = make_shared(u8, PartialShape{3}); + auto row_splits = make_shared(i32, PartialShape{5}); + auto convert_like = make_shared(row_splits, strings); + + auto const_one = make_shared(u8, Shape{}, 1); + Output mul = make_shared(convert_like, const_one); + auto const_three = make_shared(u8, Shape{}, 3); + Output sub = make_shared(strings, const_three); + + auto target_shape1 = make_shared(i32, Shape{1}, -1); + auto reshape1 = make_shared(mul, target_shape1, false); + auto target_shape2 = make_shared(i32, Shape{1}, -1); + auto reshape2 = make_shared(sub, target_shape2, false); + + auto concat = make_shared(OutputVector{reshape1, reshape2}, 0); + + model_ref = make_shared(OutputVector{concat}, ParameterVector{row_splits, strings}); + } +} + +TEST_F(TransformationTestsF, SplitInFunction) { + { + // create FAKE conversion extension for Split using named ports, this is not required for Split, but it tests + // how named ports will work if there is one name and many outputs associated with it + auto conv_ext = std::make_shared("Split", [](const NodeContext& node) { + auto axis = node.get_input(0); + auto value = node.get_input(1); + auto num_split = node.get_attribute("num_split"); + + auto split = make_shared(value, axis, num_split); + NamedOutputVector res; + for (const auto& output : split->outputs()) { + res.push_back({"output", output}); + } + return res; + }); + model = convert_model("split_in_function/split_in_function.pbtxt", conv_ext); + } + { + auto x = make_shared(f32, PartialShape{3, 20}); + + auto const_zero = make_shared(i32, Shape{}, 0); + auto split = make_shared(x, const_zero, 3); + auto add1 = make_shared(split->output(0), split->output(1)); + auto add2 = make_shared(add1, split->output(2)); + + model_ref = make_shared(OutputVector{add2}, ParameterVector{x}); + } +} diff --git a/src/frontends/tensorflow/tests/convert_unsupported.cpp b/src/frontends/tensorflow/tests/convert_unsupported.cpp index 20edc81da9a6fa..3cef3b7a03323f 100644 --- a/src/frontends/tensorflow/tests/convert_unsupported.cpp +++ b/src/frontends/tensorflow/tests/convert_unsupported.cpp @@ -4,7 +4,10 @@ #include #include +#include #include +#include +#include #include #include @@ -34,17 +37,11 @@ class TestDecoder : public ov::frontend::DecoderBase { void get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const override { throw "Not implemented"; } - void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const override { - throw "Not implemented"; - } - const std::string& get_op_type() const override { return m_op_type; } @@ -75,6 +72,39 @@ shared_ptr convert_model_partially(const string& model_path) { return model; } + +shared_ptr convert_model(const string& model_path, const ConversionExtension::Ptr& conv_ext = nullptr) { + FrontEndManager fem; + auto front_end = fem.load_by_framework(TF_FE); + if (!front_end) { + throw "TensorFlow Frontend is not initialized"; + } + if (conv_ext) { + front_end->add_extension(conv_ext); + } + auto model_filename = FrontEndTestUtils::make_model_path(string(TEST_TENSORFLOW_MODELS_DIRNAME) + model_path); + auto input_model = front_end->load(model_filename); + if (!input_model) { + throw "Input model is not read"; + } + + auto model = front_end->convert(input_model); + return model; +} + +ov::OutputVector incorrect_less_translator(const ov::frontend::NodeContext& node) { + // NOTE: pay attention that this is a fake translator for Less operation + // only serves for testing purposes + TENSORFLOW_OP_VALIDATION(node, false, "Less expects ten inputs."); + return {}; +} + +ov::OutputVector add_translator_with_unknown_exception(const ov::frontend::NodeContext& node) { + // NOTE: pay attention that this is a fake translator for Add operation + // only serves for testing purposes + throw 0; + return {}; +} } // namespace TEST(FrontEndConvertModelTest, test_unsupported_op) { @@ -110,7 +140,7 @@ TEST(FrontEndConvertModelTest, test_unsupported_tf1_while) { ASSERT_NO_THROW(frontEnd = fem.load_by_framework(TF_FE)); ASSERT_NE(frontEnd, nullptr); auto model_filename = FrontEndTestUtils::make_model_path(string(TEST_TENSORFLOW_MODELS_DIRNAME) + - string("model_tf1_while/model_tf1_while.pb")); + string("model_tf1_while/model_tf1_while.pbtxt")); ASSERT_NO_THROW(inputModel = frontEnd->load(model_filename)); ASSERT_NE(inputModel, nullptr); shared_ptr model; @@ -145,3 +175,43 @@ TEST_F(TransformationTestsF, ModelWithDynamicType) { model_ref = make_shared(OutputVector{log1p_node}, ParameterVector{x}); } } + +TEST(FrontEndConvertModelTest, test_unsupported_tf1_while_and_incorrect_less_translator) { + shared_ptr model = nullptr; + try { + auto conv_ext = std::make_shared("Less", incorrect_less_translator); + model = convert_model("model_tf1_while/model_tf1_while.pbtxt", conv_ext); + FAIL() << "TensorFlow 1 While is not supported and the fake translator registered in TF FE but conversion " + "passed without errors. " + "OpConversionFailure is expected."; + } catch (const OpConversionFailure& error) { + string error_message = error.what(); + string ref_message = "Less expects ten inputs.\n" + "\n" + "[TensorFlow Frontend] Internal error: No translator found for Enter node."; + ASSERT_TRUE(error_message.find(ref_message) != string::npos); + ASSERT_EQ(model, nullptr); + } catch (...) { + FAIL() << "Conversion of TensorFlow 1 While failed by wrong reason."; + } +} + +TEST(FrontEndConvertModelTest, conversion_with_unknown_exception) { + shared_ptr model = nullptr; + try { + auto conv_ext = + std::make_shared("Add", add_translator_with_unknown_exception); + model = convert_model("model_tf1_while/model_tf1_while.pbtxt", conv_ext); + FAIL() << "TensorFlow 1 While is not supported and the fake translator registered in TF FE but conversion " + "passed without errors. " + "OpConversionFailure is expected."; + } catch (const OpConversionFailure& error) { + string error_message = error.what(); + string ref_message = "Unknown exception type\n" + "[TensorFlow Frontend] Internal error: No translator found for Enter node."; + ASSERT_TRUE(error_message.find(ref_message) != string::npos); + ASSERT_EQ(model, nullptr); + } catch (...) { + FAIL() << "Conversion of TensorFlow 1 While failed by wrong reason."; + } +} diff --git a/src/frontends/tensorflow/tests/requirements.txt b/src/frontends/tensorflow/tests/requirements.txt index 6ad62764d61fcf..2f8b23acc09f1f 100644 --- a/src/frontends/tensorflow/tests/requirements.txt +++ b/src/frontends/tensorflow/tests/requirements.txt @@ -1,2 +1,3 @@ +-c ../../../bindings/python/constraints.txt numpy>=1.16.6,<=1.23.4 -tensorflow>=1.15.5,<=2.11.0 +tensorflow diff --git a/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_model_with_assert.py b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_model_with_assert.py new file mode 100644 index 00000000000000..79c6d84bf2a2c5 --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_model_with_assert.py @@ -0,0 +1,38 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# +# model with Assert node generator +# + +import os +import sys + +import numpy as np +import tensorflow as tf + + +def main(): + tf.compat.v1.reset_default_graph() + + # Create the graph and model + with tf.compat.v1.Session() as sess: + x = tf.compat.v1.placeholder(dtype=tf.int32, shape=[None], name='x') + y = tf.compat.v1.placeholder(dtype=tf.int32, shape=[None], name='y') + tf.raw_ops.AddV2(x=x, y=y) + shape1 = tf.raw_ops.Shape(input=x) + shape2 = tf.raw_ops.Shape(input=y) + equal = tf.raw_ops.Equal(x=shape1, y=shape2) + axis = tf.constant([0], dtype=tf.int32) + all_equal = tf.raw_ops.All(input=equal, axis=axis) + message = tf.constant("Shapes of operands are incompatible", dtype=tf.string) + tf.raw_ops.Assert(condition=all_equal, data=[message]) + + tf.compat.v1.global_variables_initializer() + tf_net = sess.graph_def + + tf.io.write_graph(tf_net, os.path.join(sys.argv[1], "model_with_assert"), "model_with_assert.pb", False) + + +if __name__ == "__main__": + main() diff --git a/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_partitioned_call_with_unique.py b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_partitioned_call_with_unique.py new file mode 100644 index 00000000000000..7ed60ab836d128 --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_partitioned_call_with_unique.py @@ -0,0 +1,33 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +import tensorflow.compat.v1 as tf + + +def main(): + tf.compat.v1.reset_default_graph() + + @tf.function + def second_func(x): + relu = tf.raw_ops.Relu(features=x) + y, idx = tf.raw_ops.Unique(x=relu) + return y, idx + + @tf.function + def first_func(x): + y, idx = second_func(x) + sigmoid = tf.raw_ops.Sigmoid(x=y) + const_one = tf.constant(1, dtype=tf.int32) + add = tf.raw_ops.AddV2(x=idx, y=const_one) + return sigmoid, add + + tf_net = first_func.get_concrete_function(tf.constant([1, 2, 3, 4, 5], dtype=tf.float32)).graph.as_graph_def() + tf.io.write_graph(tf_net, os.path.join(sys.argv[1], "partitioned_call_with_unique"), + "partitioned_call_with_unique.pb", False) + + +if __name__ == "__main__": + main() diff --git a/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_ragged_tensor_to_sparse.py b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_ragged_tensor_to_sparse.py new file mode 100644 index 00000000000000..0c1a218e684e68 --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_ragged_tensor_to_sparse.py @@ -0,0 +1,35 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +import tensorflow.compat.v1 as tf + + +def main(): + tf.compat.v1.reset_default_graph() + + @tf.function + def second_func(strings, row_splits): + sparse_indices, sparse_values, sparse_dense_shape = tf.raw_ops.RaggedTensorToSparse( + rt_nested_splits=[row_splits], + rt_dense_values=strings) + return sparse_indices, sparse_values, sparse_dense_shape + + @tf.function + def first_func(strings, row_splits): + sparse_indices, _, sparse_dense_shape = second_func(strings, row_splits) + sparse_indices = tf.raw_ops.Reshape(tensor=sparse_indices, shape=[-1]) + sparse_dense_shape = tf.raw_ops.Reshape(tensor=sparse_dense_shape, shape=[-1]) + concat = tf.raw_ops.Concat(concat_dim=0, values=[sparse_indices, sparse_dense_shape]) + return concat + + tf_net = first_func.get_concrete_function(tf.constant(["abc", "bcd", "cc"], dtype=tf.string), + tf.constant([0, 2, 2, 3, 3], dtype=tf.int32)).graph.as_graph_def() + tf.io.write_graph(tf_net, os.path.join(sys.argv[1], "ragged_tensor_to_sparse"), + "ragged_tensor_to_sparse.pb", False) + + +if __name__ == "__main__": + main() diff --git a/src/frontends/tensorflow/tests/test_models/gen_scripts/split_in_function.py b/src/frontends/tensorflow/tests/test_models/gen_scripts/split_in_function.py new file mode 100644 index 00000000000000..1af51a2aaea217 --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/gen_scripts/split_in_function.py @@ -0,0 +1,30 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +import tensorflow.compat.v1 as tf +import numpy as np + + +def main(): + tf.compat.v1.reset_default_graph() + + @tf.function + def second_func(x): + x1, x2, x3 = tf.split(x, 3) + return x1 + x2 + x3 + + @tf.function + def first_func(x): + return second_func(x) + + tf_net = first_func.get_concrete_function(tf.constant( + np.random.randn(3, 20), dtype=tf.float32)).graph.as_graph_def() + tf.io.write_graph(tf_net, os.path.join(sys.argv[1], "split_in_function"), + "split_in_function.pb", False) + + +if __name__ == "__main__": + main() diff --git a/src/frontends/tensorflow/tests/test_models/gen_wrapper.py b/src/frontends/tensorflow/tests/test_models/gen_wrapper.py index 0e4b726e9b2a61..285e5851fb525d 100644 --- a/src/frontends/tensorflow/tests/test_models/gen_wrapper.py +++ b/src/frontends/tensorflow/tests/test_models/gen_wrapper.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import shutil import subprocess import sys @@ -19,16 +20,14 @@ if gen_script.endswith('.py'): subprocess.run([sys.executable, gen_script, out_folder], env=os.environ) elif gen_script.endswith('.pbtxt'): - import tensorflow.compat.v1 as tf - from google.protobuf import text_format - model_pbtxt = gen_script - with open(model_pbtxt, "r") as f: - model_name = os.path.basename(model_pbtxt).split('.')[0] - graph_def = tf.GraphDef() - text_format.Merge(f.read(), graph_def) - tf.import_graph_def(graph_def, name='') - tf.io.write_graph(graph_def, os.path.join(sys.argv[2], model_name), model_name + '.pb', False) + model_name = os.path.basename(model_pbtxt).split('.')[0] + dest_path = os.path.join(out_folder, model_name, model_name + '.pbtxt') + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + try: + shutil.copy(model_pbtxt, dest_path) + except shutil.SameFileError: + pass # Create mark file indicating that script was executed with open(mark_file, "w") as fp: diff --git a/src/frontends/tensorflow/tests/test_models/models_pbtxt/model_ngram.pbtxt b/src/frontends/tensorflow/tests/test_models/models_pbtxt/model_ngram.pbtxt new file mode 100644 index 00000000000000..f13ca814b6774f --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/models_pbtxt/model_ngram.pbtxt @@ -0,0 +1,1821 @@ +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/LayerNorm/batchnorm/add_1" + op: "Placeholder" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + dim { + size: -1 + } + dim { + size: 256 + } + } + } + } +} +node { + name: "StatefulPartitionedCall/text_preprocessor/RaggedToSparse/RaggedTensorToSparse" + op: "Placeholder" + attr { + key: "dtype" + value { + type: DT_INT64 + } + } + attr { + key: "shape" + value { + shape { + dim { + size: -1 + } + dim { + size: 2 + } + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill/dims" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\002\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Cast/x" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: -1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Cast" + op: "Cast" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Cast/x" + attr { + key: "DstT" + value { + type: DT_INT64 + } + } + attr { + key: "SrcT" + value { + type: DT_INT32 + } + } + attr { + key: "Truncate" + value { + b: false + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill" + op: "Fill" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill/dims" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Cast" + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "index_type" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill_1/dims" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\002\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill_1" + op: "Fill" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill_1/dims" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Cast" + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "index_type" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat/axis" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat" + op: "ConcatV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill" + input: "StatefulPartitionedCall/text_preprocessor/RaggedToSparse/RaggedTensorToSparse" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Fill_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat/axis" + attr { + key: "N" + value { + i: 3 + } + } + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\000\000\000\000\000\000\000\000" + } + } + } +} + +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Shape" + op: "Shape" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/LayerNorm/batchnorm/add_1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "out_type" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice/stack_1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Shape" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 1 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add/y" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack_1/1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack_1/1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\001\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 2 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\000\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_1/x" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_1" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_1/x" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack_1/1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack_1/1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\001\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 2 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/Equal" + op: "Equal" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_1" + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "incompatible_shape_error" + value { + b: true + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 256 + } + } + float_val: 0.0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 256 + } + } + float_val: 0.0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_1/axis" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_1" + op: "ConcatV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/LayerNorm/batchnorm/add_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_1/axis" + attr { + key: "N" + value { + i: 3 + } + } + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_2/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_2" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_2/y" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/add_2" + attr { + key: "N" + value { + i: 1 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2/packed/1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 256 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2/packed" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2/packed/1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2/Const" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + } + float_val: 0.0 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2" + op: "Fill" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2/packed" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2/Const" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "index_type" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/Select" + op: "Select" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/Equal" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/strided_slice_2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2" + attr { + key: "T" + value { + type: DT_FLOAT + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/add/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/add" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/add/y" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/add" + attr { + key: "N" + value { + i: 1 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\002\000\000\000\000\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 2 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add/y" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack_1/1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack_1/1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\001\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 2 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\000\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_1/x" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_1" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_1/x" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack_1/1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack_1/1" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\001\000\000\000\001\000\000\000" + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 2 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/Equal" + op: "Equal" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_1" + attr { + key: "T" + value { + type: DT_INT64 + } + } + attr { + key: "incompatible_shape_error" + value { + b: true + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2/stack" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 2 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_2/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 2 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_2" + op: "AddV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_2/y" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2/stack_1" + op: "Pack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/add_2" + attr { + key: "N" + value { + i: 1 + } + } + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "axis" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2/stack_2" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 1 + } + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2" + op: "StridedSlice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2/stack" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2/stack_1" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2/stack_2" + attr { + key: "Index" + value { + type: DT_INT32 + } + } + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "begin_mask" + value { + i: 0 + } + } + attr { + key: "ellipsis_mask" + value { + i: 0 + } + } + attr { + key: "end_mask" + value { + i: 0 + } + } + attr { + key: "new_axis_mask" + value { + i: 0 + } + } + attr { + key: "shrink_axis_mask" + value { + i: 0 + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/Select" + op: "Select" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/Equal" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/strided_slice_2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/zeros_2" + attr { + key: "T" + value { + type: DT_FLOAT + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_2/axis" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_2" + op: "ConcatV2" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-0/Select" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-1/strided_slice" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/Window-2/Select" + input: "StatefulPartitionedCall/EncoderDNN/CNN_layers/ngram_order_3/Ngram-3-Conv/concat_2/axis" + attr { + key: "N" + value { + i: 3 + } + } + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "Tidx" + value { + type: DT_INT32 + } + } +} diff --git a/src/frontends/tensorflow/tests/test_models/models_pbtxt/split_in_function.pbtxt b/src/frontends/tensorflow/tests/test_models/models_pbtxt/split_in_function.pbtxt new file mode 100644 index 00000000000000..1de73f6fe11aa5 --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/models_pbtxt/split_in_function.pbtxt @@ -0,0 +1,229 @@ +node { + name: "x" + op: "Placeholder" + attr { + key: "_user_specified_name" + value { + s: "x" + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + dim { + size: 3 + } + dim { + size: 20 + } + } + } + } +} +node { + name: "PartitionedCall" + op: "PartitionedCall" + input: "x" + attr { + key: "Tin" + value { + list { + type: DT_FLOAT + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_FLOAT + } + } + } + attr { + key: "_collective_manager_ids" + value { + list { + } + } + } + attr { + key: "_read_only_resource_inputs" + value { + list { + } + } + } + attr { + key: "config" + value { + s: "" + } + } + attr { + key: "config_proto" + value { + s: "\n\007\n\003CPU\020\001\n\007\n\003GPU\020\0002\002J\0008\001\202\001\000" + } + } + attr { + key: "executor_type" + value { + s: "" + } + } + attr { + key: "f" + value { + func { + name: "__inference_second_func_15" + } + } + } +} +node { + name: "Identity" + op: "Identity" + input: "PartitionedCall" + attr { + key: "T" + value { + type: DT_FLOAT + } + } +} +library { + function { + signature { + name: "__inference_second_func_15" + input_arg { + name: "x" + type: DT_FLOAT + } + output_arg { + name: "identity" + type: DT_FLOAT + } + } + node_def { + name: "split/split_dim" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } + } + node_def { + name: "split" + op: "Split" + input: "split/split_dim:output:0" + input: "x" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "num_split" + value { + i: 3 + } + } + } + node_def { + name: "add" + op: "AddV2" + input: "split:output:0" + input: "split:output:1" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + } + node_def { + name: "add_1" + op: "AddV2" + input: "add:z:0" + input: "split:output:2" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + } + node_def { + name: "Identity" + op: "Identity" + input: "add_1:z:0" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + } + ret { + key: "identity" + value: "Identity:output:0" + } + attr { + key: "_construction_context" + value { + s: "kEagerRuntime" + } + } + arg_attr { + value { + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 3 + } + dim { + size: 20 + } + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "x" + } + } + } + } + } +} +versions { + producer: 1286 + min_consumer: 12 +} diff --git a/src/frontends/tensorflow_common/include/common_op_table.hpp b/src/frontends/tensorflow_common/include/common_op_table.hpp index eeec128e8c1de3..db49abd5e59186 100644 --- a/src/frontends/tensorflow_common/include/common_op_table.hpp +++ b/src/frontends/tensorflow_common/include/common_op_table.hpp @@ -10,7 +10,6 @@ #include "openvino/core/node_vector.hpp" #include "openvino/frontend/node_context.hpp" -#include "openvino_conversions.hpp" #include "utils.hpp" #define TENSORFLOW_OP_VALIDATION(node_context, ...) \ @@ -22,7 +21,8 @@ namespace ov { namespace frontend { namespace tensorflow { namespace op { -#define OP_CONVERTER(op) OutputVector op(const ov::frontend::NodeContext& node) +#define OP_CONVERTER(op) OutputVector op(const ov::frontend::NodeContext& node) +#define OP_CONVERTER_NAMED(op) NamedOutputVector op(const ov::frontend::NodeContext& node) #define OP_T_CONVERTER(op) \ template \ OutputVector op(const ov::frontend::NodeContext& node) @@ -34,7 +34,6 @@ OP_T_CONVERTER(translate_direct_reduce_op); OP_CONVERTER(translate_add_n_op); OP_CONVERTER(translate_arg_max_op); OP_CONVERTER(translate_arg_min_op); -OP_CONVERTER(translate_assert_op); OP_CONVERTER(translate_avg_pool_op); OP_CONVERTER(translate_batch_mat_mul_op); OP_CONVERTER(translate_batch_to_space_nd_op); @@ -50,7 +49,7 @@ OP_CONVERTER(translate_conv_2d_op); OP_CONVERTER(translate_conv_2d_backprop_input_op); OP_CONVERTER(translate_conv_3d_op); OP_CONVERTER(translate_conv_3d_backprop_input_v2_op); -OP_CONVERTER(translate_ctc_greedy_decoder_op); +OP_CONVERTER_NAMED(translate_ctc_greedy_decoder_op); OP_CONVERTER(translate_ctc_loss_op); OP_CONVERTER(translate_cumsum_op); OP_CONVERTER(translate_crop_and_resize_op); @@ -64,7 +63,7 @@ OP_CONVERTER(translate_extract_image_patches_op); OP_CONVERTER(translate_fake_quant_op); OP_CONVERTER(translate_fill_op); OP_CONVERTER(translate_floor_div_op); -OP_CONVERTER(translate_fused_batch_norm_op); +OP_CONVERTER_NAMED(translate_fused_batch_norm_op); OP_CONVERTER(translate_gather_op); OP_CONVERTER(translate_gather_v2_op); OP_CONVERTER(translate_gather_nd_op); @@ -134,8 +133,8 @@ OP_CONVERTER(translate_tensor_list_reserve_op); OP_CONVERTER(translate_tensor_list_set_item_op); OP_CONVERTER(translate_tensor_list_stack_op); OP_CONVERTER(translate_tile_op); -OP_CONVERTER(translate_top_k_op); -OP_CONVERTER(translate_top_k_v2_op); +OP_CONVERTER_NAMED(translate_top_k_op); +OP_CONVERTER_NAMED(translate_top_k_v2_op); OP_CONVERTER(translate_transpose_op); OP_CONVERTER(translate_unpack_op); OP_CONVERTER(translate_where_op); @@ -143,7 +142,7 @@ OP_CONVERTER(translate_x_div_y_op); OP_CONVERTER(translate_zeros_like_op); // Translators for internal operations -OP_CONVERTER(translate_unique_op); +OP_CONVERTER_NAMED(translate_unique_op); } // namespace op } // namespace tensorflow diff --git a/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp b/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp index f882474c79c49b..2f91c4f4232f6b 100644 --- a/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp +++ b/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp @@ -30,19 +30,12 @@ class DecoderFake : public ov::frontend::DecoderBase { void get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const override { FRONT_END_OP_CONVERSION_CHECK(false, "Internal error: the get_input_node method of the fake node decoder is invoked."); } - void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const override { - FRONT_END_OP_CONVERSION_CHECK(false, - "Internal error: the get_input_node method of the fake node decoder is invoked."); - } - const std::string& get_op_type() const override { // this method must not throw an exception since it is used by TF FE FrameworkNode constructor return op_type; diff --git a/src/frontends/tensorflow_common/include/helper_ops/sparse_segment_ops.hpp b/src/frontends/tensorflow_common/include/helper_ops/sparse_segment_ops.hpp index 0e867c2fd0e112..b4d38f54bd4766 100644 --- a/src/frontends/tensorflow_common/include/helper_ops/sparse_segment_ops.hpp +++ b/src/frontends/tensorflow_common/include/helper_ops/sparse_segment_ops.hpp @@ -52,7 +52,9 @@ class SparseSegmentSum : public ov::frontend::tensorflow::InternalOperation { // num_segments input is optional so it is not always possible to deduce the first dimension of the output shape if (get_input_size() > 3) { ov::PartialShape num_segments_value; + OPENVINO_SUPPRESS_DEPRECATED_START if (output_rank.is_static() && ov::evaluate_as_partial_shape(input_value(3), num_segments_value)) { + OPENVINO_SUPPRESS_DEPRECATED_END FRONT_END_OP_CONVERSION_CHECK(output_rank.get_length() >= 1, "Data input of SparseSegmentSum must be of rank >= 1."); output_shape[0] = num_segments_value[0]; diff --git a/src/frontends/tensorflow_common/include/helper_ops/string_constant.hpp b/src/frontends/tensorflow_common/include/helper_ops/string_constant.hpp new file mode 100644 index 00000000000000..40d4a4ce306d7c --- /dev/null +++ b/src/frontends/tensorflow_common/include/helper_ops/string_constant.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "internal_operation.hpp" + +namespace ov { +namespace frontend { +namespace tensorflow { + +/// Pseudo-entity for storing strings +class StringConstant : public InternalOperation { +public: + OPENVINO_OP("StringConstant", "ov::frontend::tensorflow::util", InternalOperation); + + StringConstant(ov::Any data, const std::shared_ptr& decoder = std::make_shared()) + : InternalOperation(decoder, {}, 1), + m_data(data) { + validate_and_infer_types(); + } + + StringConstant(std::string& str, const std::shared_ptr& decoder = std::make_shared()) + : InternalOperation(decoder, {}, 1), + m_data({str}) { + validate_and_infer_types(); + } + + StringConstant(const std::shared_ptr& decoder = std::make_shared()) + : InternalOperation(decoder, {}, 1) { + validate_and_infer_types(); + } + + void validate_and_infer_types() override { + set_output_type(0, ov::element::dynamic, ov::PartialShape::dynamic()); + } + + ov::Any get_data() { + return m_data; + } + + std::string& get_string() { + return m_data.as>()[0]; + } + +private: + ov::Any m_data; + ov::Shape m_shape; +}; + +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow_common/include/helper_transforms/saved_model_unused_remover.hpp b/src/frontends/tensorflow_common/include/helper_transforms/saved_model_unused_remover.hpp new file mode 100644 index 00000000000000..b86ddf31ce57db --- /dev/null +++ b/src/frontends/tensorflow_common/include/helper_transforms/saved_model_unused_remover.hpp @@ -0,0 +1,27 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { +namespace tensorflow { +namespace pass { + +// This transformation removes isolated subgraph unused Parameters and +// Results marked as unused by Saved Model settings +class SavedModelUnusedRemover : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("ov::frontend::tensorflow::pass::SavedModelUnusedRemover"); + SavedModelUnusedRemover() {} + + bool run_on_model(const std::shared_ptr& m) override; +}; + +} // namespace pass +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow_common/include/openvino_conversions.hpp b/src/frontends/tensorflow_common/include/openvino_conversions.hpp deleted file mode 100644 index 90ad15398bf7eb..00000000000000 --- a/src/frontends/tensorflow_common/include/openvino_conversions.hpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include "openvino/opsets/opset8.hpp" - -namespace ov { -namespace frontend { -namespace tensorflow { - -std::shared_ptr make_transpose(const ov::Output& arg, - const ov::AxisVector& input_order); - -std::shared_ptr make_reshape(const ov::Output& arg, - const std::vector& new_shape); - -namespace detail { -template -void convert_nhwc_to_hw(const std::vector& src, std::vector& dst) { - if (dst.size() >= 2) { - dst[0] = src[1]; - dst[1] = src[2]; - } - if (dst.size() >= 3) { - dst[2] = src[3]; - } -} - -template -void convert_nchw_to_hw(const std::vector& src, std::vector& dst) { - if (dst.size() >= 2) { - dst[0] = src[2]; - dst[1] = src[3]; - } - if (dst.size() >= 3) { - dst[2] = src[4]; - } -} -} // namespace detail - -void convert_nhwc_to_nchw(bool need_convert, ov::Output& node, ov::Rank input_rank = ov::Rank::dynamic()); - -void convert_nchw_to_nhwc(bool need_convert, ov::Output& node, ov::Rank input_rank = ov::Rank::dynamic()); - -template -void convert_nhwc_to_hw(bool is_nhwc, const std::vector& src, std::vector& dst) { - if (is_nhwc) { - detail::convert_nhwc_to_hw(src, dst); - } else { - detail::convert_nchw_to_hw(src, dst); - } -} - -} // namespace tensorflow -} // namespace frontend -} // namespace ov diff --git a/src/frontends/tensorflow_common/include/pass/transpose_sinking.hpp b/src/frontends/tensorflow_common/include/pass/transpose_sinking.hpp deleted file mode 100644 index 9753df19b9e8eb..00000000000000 --- a/src/frontends/tensorflow_common/include/pass/transpose_sinking.hpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "openvino/pass/pass.hpp" - -namespace ov { -namespace frontend { -namespace tensorflow { -namespace pass { - -class TransposeSinking : public ov::pass::ModelPass { -public: - OPENVINO_RTTI("ov::frontend::tensorflow::pass::TransposeSinking"); - TransposeSinking() = default; - bool run_on_model(const std::shared_ptr& function) override; -}; - -} // namespace pass -} // namespace tensorflow -} // namespace frontend -} // namespace ov diff --git a/src/frontends/tensorflow_common/include/tf_framework_node.hpp b/src/frontends/tensorflow_common/include/tf_framework_node.hpp index 0d31c1dad43574..cffba769751a7d 100644 --- a/src/frontends/tensorflow_common/include/tf_framework_node.hpp +++ b/src/frontends/tensorflow_common/include/tf_framework_node.hpp @@ -15,6 +15,7 @@ namespace tensorflow { class FrameworkNode : public ov::op::util::FrameworkNode { public: + static constexpr const char* failed_conversion_key = "tensorflow::FrameworkNode::failed_conversion_key"; OPENVINO_OP("FrameworkNode", "util", ::ov::op::util::FrameworkNode); FrameworkNode(const std::shared_ptr& decoder, const OutputVector& inputs, size_t num_outputs) diff --git a/src/frontends/tensorflow_common/include/utils.hpp b/src/frontends/tensorflow_common/include/utils.hpp index 06e68771aef80b..1abbb13a3b29b9 100644 --- a/src/frontends/tensorflow_common/include/utils.hpp +++ b/src/frontends/tensorflow_common/include/utils.hpp @@ -7,7 +7,6 @@ #include "openvino/core/validation_util.hpp" #include "openvino/frontend/node_context.hpp" #include "openvino/opsets/opset10.hpp" -#include "openvino/opsets/opset8.hpp" #include "openvino/pass/graph_rewrite.hpp" #ifndef TENSORFLOW_OP_VALIDATION @@ -20,7 +19,7 @@ namespace ov { namespace frontend { namespace tensorflow { -using OpMap = std::unordered_map>>; +using OpMap = std::unordered_map; void extract_operation_name_and_port(const std::string& port_name, std::string& operation_name, @@ -68,7 +67,9 @@ void get_const_input(const NodeContext& node, int input_index, std::vector* v std::to_string(input_size) + " inputs, but requested input port index to be " + std::to_string(input_size)); auto ov_input = node.get_input(input_index); + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = get_constant_from_source(ov_input)) { + OPENVINO_SUPPRESS_DEPRECATED_END *vector = constant->cast_vector(); return; } @@ -98,6 +99,56 @@ ov::op::PadMode convert_padding_mode(const NodeContext& node, const std::string& Output compute_subgraph_scalar_rank(const Output& output, element::Type output_type, bool as_scalar = false); + +std::shared_ptr make_transpose(const ov::Output& arg, + const ov::AxisVector& input_order); + +std::shared_ptr make_reshape(const ov::Output& arg, + const std::vector& new_shape); + +template +void convert_nhwc_to_hw(const std::vector& src, std::vector& dst) { + if (dst.size() >= 2) { + FRONT_END_GENERAL_CHECK(src.size() > 2, + "[TensorFlow Frontend] Internal error: source vector size must be greater than 2."); + dst[0] = src[1]; + dst[1] = src[2]; + } + if (dst.size() >= 3) { + FRONT_END_GENERAL_CHECK(src.size() > 3, + "[TensorFlow Frontend] Internal error: source vector size must be greater than 3."); + dst[2] = src[3]; + } +} + +template +void convert_nchw_to_hw(const std::vector& src, std::vector& dst) { + if (dst.size() >= 2) { + FRONT_END_GENERAL_CHECK(src.size() > 3, + "[TensorFlow Frontend] Internal error: source vector size must be greater than 3."); + dst[0] = src[2]; + dst[1] = src[3]; + } + if (dst.size() >= 3) { + FRONT_END_GENERAL_CHECK(src.size() > 4, + "[TensorFlow Frontend] Internal error: source vector size must be greater than 4."); + dst[2] = src[4]; + } +} + +void convert_nhwc_to_nchw(bool need_convert, ov::Output& node, ov::Rank input_rank = ov::Rank::dynamic()); + +void convert_nchw_to_nhwc(bool need_convert, ov::Output& node, ov::Rank input_rank = ov::Rank::dynamic()); + +template +void convert_nhwc_to_hw(bool is_nhwc, const std::vector& src, std::vector& dst) { + if (is_nhwc) { + convert_nhwc_to_hw(src, dst); + } else { + convert_nchw_to_hw(src, dst); + } +} + } // namespace tensorflow } // namespace frontend } // namespace ov diff --git a/src/frontends/tensorflow_common/src/helper_transforms/const_to_result_remover.cpp b/src/frontends/tensorflow_common/src/helper_transforms/const_to_result_remover.cpp index 1963bcf47dae22..04dd756b3e320c 100644 --- a/src/frontends/tensorflow_common/src/helper_transforms/const_to_result_remover.cpp +++ b/src/frontends/tensorflow_common/src/helper_transforms/const_to_result_remover.cpp @@ -4,6 +4,7 @@ #include "helper_transforms/const_to_result_remover.hpp" +#include "helper_ops/string_constant.hpp" #include "helper_ops/unsupported_constant.hpp" #include "openvino/opsets/opset10.hpp" @@ -22,7 +23,8 @@ bool ConstToResultRemover::run_on_model(const std::shared_ptr& m) { for (const auto& result : m->get_results()) { auto unsupported_const = as_type_ptr(result->get_input_node_shared_ptr(0)); auto const_node = as_type_ptr(result->get_input_node_shared_ptr(0)); - if (unsupported_const || const_node) { + auto string_const = as_type_ptr(result->get_input_node_shared_ptr(0)); + if (unsupported_const || const_node || string_const) { results_to_remove.push_back(result); } } diff --git a/src/frontends/tensorflow_common/src/helper_transforms/saved_model_unused_remover.cpp b/src/frontends/tensorflow_common/src/helper_transforms/saved_model_unused_remover.cpp new file mode 100644 index 00000000000000..a7e21c670e782a --- /dev/null +++ b/src/frontends/tensorflow_common/src/helper_transforms/saved_model_unused_remover.cpp @@ -0,0 +1,74 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "helper_transforms/saved_model_unused_remover.hpp" + +#include "openvino/opsets/opset8.hpp" + +using namespace std; +using namespace ov::opset8; + +namespace ov { +namespace frontend { +namespace tensorflow { +namespace pass { + +bool SavedModelUnusedRemover::run_on_model(const std::shared_ptr& m) { + ParameterVector params_to_remove; + ResultVector results_to_remove; + + // There is two cases + // 1. When we found unused result with/without unused parameter + // 2. When we found unused parameter + + for (const auto& result : m->get_results()) { + bool isUsed = false; + for (size_t i = 0; i < result->get_input_size(); ++i) { + const auto& node_names = result->get_input_tensor(i).get_names(); + isUsed = std::find(node_names.begin(), node_names.end(), "saved_model_unused") == node_names.end(); + } + if (!isUsed) { + results_to_remove.push_back(result); + continue; + } + + auto param = as_type_ptr(result->get_input_node_shared_ptr(0)); + if (param) { + for (size_t i = 0; i < param->get_output_size(); ++i) { + const auto& node_names = param->get_output_tensor(i).get_names(); + isUsed = std::find(node_names.begin(), node_names.end(), "saved_model_unused") == node_names.end(); + } + if (!isUsed) { + results_to_remove.push_back(result); + params_to_remove.push_back(param); + } + } + } + + for (const auto& param : m->get_parameters()) { + bool isUsed = false; + for (size_t i = 0; i < param->get_output_size(); ++i) { + const auto& node_names = param->get_output_tensor(i).get_names(); + isUsed = std::find(node_names.begin(), node_names.end(), "saved_model_unused") == node_names.end(); + } + if (!isUsed && std::find(params_to_remove.begin(), params_to_remove.end(), param) == params_to_remove.end()) { + params_to_remove.push_back(param); + } + } + + for (const auto& result : results_to_remove) { + m->remove_result(result); + } + + for (const auto& param : params_to_remove) { + m->remove_parameter(param); + } + + return true; +} + +} // namespace pass +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow_common/src/op/assert.cpp b/src/frontends/tensorflow_common/src/op/assert.cpp deleted file mode 100644 index 5275e85a8c2edc..00000000000000 --- a/src/frontends/tensorflow_common/src/op/assert.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include - -#include "common_op_table.hpp" -#include "openvino/core/validation_util.hpp" - -using namespace std; - -namespace ov { -namespace frontend { -namespace tensorflow { -namespace op { - -OutputVector translate_assert_op(const NodeContext& node) { - default_op_checks(node, 1, {"Assert"}); - auto cond = node.get_input(0); - auto cond_const = get_constant_from_source(cond); - TENSORFLOW_OP_VALIDATION(node, - cond_const, - "[TensorFlow Frontend] The condition must be constant for further model conversion."); - auto cond_values = cond_const->cast_vector(); - TENSORFLOW_OP_VALIDATION(node, - cond_values.size() == 1, - "[TensorFlow Frontend] Incorrect model - the condition must have one element."); - TENSORFLOW_OP_VALIDATION(node, - cond_values[0], - "[TensorFlow Frontend] The condition must be true for further model conversion."); - return {}; -} -} // namespace op -} // namespace tensorflow -} // namespace frontend -} // namespace ov diff --git a/src/frontends/tensorflow_common/src/op/concat.cpp b/src/frontends/tensorflow_common/src/op/concat.cpp index 07c4844b010d06..812a593fd2f3e3 100644 --- a/src/frontends/tensorflow_common/src/op/concat.cpp +++ b/src/frontends/tensorflow_common/src/op/concat.cpp @@ -9,7 +9,7 @@ using namespace std; using namespace ov; using namespace opset8; using namespace ov::frontend; -using namespace frontend::tensorflow::detail; +using namespace ov::frontend::tensorflow; namespace ov { namespace frontend { diff --git a/src/frontends/tensorflow_common/src/op/const.cpp b/src/frontends/tensorflow_common/src/op/const.cpp index 292b9e0190a42d..956d2f1c0bcbb8 100644 --- a/src/frontends/tensorflow_common/src/op/const.cpp +++ b/src/frontends/tensorflow_common/src/op/const.cpp @@ -3,6 +3,7 @@ // #include "common_op_table.hpp" +#include "helper_ops/string_constant.hpp" #include "helper_ops/unsupported_constant.hpp" #include "openvino/opsets/opset8.hpp" @@ -16,10 +17,15 @@ namespace tensorflow { namespace op { OutputVector translate_const_op(const NodeContext& node) { - auto ov_type = node.get_attribute("dtype"); + auto ov_type = node.get_attribute_as_any("dtype"); std::shared_ptr const_node; - if (ov_type == element::dynamic) { - const_node = std::make_shared(); + if (!ov_type.is() || ov_type.as() == ov::element::dynamic || + ov_type.as() == ov::element::undefined) { + if (ov_type.is() && ov_type.as() == "DT_STRING") { + const_node = std::make_shared(node.get_attribute_as_any("value")); + } else { + const_node = std::make_shared(); + } } else { auto tensor = node.get_attribute("value"); const_node = std::make_shared(tensor.get_element_type(), tensor.get_shape(), tensor.data()); diff --git a/src/frontends/tensorflow_common/src/op/ctc_greedy_decoder.cpp b/src/frontends/tensorflow_common/src/op/ctc_greedy_decoder.cpp index 49f2c35eeec46d..ea98d701d1a1cf 100644 --- a/src/frontends/tensorflow_common/src/op/ctc_greedy_decoder.cpp +++ b/src/frontends/tensorflow_common/src/op/ctc_greedy_decoder.cpp @@ -10,14 +10,14 @@ using namespace std; using namespace ov; using namespace opset8; using namespace frontend; -using namespace frontend::tensorflow::detail; +using namespace frontend::tensorflow; namespace ov { namespace frontend { namespace tensorflow { namespace op { -OutputVector translate_ctc_greedy_decoder_op(const NodeContext& node) { +NamedOutputVector translate_ctc_greedy_decoder_op(const NodeContext& node) { default_op_checks(node, 2, {"CTCGreedyDecoder"}); auto inputs = node.get_input(0); auto sequence_length = node.get_input(1); @@ -82,7 +82,10 @@ OutputVector translate_ctc_greedy_decoder_op(const NodeContext& node) { set_node_name(node.get_name() + ":2", dense_shape); set_node_name(node.get_name() + ":3", neg_sum_logits); - return {decoded_indices_transposed, decoded_values, dense_shape, neg_sum_logits}; + return {{"decoded_indices", decoded_indices_transposed}, + {"decoded_values", decoded_values}, + {"decoded_shape", dense_shape}, + {"log_probability", neg_sum_logits}}; } } // namespace op } // namespace tensorflow diff --git a/src/frontends/tensorflow_common/src/op/ctc_loss.cpp b/src/frontends/tensorflow_common/src/op/ctc_loss.cpp index 84f2f6914f35e5..b4b72bb407fa0c 100644 --- a/src/frontends/tensorflow_common/src/op/ctc_loss.cpp +++ b/src/frontends/tensorflow_common/src/op/ctc_loss.cpp @@ -9,7 +9,7 @@ using namespace std; using namespace ov; using namespace opset8; using namespace ov::frontend; -using namespace frontend::tensorflow::detail; +using namespace frontend::tensorflow; namespace ov { namespace frontend { diff --git a/src/frontends/tensorflow_common/src/op/fused_batch_norm.cpp b/src/frontends/tensorflow_common/src/op/fused_batch_norm.cpp index 7af6ce29497384..3b1d5613db2d0b 100644 --- a/src/frontends/tensorflow_common/src/op/fused_batch_norm.cpp +++ b/src/frontends/tensorflow_common/src/op/fused_batch_norm.cpp @@ -8,7 +8,7 @@ using namespace std; using namespace ov; -using namespace ov::opset8; +using namespace ov::opset10; namespace ov { namespace frontend { @@ -234,7 +234,7 @@ void compute_fused_batch_norm_training(const NodeContext& node, } } // namespace -OutputVector translate_fused_batch_norm_op(const NodeContext& node) { +NamedOutputVector translate_fused_batch_norm_op(const NodeContext& node) { default_op_checks(node, 3, {"FusedBatchNorm", "FusedBatchNormV2", "FusedBatchNormV3"}); auto scale = node.get_input(1); @@ -263,11 +263,15 @@ OutputVector translate_fused_batch_norm_op(const NodeContext& node) { set_node_name(node.get_name() + ":3", zero_const.get_node_shared_ptr()); set_node_name(node.get_name() + ":4", zero_const2.get_node_shared_ptr()); - OutputVector results = OutputVector{fused_batch_norm, batch_mean, batch_variance, zero_const, zero_const2}; + auto results = NamedOutputVector{{"y", fused_batch_norm}, + {"batch_mean", batch_mean}, + {"batch_variance", batch_variance}, + {"reserve_space_1", zero_const}, + {"reserve_space_2", zero_const2}}; if (is_v3) { auto zero_const3 = create_same_type_const_scalar(scale, 0); set_node_name(node.get_name() + ":5", zero_const3.get_node_shared_ptr()); - results.push_back(zero_const3); + results.push_back({"reserve_space_3", zero_const3}); } return results; diff --git a/src/frontends/tensorflow_common/src/op/identity.cpp b/src/frontends/tensorflow_common/src/op/identity.cpp index 7bd6d7735e37fb..a2b36f1f1a6083 100644 --- a/src/frontends/tensorflow_common/src/op/identity.cpp +++ b/src/frontends/tensorflow_common/src/op/identity.cpp @@ -14,7 +14,13 @@ namespace tensorflow { namespace op { OutputVector translate_identity_op(const NodeContext& node) { - vector supported_ops = {"Identity", "PreventGradient", "Snapshot", "StopGradient"}; + vector supported_ops = {"Identity", + "PreventGradient", + "Snapshot", + "StopGradient", + "ReadVariableOp", + "ShardedFilename", + "MergeV2Checkpoints"}; default_op_checks(node, 1, supported_ops); auto input = node.get_input(0); diff --git a/src/frontends/tensorflow_common/src/op/max_pool.cpp b/src/frontends/tensorflow_common/src/op/max_pool.cpp index 6d698b6c3b88cf..b2a4520249eb78 100644 --- a/src/frontends/tensorflow_common/src/op/max_pool.cpp +++ b/src/frontends/tensorflow_common/src/op/max_pool.cpp @@ -8,7 +8,7 @@ using namespace std; using namespace ov; -using namespace ov::frontend::tensorflow::detail; +using namespace ov::frontend::tensorflow; namespace ov { namespace frontend { @@ -91,10 +91,12 @@ OutputVector translate_max_pool_v2(const NodeContext& node) { auto ksize = node.get_input(1); auto strides = node.get_input(2); + OPENVINO_SUPPRESS_DEPRECATED_START auto ksize_constant = get_constant_from_source(ksize); TENSORFLOW_OP_VALIDATION(node, ksize_constant, "MaxPoolV2 is supported only with constant ksize."); auto strides_constant = get_constant_from_source(strides); TENSORFLOW_OP_VALIDATION(node, ksize_constant, "MaxPoolV2 is supported only with constant strides."); + OPENVINO_SUPPRESS_DEPRECATED_END auto ksize_vector = ksize_constant->cast_vector(); auto strides_vector = strides_constant->cast_vector(); diff --git a/src/frontends/tensorflow_common/src/op/no_op.cpp b/src/frontends/tensorflow_common/src/op/no_op.cpp index 9d8552b6f71c24..388d5c5c6af573 100644 --- a/src/frontends/tensorflow_common/src/op/no_op.cpp +++ b/src/frontends/tensorflow_common/src/op/no_op.cpp @@ -15,7 +15,7 @@ namespace op { OutputVector translate_no_op(const NodeContext& node) { // the operation does nothing in terms of data generation - default_op_checks(node, 0, {"NoOp", "SaveV2"}); + default_op_checks(node, 0, {"NoOp", "SaveV2", "Assert", "LookupTableInsert", "LookupTableInsertV2"}); return {}; } } // namespace op diff --git a/src/frontends/tensorflow_common/src/op/strided_slice.cpp b/src/frontends/tensorflow_common/src/op/strided_slice.cpp index e0e7e6761bd8f9..50ceb763a80658 100644 --- a/src/frontends/tensorflow_common/src/op/strided_slice.cpp +++ b/src/frontends/tensorflow_common/src/op/strided_slice.cpp @@ -16,42 +16,56 @@ namespace tensorflow { namespace op { OutputVector translate_strided_slice_op(const NodeContext& node) { + default_op_checks(node, 4, {"StridedSlice", "STRIDED_SLICE"}); auto input = node.get_input(0); auto begin = node.get_input(1); auto end = node.get_input(2); auto strides = node.get_input(3); - auto begin_mask = node.get_attribute("begin_mask", 0); - auto end_mask = node.get_attribute("end_mask", 0); - auto new_axis_mask = node.get_attribute("new_axis_mask", 0); - auto ellipsis_mask = node.get_attribute("ellipsis_mask", 0); - auto shrink_axis_mask = node.get_attribute("shrink_axis_mask", 0); - auto mask_to_vector = [](int64_t mask) { - size_t length = sizeof(mask) * CHAR_BIT; - vector vec(length, 0); if (mask == 0) { - return vec; + return vector{}; } - for (size_t i = 0; i < length; ++i) { - if (static_cast(mask >> i & 0x1) == 1) { + size_t max_length = sizeof(mask) * CHAR_BIT; + vector vec; + vec.reserve(max_length); + for (size_t i = 0; i < max_length; ++i) { + if (((mask >> i) & 0x1) == 1) { + // resize the vector by appending with required number of zeros + vec.resize(i + 1, 0); vec[i] = 1; } } return vec; }; - auto res = make_shared(input, - begin, - end, - strides, - mask_to_vector(begin_mask), - mask_to_vector(end_mask), - mask_to_vector(new_axis_mask), - mask_to_vector(shrink_axis_mask), - mask_to_vector(ellipsis_mask)); - set_node_name(node.get_name(), res); - return res->outputs(); + // retrieve attributes for StridedSlice operation + auto begin_mask = mask_to_vector(node.get_attribute("begin_mask", 0)); + auto end_mask = mask_to_vector(node.get_attribute("end_mask", 0)); + auto new_axis_mask = mask_to_vector(node.get_attribute("new_axis_mask", 0)); + auto ellipsis_mask = mask_to_vector(node.get_attribute("ellipsis_mask", 0)); + auto shrink_axis_mask = mask_to_vector(node.get_attribute("shrink_axis_mask", 0)); + + // the masks can be of different length and we need to align them by the maximum length + size_t max_length = std::max( + {begin_mask.size(), end_mask.size(), new_axis_mask.size(), ellipsis_mask.size(), shrink_axis_mask.size()}); + begin_mask.resize(max_length, 0); + end_mask.resize(max_length, 0); + new_axis_mask.resize(max_length, 0); + ellipsis_mask.resize(max_length, 0); + shrink_axis_mask.resize(max_length, 0); + + auto strided_slice = make_shared(input, + begin, + end, + strides, + begin_mask, + end_mask, + new_axis_mask, + shrink_axis_mask, + ellipsis_mask); + set_node_name(node.get_name(), strided_slice); + return {strided_slice}; } } // namespace op diff --git a/src/frontends/tensorflow_common/src/op/top_k.cpp b/src/frontends/tensorflow_common/src/op/top_k.cpp index 42c38975cdcb98..b3761b4348f318 100644 --- a/src/frontends/tensorflow_common/src/op/top_k.cpp +++ b/src/frontends/tensorflow_common/src/op/top_k.cpp @@ -3,16 +3,18 @@ // #include "common_op_table.hpp" -#include "openvino/opsets/opset8.hpp" +#include "openvino/opsets/opset11.hpp" using namespace std; -using namespace ov::opset8; +using namespace ov::opset11; namespace ov { namespace frontend { namespace tensorflow { namespace op { -OutputVector translate_top_k_base_op(const NodeContext& node, const ov::Output& k_input, int min_input_size) { +NamedOutputVector translate_top_k_base_op(const NodeContext& node, + const ov::Output& k_input, + int min_input_size) { default_op_checks(node, min_input_size, {"TopK", "TopKV2"}); auto input = node.get_input(0); @@ -23,23 +25,23 @@ OutputVector translate_top_k_base_op(const NodeContext& node, const ov::Outputoutputs(); + return {{"values", top_k->output(0)}, {"indices", top_k->output(1)}}; } -OutputVector translate_top_k_op(const NodeContext& node) { +NamedOutputVector translate_top_k_op(const NodeContext& node) { // retrieve k attribute auto k = node.get_attribute("k"); auto k_input = make_shared(ov::element::i64, Shape{}, std::vector({k})); return translate_top_k_base_op(node, k_input, 1); } -OutputVector translate_top_k_v2_op(const NodeContext& node) { +NamedOutputVector translate_top_k_v2_op(const NodeContext& node) { default_op_checks(node, 2, {"TopKV2"}); auto k_input = node.get_input(1); return translate_top_k_base_op(node, k_input, 1); } - } // namespace op } // namespace tensorflow } // namespace frontend diff --git a/src/frontends/tensorflow_common/src/op/unique.cpp b/src/frontends/tensorflow_common/src/op/unique.cpp index 379ffe9e8ea39d..9c36001d9bc309 100644 --- a/src/frontends/tensorflow_common/src/op/unique.cpp +++ b/src/frontends/tensorflow_common/src/op/unique.cpp @@ -12,7 +12,7 @@ namespace ov { namespace frontend { namespace tensorflow { namespace op { -OutputVector translate_unique_op(const NodeContext& node) { +NamedOutputVector translate_unique_op(const NodeContext& node) { // This operation returns a tensor y containing all of the unique elements of x sorted in the same order that they // occur in x. This operation also returns a tensor idx the same size as x that contains the index of each value of // x in the unique output y. @@ -28,7 +28,7 @@ OutputVector translate_unique_op(const NodeContext& node) { set_out_name(node_name + ":0", unique->output(0)); set_out_name(node_name + ":1", unique->output(2)); - return {unique->output(0), unique->output(2)}; + return {{"y", unique->output(0)}, {"idx", unique->output(2)}}; } } // namespace op diff --git a/src/frontends/tensorflow_common/src/openvino_conversions.cpp b/src/frontends/tensorflow_common/src/openvino_conversions.cpp deleted file mode 100644 index 6fcaecd1df9a25..00000000000000 --- a/src/frontends/tensorflow_common/src/openvino_conversions.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "openvino_conversions.hpp" - -#include "utils.hpp" - -namespace ov { -namespace frontend { -namespace tensorflow { - -void convert_nhwc_to_nchw(bool need_convert, ov::Output& node, ov::Rank input_rank) { - if (need_convert) { - if (input_rank.is_dynamic()) { - // TODO: use ShapeOf sub-graph to generate permutation vector - OPENVINO_ASSERT(node.get_partial_shape().rank().is_static(), - "For conversion into the first channel format, the input rank must be static or determined " - "based on the operation."); - input_rank = node.get_partial_shape().rank(); - } - auto rank_value = input_rank.get_length(); - if (rank_value == 4) { - node = make_transpose(node, {0, 3, 1, 2}); - } else if (rank_value == 5) { - node = make_transpose(node, {0, 4, 1, 2, 3}); - } - } -} - -void convert_nchw_to_nhwc(bool need_convert, ov::Output& node, ov::Rank input_rank) { - if (need_convert) { - if (input_rank.is_dynamic()) { - // TODO: use ShapeOf sub-graph to generate permutation vector - OPENVINO_ASSERT(node.get_partial_shape().rank().is_static(), - "For conversion into the last channel format, the input rank must be static or determined " - "based on the operation."); - input_rank = node.get_partial_shape().rank(); - } - auto rank_value = input_rank.get_length(); - if (rank_value == 4) { - node = make_transpose(node, {0, 2, 3, 1}); - } else if (rank_value == 5) { - node = make_transpose(node, {0, 2, 3, 4, 1}); - } - } -} - -std::shared_ptr make_transpose(const ov::Output& arg, - const ov::AxisVector& input_order) { - auto order = std::make_shared(element::i64, Shape{input_order.size()}, input_order); - auto transpose = std::make_shared(arg, order); - return transpose; -} - -std::shared_ptr make_reshape(const ov::Output& arg, - const std::vector& new_shape) { - auto new_shape_node = std::make_shared(element::i64, Shape{new_shape.size()}, new_shape); - auto reshape = std::make_shared(arg, new_shape_node, true); - return reshape; -} - -} // namespace tensorflow -} // namespace frontend -} // namespace ov diff --git a/src/frontends/tensorflow_common/src/pass/transpose_sinking.cpp b/src/frontends/tensorflow_common/src/pass/transpose_sinking.cpp deleted file mode 100644 index 3b570d14b265ec..00000000000000 --- a/src/frontends/tensorflow_common/src/pass/transpose_sinking.cpp +++ /dev/null @@ -1,519 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "pass/transpose_sinking.hpp" - -#include "openvino/op/util/op_types.hpp" -#include "openvino/opsets/opset8.hpp" -#include "openvino/pass/pattern/op/label.hpp" -#include "openvino/util/common_util.hpp" -#include "openvino/util/log.hpp" -#include "openvino_conversions.hpp" -#include "utils.hpp" - -using namespace std; -using namespace ov; -using namespace ov::frontend::tensorflow; -using namespace opset8; - -using TransposeMap = unordered_map>; - -template -static T apply_permutation(const T& input, AxisVector order) { - T output(input.size()); - for (size_t i = 0; i < order.size(); i++) { - output[i] = input.at(order.at(i)); - } - return output; -} - -static AxisVector permutation_to_default_order(const AxisVector& axis_order) { - AxisVector out(axis_order.size()); - for (size_t i = 0; i < axis_order.size(); i++) { - out.at(axis_order[i]) = i; - } - return out; -} - -static AxisVector get_default_order(size_t rank) { - AxisVector default_order(rank); - std::iota(begin(default_order), end(default_order), 0); - return default_order; -} - -static size_t get_static_rank(const Output& output) { - auto rank = output.get_partial_shape().rank(); - OPENVINO_ASSERT(rank.is_static(), "Dynamic rank is not supported in TransposeSinking transformation."); - return rank.get_length(); -} - -template -static string describe(shared_ptr node) { - // ensure that it's either a reshape or a transpose - // TODO: use static_assert - if (!(std::is_base_of::value || std::is_base_of::value)) { - throw runtime_error("describe template specialization has to be either reshape or " - "transpose"); - } - stringstream ss; - auto transpose = as_type_ptr(node); - auto const1 = as_type_ptr(transpose->get_input_node_shared_ptr(1)); - if (transpose) { - ss << "transpose name: " << transpose->get_name(); - ss << " , input = " << transpose->input_value(0).get_node()->get_name(); - if (transpose->output(0).get_partial_shape().is_static()) { - ss << " , shape = " << ov::util::vector_to_string(transpose->output(0).get_shape()); - } - if (const1) { - ss << " , axis order = " << ov::util::vector_to_string(const1->get_axis_vector_val()); - } else { - ss << " , axis order = (unknown, not constant values)"; - } - } else { - ss << "Node can not be cast to Transpose/Reshape operations."; - } - return ss.str(); -} - -static shared_ptr make_reshape(const Output& arg, const AxisVector& input_order) { - auto order = std::make_shared(element::i64, Shape{input_order.size()}, input_order); - auto transpose = make_shared(arg, order, false); - OPENVINO_DEBUG << "Make Reshape " << describe(transpose); - return transpose; -} - -static void write_transposemap(TransposeMap& reorders, - const Output& target, - const shared_ptr& transpose) { - auto name = target.get_node()->get_name() + "." + to_string(target.get_index()); - OPENVINO_DEBUG << "Write TransposeMap[" << name << "] = " << describe(transpose); - reorders[name] = transpose; -} - -static shared_ptr read_transposemap(TransposeMap& reorders, const Output& target) { - auto name = target.get_node()->get_name() + "." + to_string(target.get_index()); - auto transpose = reorders.at(name); - OPENVINO_DEBUG << "Read TransposeMap[" << name << "] -> " << describe(transpose); - return transpose; -} - -static shared_ptr combine_transposes(const shared_ptr& t1, const shared_ptr& t2) { - auto default_order = get_default_order(get_static_rank(t1)); - auto t1_const = as_type_ptr(t1->input_value(1).get_node_shared_ptr()); - auto t2_const = as_type_ptr(t2->input_value(1).get_node_shared_ptr()); - - if (t1_const && t2_const) { - auto perm_t1 = apply_permutation(default_order, t1_const->get_axis_vector_val()); - auto perm_t2 = apply_permutation(perm_t1, t2_const->get_axis_vector_val()); - - auto combined = make_transpose(t2->input_value(0), perm_t2); - OPENVINO_DEBUG << "Combining " << describe(t1) << " and " << describe(t2) << " into " - << describe(combined); - return combined; - } - return {}; -} - -static void insert_transpose(const shared_ptr& target, const shared_ptr& transpose, size_t input_index) { - OPENVINO_DEBUG << "Inserting transpose at input " << target->get_name() << " input index " << input_index; - auto arg = target->input(input_index).get_source_output(); - if (arg.get_partial_shape().is_static()) { - OPENVINO_DEBUG << "Arg shape: " << arg.get_shape(); - } - auto new_order = as_type_ptr(transpose->input_value(1).get_node_shared_ptr()); - auto new_transpose = make_transpose(arg.get_node_shared_ptr(), new_order->get_axis_vector_val()); - OPENVINO_DEBUG << "Inserting transpose " << describe(new_transpose) << " at input " << target->get_name() - << " input index " << input_index; - - target->input(input_index).replace_source_output(new_transpose->output(0)); - if (std::dynamic_pointer_cast(target)) { - new_transpose->output(0).add_names(arg.get_names()); - arg.set_names({}); - } -} - -static void delete_transpose(const shared_ptr& transpose) { - OPENVINO_DEBUG << "Removing transpose " << transpose->get_name(); - if (!transpose->get_users().empty()) { - Output output = transpose->output(0); - OPENVINO_DEBUG << "output " << output.get_node_shared_ptr()->get_name(); - OPENVINO_DEBUG << "target input size " << output.get_target_inputs().size(); - output.replace(transpose->input_value(0)); - } -} - -static void mark_transpose_for_deletion(const shared_ptr& transpose, - set>& transposes_to_delete) { - OPENVINO_DEBUG << "Marking transpose " << transpose->get_name() << " for deletion"; - transposes_to_delete.insert(transpose); -} - -static shared_ptr create_default_transpose(const Output& n) { - auto default_order = get_default_order(get_static_rank(n)); - auto order = std::make_shared(element::i64, Shape{default_order.size()}, default_order); - return make_shared(n, order); -} - -// convert_binary_to_default_order is used when one of the arguments -// of a binary op isn't in the default format (i.e. nhwc instead of nchw) -// We normalize the "left" argument to match the order of the "right" argument -// by either inserting a transpose or a reshape, depending on the shape of the -// "left" argument. -static void convert_binary_to_default_order(const shared_ptr& binary, - const Input& input, - const Output& right, - TransposeMap& reorders, - set>& transposes_to_delete) { - auto left = input.get_source_output(); - auto right_t = read_transposemap(reorders, right); - auto right_const = as_type_ptr(right_t->input_value(1).get_node_shared_ptr()); - auto perm_to_def = permutation_to_default_order(right_const->get_axis_vector_val()); - - // if right input is being implicitly broadcasted, insert a reshape - // instead of a transpose - shared_ptr new_node; - auto left_rank = get_static_rank(left); - if (left_rank < perm_to_def.size() && left.get_partial_shape().is_static()) { - auto left_shape = left.get_shape(); - left_shape.insert(left_shape.begin(), perm_to_def.size() - left_rank, 1); - - auto new_shape = apply_permutation(left_shape, perm_to_def); - new_node = make_reshape(left, new_shape); - } else if (left_rank == perm_to_def.size()) { - new_node = make_transpose(left, perm_to_def); - } else { - throw runtime_error("case not supported when converting binary to default order"); - } - input.replace_source_output(new_node->output(0)); - - if (right.get_partial_shape().is_static()) { - OPENVINO_DEBUG << "right = " << ov::util::vector_to_string(right.get_shape()) << ", " - << right.get_node_shared_ptr()->get_name(); - } else { - OPENVINO_DEBUG << "right = " - << "dynamic shape, " << right.get_node_shared_ptr()->get_name(); - } - // this should now insert transpose on right - mark_transpose_for_deletion(right_t, transposes_to_delete); - write_transposemap(reorders, binary, right_t); -} - -static void materialize_shapes(const shared_ptr& n, - TransposeMap& reorders, - set>& transposes_to_delete) { - // For each node, create a default transpose for - // each of the outputs and store in the map - for (auto& it : n->outputs()) { - write_transposemap(reorders, it, create_default_transpose(it)); - } - - for (size_t i = 0; i < n->input_values().size(); i++) { - // materialize all pending transposes, flush pending transposes - auto arg = n->input_value(i); - auto arg_transpose = read_transposemap(reorders, arg); - OPENVINO_DEBUG << "Materializing " << describe(arg_transpose) << " for " - << arg.get_node_shared_ptr()->get_name(); - mark_transpose_for_deletion(arg_transpose, transposes_to_delete); - auto arg_transpose_order = as_type_ptr(arg_transpose->input_value(1).get_node_shared_ptr()); - if (arg_transpose_order && - arg_transpose_order->get_axis_vector_val() != get_default_order(get_static_rank(arg))) { - // Insert if arg needs to be transposed. - insert_transpose(n, arg_transpose, i); - } - } -} - -static bool sink_transpose(const shared_ptr& transpose, - TransposeMap& reorders, - set>& transposes_to_delete) { - OPENVINO_DEBUG << "Sinking Transpose :" << describe(transpose); - auto transpose_in = transpose->input_value(0); - auto orig_transpose = read_transposemap(reorders, transpose_in); - // combine both transposes - auto new_transpose = combine_transposes(orig_transpose, transpose); - if (new_transpose) { - // remove original transpose now it's combined with a new one - // should be safe to remove an already detached node - mark_transpose_for_deletion(orig_transpose, transposes_to_delete); - // replace transpose with combined one - replace_node(transpose, new_transpose); - mark_transpose_for_deletion(new_transpose, transposes_to_delete); - write_transposemap(reorders, new_transpose, new_transpose); - } else { - // combine_transposes failed - // transpose remains in the graph - OPENVINO_DEBUG << "CombineTranspose has failed. Writing original transpose to the transpose map."; - return false; - } - return true; -} - -static bool sink_unary(const shared_ptr& n, - TransposeMap& reorders, - set>& /* transposes_to_delete */) { - auto arg_transpose = read_transposemap(reorders, n->input_value(0)); - OPENVINO_DEBUG << "Propagating " << describe(arg_transpose) << " for " << n->get_name(); - write_transposemap(reorders, n, arg_transpose); - return true; -} - -static bool sink_binary(const shared_ptr& binary, - TransposeMap& reorders, - set>& transposes_to_delete) { - auto left = binary->input_value(0); - auto right = binary->input_value(1); - auto left_t = read_transposemap(reorders, left); - auto right_t = read_transposemap(reorders, right); - auto left_const = as_type_ptr(left_t->input_value(1).get_node_shared_ptr()); - auto right_const = as_type_ptr(right_t->input_value(1).get_node_shared_ptr()); - if (!(left_const && right_const)) { - OPENVINO_DEBUG << "TransposeSinking failed for binary op " << binary->get_name() - << "2nd inputs to Transposes must be constants."; - return false; - } - - auto left_order = left_const->get_axis_vector_val(); - auto right_order = right_const->get_axis_vector_val(); - - auto left_rank = get_static_rank(left); - auto right_rank = get_static_rank(right); - auto left_mismatch = left_order != get_default_order(left_rank); - auto right_mismatch = right_order != get_default_order(right_rank); - - OPENVINO_DEBUG << "Sink binary " << binary->get_name() - << " left transpose: " << ov::util::vector_to_string(left_order) - << " left default: " << ov::util::vector_to_string(get_default_order(left_rank)) - << " right transpose: " << ov::util::vector_to_string(right_order) - << " right default: " << ov::util::vector_to_string(get_default_order(right_rank)); - - if ((left_order.size() == right_order.size() && left_order == right_order) || (!left_mismatch && !right_mismatch)) { - // Propagate the reshape which matches the shape of the binary node - auto new_transpose = (binary->get_output_shape(0).size() == left.get_shape().size()) ? left_t : right_t; - OPENVINO_DEBUG << "Propagating " << describe(new_transpose) << " for " << binary->get_name(); - write_transposemap(reorders, binary, new_transpose); - // at this point, both transposes will be eventually removed - mark_transpose_for_deletion(left_t, transposes_to_delete); - mark_transpose_for_deletion(right_t, transposes_to_delete); - } else { - try { - if (right_mismatch) { - convert_binary_to_default_order(binary, binary->input(0), right, reorders, transposes_to_delete); - } else { - if (left_mismatch) { - convert_binary_to_default_order(binary, binary->input(1), left, reorders, transposes_to_delete); - } - } - } catch (const std::exception&) { - return false; - } - } - return true; -} - -static bool sink_pad(shared_ptr n, TransposeMap& reorders, set>& /* transposes_to_delete */) { - auto n_in = n->input_value(0); - auto arg_transpose = read_transposemap(reorders, n_in); - describe(arg_transpose); - if (arg_transpose->get_output_partial_shape(0).is_static()) { - auto arg_transpose_order = as_type_ptr(arg_transpose->input_value(1).get_node_shared_ptr()); - auto order = arg_transpose_order->get_axis_vector_val(); - // we need the correct input shape to produce the right output shape - // we are going to create a label of the right input shape, - // so a new pad will have the right shape - auto def_order = permutation_to_default_order(order); - - auto input_shape = apply_permutation(arg_transpose->get_shape(), def_order); - - auto dummy_correct_shape = - make_shared(arg_transpose->get_element_type(), input_shape); - - auto pad_begin = apply_permutation(n->get_pads_begin(), def_order); - auto pad_end = apply_permutation(n->get_pads_end(), def_order); - - auto new_begin = make_shared(element::i64, Shape{pad_begin.size()}, pad_begin); - auto new_end = make_shared(element::i64, Shape{pad_end.size()}, pad_end); - auto new_pad = make_shared(dummy_correct_shape, new_begin, new_end, n->input_value(3), n->get_pad_mode()); - replace_node(dummy_correct_shape, n->input_value(0).get_node_shared_ptr()); - OPENVINO_DEBUG << "Replacing " << n->get_name() << " with " << new_pad->get_name(); - replace_node(n, new_pad); - auto new_transpose = make_transpose(new_pad, order); - OPENVINO_DEBUG << "Propagating " << describe(new_transpose) << " for " << n->get_name(); - write_transposemap(reorders, new_pad, new_transpose); - } else { - OPENVINO_DEBUG << "TransposeSinking failed for Pad op " << n->get_name() - << " . Output shape of Transpose op must be static."; - return false; - } - return true; -} - -static bool sink_concat(const shared_ptr& n, - TransposeMap& reorders, - set>& transposes_to_delete) { - auto n_in = n->input_value(0); - auto arg_transpose = read_transposemap(reorders, n_in); - if (arg_transpose->get_output_partial_shape(0).is_static()) { - auto arg_transpose_order = as_type_ptr(arg_transpose->input_value(1).get_node_shared_ptr()); - auto order = arg_transpose_order->get_axis_vector_val(); - // we need the correct input shape to produce the right output shape - // we are going to create a label of the right input shape, - // so a new concat will have the right shape - auto def_order = permutation_to_default_order(order); - - auto input_shape = apply_permutation(arg_transpose->get_shape(), def_order); - - auto dummy_correct_shape = - make_shared(arg_transpose->get_element_type(), input_shape); - - NodeVector new_args; - new_args.push_back(dummy_correct_shape); - - for (size_t i = 1; i < n->get_input_size(); i++) { - auto iarg = n->input_value(i); - auto iarg_transpose = read_transposemap(reorders, iarg); - auto iarg_transpose_order = as_type_ptr(iarg_transpose->input_value(1).get_node_shared_ptr()); - auto iorder = iarg_transpose_order->get_axis_vector_val(); - if (iorder != order) { - OPENVINO_DEBUG << " input order at " << i << "-th arg is different from first arg"; - return false; - } - - if (iarg_transpose->get_output_partial_shape(0).is_dynamic()) { - OPENVINO_DEBUG << "TransposeSinking failed for Concat op " << n->get_name() - << " . Input Transpose ops" - " must have static shapes. "; - return false; - } - auto iinput_shape = apply_permutation(iarg_transpose->get_shape(), def_order); - - auto idummy_correct_shape = - make_shared(iarg_transpose->get_element_type(), iinput_shape); - new_args.push_back(idummy_correct_shape); - } - - auto new_axis = order.at(n->get_concatenation_axis()); - auto new_concat = make_shared(new_args, new_axis); - // put back the original arguments - for (size_t i = 0; i < new_concat->get_input_size(); i++) { - OPENVINO_DEBUG << "Replacing " << new_concat->get_name() << " input " << i << " with " << n->get_name() - << " input " << i; - new_concat->input(i).replace_source_output(n->input_value(i)); - } - OPENVINO_DEBUG << "Replacing " << n->get_name() << " with " << new_concat->get_name(); - replace_node(n, new_concat); - auto new_transpose = make_transpose(new_concat, order); - OPENVINO_DEBUG << "Propagating " << describe(new_transpose) << " for " << n->get_name(); - write_transposemap(reorders, new_concat, new_transpose); - } else { - OPENVINO_DEBUG << "TransposeSinking failed for Concat op " << n->get_name() - << " . Output shape of Transpose op must be static."; - return false; - } - return true; -} - -static bool sink_prelu(const shared_ptr& prelu, - TransposeMap& reorders, - set>& transposes_to_delete) { - FRONT_END_GENERAL_CHECK(prelu, "Null pointer is given to PRelu node."); - FRONT_END_GENERAL_CHECK(prelu->get_input_size() > 1, "The PRelu node must contain at least two inputs."); - auto slope_shape = prelu->input_value(1).get_partial_shape(); - if (slope_shape.is_static() && shape_size(slope_shape.to_shape()) == 1) { - // handle a case covering LeakyRelu decomposition - auto arg_transpose = read_transposemap(reorders, prelu->input_value(0)); - OPENVINO_DEBUG << "Propagating " << describe(arg_transpose) << " for " << prelu->get_name(); - write_transposemap(reorders, prelu, arg_transpose); - } else { - return false; - } - return true; -} - -void purge_transposes(const set>& transposes_to_delete) { - for (const auto& r : transposes_to_delete) { - delete_transpose(r); - } -} - -// The goal of TransposeSinking is to remove -// round-trip transposes(i.e. nhwc->nchw(nchw-only-op)->nhwc) -// around nchw-only-op (e.g.Convolution, Batchnorm, Avg/MaxPool) -// This is achieved by both **sinking**, propagating transposes -// through ops towards op::Results, -// or **swimming** Transposes up towards op::Parameter -// For each op type we support we can either combine -// two transposes by replacing the existing Transpose, -// materialize pending transposes if they can't be propagated through op -bool ov::frontend::tensorflow::pass::TransposeSinking::run_on_model(const shared_ptr& f) { - TransposeMap reorders; - set> transposes_to_delete; - unordered_map orig_result_out_shape; - - // STEP 1 : Sink or Swim transposes away for op clusters - try { - for (const auto& n : f->get_ordered_ops()) { - OPENVINO_DEBUG << "Processing " << n->get_name(); - // collect output shape of all Result nodes for a sanity check - if (ov::op::util::is_output(n)) { - orig_result_out_shape[n->get_name()] = n->get_output_partial_shape(0); - } - - bool sink_res = false; - if (auto transpose = as_type_ptr(n)) { - sink_res = sink_transpose(transpose, reorders, transposes_to_delete); - } else if (ov::op::util::is_unary_elementwise_arithmetic(n) || as_type_ptr(n) || - as_type_ptr(n) || as_type_ptr(n) || as_type_ptr(n)) { - // Some unary operations are inherrited from Op class - // so we need explicitly to check them - sink_res = sink_unary(n, reorders, transposes_to_delete); - } else if (ov::op::util::is_binary_elementwise_arithmetic(n)) { - sink_res = sink_binary(n, reorders, transposes_to_delete); - } else if (auto pad = as_type_ptr(n)) { - sink_res = sink_pad(pad, reorders, transposes_to_delete); - } else if (auto concat = as_type_ptr(n)) { - sink_res = sink_concat(concat, reorders, transposes_to_delete); - } else if (auto prelu = as_type_ptr(n)) { - sink_res = sink_prelu(prelu, reorders, transposes_to_delete); - } - - if (!sink_res) { - materialize_shapes(n, reorders, transposes_to_delete); - } - } - } catch (...) { - OPENVINO_DEBUG << "Caught exception while sinking op"; - purge_transposes(transposes_to_delete); - return false; - } - - // STEP 2: purge all the transposes we either sunk or swam. - OPENVINO_DEBUG << "Purging transposes "; - purge_transposes(transposes_to_delete); - - // STEP 3: fix wrong shape info wholesale - OPENVINO_DEBUG << "Fixing wrong shape info for the whole graph"; - for (const auto& n : f->get_ordered_ops()) { - n->revalidate_and_infer_types(); - } - - const ResultVector& results = f->get_results(); - for (const auto& r : results) { - // make sure shapes are always materialized before results - FRONT_END_GENERAL_CHECK(r->get_output_partial_shape(0) == r->get_input_partial_shape(0) && - r->get_element_type() == r->input_value(0).get_element_type(), - " op::Result = ", - *r, - ", Arg = ", - r->input_value(0).get_node()); - - // make sure that after TransposeSinking pass the output_shape for Result - // does not change from the expected output_shape before the pass - FRONT_END_GENERAL_CHECK(r->get_output_partial_shape(0) == orig_result_out_shape[r->get_name()], - " op::Result = ", - *r, - " expected output shape = ", - orig_result_out_shape[r->get_name()]); - } - return true; -} diff --git a/src/frontends/tensorflow_common/src/utils.cpp b/src/frontends/tensorflow_common/src/utils.cpp index 13eded0e5d9af9..05aca5ad9ba6c6 100644 --- a/src/frontends/tensorflow_common/src/utils.cpp +++ b/src/frontends/tensorflow_common/src/utils.cpp @@ -6,65 +6,66 @@ #include +#include "common_op_table.hpp" #include "openvino/opsets/opset10.hpp" -#include "openvino/opsets/opset8.hpp" -#include "openvino_conversions.hpp" using namespace ov; using namespace ov::op; using namespace ov::opset10; -using namespace ov::opset8; using namespace std; using namespace ov::frontend::tensorflow; -void ov::frontend::tensorflow::set_node_name(const std::string& node_name, const std::shared_ptr& node) { +namespace ov { +namespace frontend { +namespace tensorflow { + +void set_node_name(const string& node_name, const shared_ptr& node) { const auto& outputs = node->outputs(); node->set_friendly_name(node_name); if (outputs.size() == 1) { set_out_name(node_name, outputs[0]); } for (size_t idx = 0; idx < outputs.size(); ++idx) { - set_out_name({node_name + ":" + std::to_string(idx)}, outputs[idx]); + set_out_name({node_name + ":" + to_string(idx)}, outputs[idx]); } } -void ov::frontend::tensorflow::set_out_name(const std::string& out_name, const ov::Output& output) { +void set_out_name(const string& out_name, const Output& output) { output.get_tensor().add_names({out_name}); } -ov::op::PadType ov::frontend::tensorflow::convert_tf_padding(const ov::frontend::NodeContext& node, - const std::string& tf_padding) { - std::set supported_ops = {"Conv2D", - "Conv2DBackpropInput", - "Conv3D", - "Conv3DBackpropInputV2", - "MaxPool", - "MaxPoolV2", - "MaxPool3D", - "ExtractImagePatches", - "DepthwiseConv2dNative", - "AvgPool", - "AvgPool3D"}; +PadType convert_tf_padding(const frontend::NodeContext& node, const string& tf_padding) { + set supported_ops = {"Conv2D", + "Conv2DBackpropInput", + "Conv3D", + "Conv3DBackpropInputV2", + "MaxPool", + "MaxPoolV2", + "MaxPool3D", + "ExtractImagePatches", + "DepthwiseConv2dNative", + "AvgPool", + "AvgPool3D"}; auto op_type = node.get_op_type(); TENSORFLOW_OP_VALIDATION( node, supported_ops.count(op_type), "OpenVINO TensorFlow Frontend does not support conversion of padding type for " + op_type + " operation."); - std::set supported_modes = {"VALID", "SAME", "EXPLICIT"}; + set supported_modes = {"VALID", "SAME", "EXPLICIT"}; TENSORFLOW_OP_VALIDATION(node, supported_modes.count(tf_padding), "OpenVINO TensorFlow Frontend does not support " + tf_padding + " padding mode."); if (tf_padding == "VALID") { - return ov::op::PadType::VALID; + return PadType::VALID; } if (op_type == "Conv2DBackpropInput" || op_type == "Conv3DBackpropInputV2") { if (tf_padding == "SAME") { // According to the formulas for calculating auto_pad values of the // ConvBackpropData layer in the Operation specification, // the SAME_LOWER value matches to the SAME value in TensorFlow - return ov::op::PadType::SAME_LOWER; + return PadType::SAME_LOWER; } } else if (op_type == "Conv2D" || op_type == "Conv3D" || op_type == "MaxPool" || op_type == "MaxPoolV2" || op_type == "MaxPool3D" || op_type == "ExtractImagePatches" || op_type == "DepthwiseConv2dNative" || @@ -73,20 +74,20 @@ ov::op::PadType ov::frontend::tensorflow::convert_tf_padding(const ov::frontend: // According to the formulas for calculating auto_pad values of the // Conv layer in the Operation specification, // the SAME_UPPER value matches to the SAME value in TensorFlow - return ov::op::PadType::SAME_UPPER; + return PadType::SAME_UPPER; } } - return ov::op::PadType::EXPLICIT; + return PadType::EXPLICIT; } -void ov::frontend::tensorflow::fill_explicit_pads_vectors(const ov::frontend::NodeContext& node, - bool is_nhwc, - size_t spatial_dims_num, - const std::vector& tf_explicit_paddings, - ov::CoordinateDiff& pads_begin, - ov::CoordinateDiff& pads_end) { - auto fullfill_pads = [&](ov::CoordinateDiff& pads, const std::vector& indexes) { +void fill_explicit_pads_vectors(const frontend::NodeContext& node, + bool is_nhwc, + size_t spatial_dims_num, + const vector& tf_explicit_paddings, + CoordinateDiff& pads_begin, + CoordinateDiff& pads_end) { + auto fullfill_pads = [&](CoordinateDiff& pads, const vector& indexes) { pads.resize(indexes.size()); for (size_t i = 0; i < indexes.size(); ++i) { pads[i] = tf_explicit_paddings[indexes[i]]; @@ -128,8 +129,7 @@ void ov::frontend::tensorflow::fill_explicit_pads_vectors(const ov::frontend::No } } -ov::OutputVector ov::frontend::tensorflow::translate_convolution_op(const ov::frontend::NodeContext& node, - size_t spatial_dims_num) { +OutputVector translate_convolution_op(const frontend::NodeContext& node, size_t spatial_dims_num) { TENSORFLOW_OP_VALIDATION(node, spatial_dims_num == 2 || spatial_dims_num == 3, "Conv2D or Conv3D are supported only."); @@ -138,20 +138,20 @@ ov::OutputVector ov::frontend::tensorflow::translate_convolution_op(const ov::fr auto filter = node.get_input(1); // retrieve attributes for Conv2D - auto tf_strides = node.get_attribute>("strides"); - auto tf_padding_type = node.get_attribute("padding"); - ov::op::PadType auto_pad = convert_tf_padding(node, tf_padding_type); + auto tf_strides = node.get_attribute>("strides"); + auto tf_padding_type = node.get_attribute("padding"); + PadType auto_pad = convert_tf_padding(node, tf_padding_type); // retrieve optional attributes - auto tf_data_format = node.get_attribute("data_format", spatial_dims_num == 2 ? "NHWC" : "NDHWC"); - auto tf_explicit_paddings = std::vector{}; - if (auto_pad == ov::op::PadType::EXPLICIT) { - tf_explicit_paddings = node.get_attribute>("explicit_paddings", {}); + auto tf_data_format = node.get_attribute("data_format", spatial_dims_num == 2 ? "NHWC" : "NDHWC"); + auto tf_explicit_paddings = vector{}; + if (auto_pad == PadType::EXPLICIT) { + tf_explicit_paddings = node.get_attribute>("explicit_paddings", {}); } - std::vector dilation_2d = {1, 1, 1, 1}; - std::vector dilation_3d = {1, 1, 1, 1, 1}; + vector dilation_2d = {1, 1, 1, 1}; + vector dilation_3d = {1, 1, 1, 1, 1}; auto tf_dilations = - node.get_attribute>("dilations", spatial_dims_num == 2 ? dilation_2d : dilation_3d); + node.get_attribute>("dilations", spatial_dims_num == 2 ? dilation_2d : dilation_3d); bool is_nhwc = true; if (spatial_dims_num == 2) { @@ -167,22 +167,22 @@ ov::OutputVector ov::frontend::tensorflow::translate_convolution_op(const ov::fr } // prepare attributes for OpenVINO Convolution operation - ov::Strides strides(spatial_dims_num); - ov::Strides dilations(spatial_dims_num); - ov::frontend::tensorflow::convert_nhwc_to_hw(is_nhwc, tf_strides, strides); - ov::frontend::tensorflow::convert_nhwc_to_hw(is_nhwc, tf_dilations, dilations); - - ov::CoordinateDiff pads_begin; - ov::CoordinateDiff pads_end; - if (auto_pad == ov::op::PadType::EXPLICIT) { + Strides strides(spatial_dims_num); + Strides dilations(spatial_dims_num); + convert_nhwc_to_hw(is_nhwc, tf_strides, strides); + convert_nhwc_to_hw(is_nhwc, tf_dilations, dilations); + + CoordinateDiff pads_begin; + CoordinateDiff pads_end; + if (auto_pad == PadType::EXPLICIT) { fill_explicit_pads_vectors(node, is_nhwc, spatial_dims_num, tf_explicit_paddings, pads_begin, pads_end); } // prepare inputs to Convolution - ov::frontend::tensorflow::convert_nhwc_to_nchw(is_nhwc, input, ov::Rank(spatial_dims_num + 2)); - ov::AxisVector permutation_2d = {3, 2, 0, 1}; - ov::AxisVector permutation_3d = {4, 3, 0, 1, 2}; - filter = ov::frontend::tensorflow::make_transpose(filter, spatial_dims_num == 2 ? permutation_2d : permutation_3d); + convert_nhwc_to_nchw(is_nhwc, input, Rank(spatial_dims_num + 2)); + AxisVector permutation_2d = {3, 2, 0, 1}; + AxisVector permutation_3d = {4, 3, 0, 1, 2}; + filter = make_transpose(filter, spatial_dims_num == 2 ? permutation_2d : permutation_3d); bool input_channels_static = false; int64_t num_groups = 1; @@ -206,11 +206,11 @@ ov::OutputVector ov::frontend::tensorflow::translate_convolution_op(const ov::fr } } - ov::Output conv; + Output conv; if (input_channels_static && num_groups == 1) { // regular convolutional operation // we assume that input channel size will not be changed if they are already static - conv = std::make_shared(input, filter, strides, pads_begin, pads_end, dilations, auto_pad); + conv = make_shared(input, filter, strides, pads_begin, pads_end, dilations, auto_pad); } else { // grouped convolutional operation // compute input channels given from the input and the filter @@ -225,59 +225,55 @@ ov::OutputVector ov::frontend::tensorflow::translate_convolution_op(const ov::fr auto num_groups = make_shared(input_cin, filter_cin); // reshape the filter based on the number of groups information - auto int_max_const = make_shared(element::i32, Shape{1}, std::numeric_limits::max()); + auto int_max_const = make_shared(element::i32, Shape{1}, numeric_limits::max()); auto filter_cout = make_shared(filter_shape, zero_const, one_const, one_const); auto filter_new_cout = make_shared(filter_cout, num_groups); auto shape_cin_xy = make_shared(filter_shape, one_const, int_max_const, one_const); auto filter_new_shape = make_shared(OutputVector{num_groups, filter_new_cout, shape_cin_xy}, 0); auto new_filter = make_shared(filter, filter_new_shape, false); - conv = - std::make_shared(input, new_filter, strides, pads_begin, pads_end, dilations, auto_pad); + conv = make_shared(input, new_filter, strides, pads_begin, pads_end, dilations, auto_pad); } - ov::frontend::tensorflow::convert_nchw_to_nhwc(is_nhwc, conv, ov::Rank(spatial_dims_num + 2)); - ov::frontend::tensorflow::set_node_name(node.get_name(), conv.get_node_shared_ptr()); + convert_nchw_to_nhwc(is_nhwc, conv, Rank(spatial_dims_num + 2)); + set_node_name(node.get_name(), conv.get_node_shared_ptr()); return {conv}; } -void ov::frontend::tensorflow::default_op_checks(const ov::frontend::NodeContext& node, - size_t min_input_size, - const std::vector& supported_ops) { +void default_op_checks(const frontend::NodeContext& node, size_t min_input_size, const vector& supported_ops) { auto op_type = node.get_op_type(); TENSORFLOW_OP_VALIDATION(node, - std::find(supported_ops.begin(), supported_ops.end(), op_type) != supported_ops.end(), + find(supported_ops.begin(), supported_ops.end(), op_type) != supported_ops.end(), op_type + " is not supported for conversion."); TENSORFLOW_OP_VALIDATION(node, node.get_input_size() >= min_input_size, - op_type + " must have at least " + std::to_string(min_input_size) + " inputs."); + op_type + " must have at least " + to_string(min_input_size) + " inputs."); } -bool ov::frontend::tensorflow::is_conditional_edge(const std::string& input_tensor_name) { +bool is_conditional_edge(const string& input_tensor_name) { return input_tensor_name.length() > 0 && input_tensor_name[0] == '^'; } -ov::Output ov::frontend::tensorflow::get_elements_number_1d(const ov::Output& output, - ov::element::Type output_type, - ov::pass::NodeRegistry& rg) { +Output get_elements_number_1d(const Output& output, element::Type output_type, pass::NodeRegistry& rg) { auto output_rank = output.get_partial_shape().rank(); if (output_rank.is_static() && output_rank.get_length() != 1) { FRONT_END_OP_CONVERSION_CHECK(false, "Internal error: get_elements_number_1d method supports only 1D input tensor."); } auto shape = rg.make(output, output_type); - auto num_elements = rg.make(shape); + auto const_zero = make_shared(element::i32, Shape{}, 0); + auto num_elements = rg.make(shape, const_zero); return num_elements; } -PadMode ov::frontend::tensorflow::convert_padding_mode(const NodeContext& node, const std::string& padding_mode) { - std::set supported_ops = {"MirrorPad"}; +PadMode convert_padding_mode(const NodeContext& node, const string& padding_mode) { + set supported_ops = {"MirrorPad"}; auto op_type = node.get_op_type(); TENSORFLOW_OP_VALIDATION( node, supported_ops.count(op_type), "OpenVINO TensorFlow Frontend does not support conversion of padding mode for " + op_type + " operation."); - std::set supported_modes = {"REFLECT", "SYMMETRIC"}; + set supported_modes = {"REFLECT", "SYMMETRIC"}; TENSORFLOW_OP_VALIDATION(node, supported_modes.count(padding_mode), "OpenVINO TensorFlow Frontend does not support " + padding_mode + " padding mode."); @@ -291,14 +287,65 @@ PadMode ov::frontend::tensorflow::convert_padding_mode(const NodeContext& node, return PadMode::REFLECT; } -Output ov::frontend::tensorflow::compute_subgraph_scalar_rank(const Output& output, - element::Type output_type, - bool as_scalar) { - auto shape_of = make_shared(output, output_type); - auto rank_of = make_shared(shape_of, output_type); +Output compute_subgraph_scalar_rank(const Output& output, element::Type output_type, bool as_scalar) { + auto shape_of = make_shared(output, output_type); + auto rank_of = make_shared(shape_of, output_type); if (as_scalar) { - return make_shared(rank_of); + auto const_zero = make_shared(element::i32, Shape{}, 0); + return make_shared(rank_of, const_zero); } return rank_of; } + +void convert_nhwc_to_nchw(bool need_convert, Output& node, Rank input_rank) { + if (need_convert) { + if (input_rank.is_dynamic()) { + // TODO: use ShapeOf sub-graph to generate permutation vector + OPENVINO_ASSERT(node.get_partial_shape().rank().is_static(), + "For conversion into the first channel format, the input rank must be static or determined " + "based on the operation."); + input_rank = node.get_partial_shape().rank(); + } + auto rank_value = input_rank.get_length(); + if (rank_value == 4) { + node = make_transpose(node, {0, 3, 1, 2}); + } else if (rank_value == 5) { + node = make_transpose(node, {0, 4, 1, 2, 3}); + } + } +} + +void convert_nchw_to_nhwc(bool need_convert, Output& node, Rank input_rank) { + if (need_convert) { + if (input_rank.is_dynamic()) { + // TODO: use ShapeOf sub-graph to generate permutation vector + OPENVINO_ASSERT(node.get_partial_shape().rank().is_static(), + "For conversion into the last channel format, the input rank must be static or determined " + "based on the operation."); + input_rank = node.get_partial_shape().rank(); + } + auto rank_value = input_rank.get_length(); + if (rank_value == 4) { + node = make_transpose(node, {0, 2, 3, 1}); + } else if (rank_value == 5) { + node = make_transpose(node, {0, 2, 3, 4, 1}); + } + } +} + +shared_ptr make_transpose(const Output& arg, const AxisVector& input_order) { + auto order = make_shared(element::i64, Shape{input_order.size()}, input_order); + auto transpose = make_shared(arg, order); + return transpose; +} + +shared_ptr make_reshape(const Output& arg, const vector& new_shape) { + auto new_shape_node = make_shared(element::i64, Shape{new_shape.size()}, new_shape); + auto reshape = make_shared(arg, new_shape_node, true); + return reshape; +} + +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow_common/tests/transpose_sinking.cpp b/src/frontends/tensorflow_common/tests/transpose_sinking.cpp deleted file mode 100644 index 71c5142f09531e..00000000000000 --- a/src/frontends/tensorflow_common/tests/transpose_sinking.cpp +++ /dev/null @@ -1,645 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "pass/transpose_sinking.hpp" - -#include -#include -#include -#include -#include - -#include "gtest/gtest.h" - -using namespace std; -using namespace ov; -using namespace opset8; -using namespace frontend::tensorflow::pass; - -template -int64_t count_ops_of_type(const shared_ptr& f) { - int64_t cnt = 0; - for (const auto& op : f->get_ops()) { - cnt += dynamic_pointer_cast(op) != nullptr; - } - return cnt; -} - -TEST(TransposeSinkingTest, DynamicShape) { - ov::PartialShape shape_nhwc(vector(4, Dimension::dynamic())); - auto a = make_shared(ngraph::element::i32, shape_nhwc); - auto ng_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose = make_shared(a, ng_order); - auto absn = make_shared(transpose); - auto absn2 = make_shared(absn); - absn2->output(0).set_names({"out_name"}); - auto res = make_shared(absn2); - auto func = make_shared(ngraph::OutputVector{res}, ngraph::ParameterVector{a}); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - EXPECT_EQ(new_transpose->output(0).get_names(), std::unordered_set({"out_name"})); -} - -TEST(TransposeSinkingTest, TensorNames) { - ngraph::Shape shape_nhwc{16, 28, 28, 1}; - auto a = make_shared(ngraph::element::i32, shape_nhwc); - auto ng_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose = make_shared(a, ng_order); - auto absn = make_shared(transpose); - auto absn2 = make_shared(absn); - absn2->output(0).set_names({"out_name"}); - auto res = make_shared(absn2); - auto func = make_shared(ngraph::OutputVector{res}, ngraph::ParameterVector{a}); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - EXPECT_EQ(new_transpose->output(0).get_names(), std::unordered_set({"out_name"})); -} - -TEST(TransposeSinkingTest, TensorNamesCombineTransposes) { - ngraph::Shape shape_nhwc{16, 28, 28, 1}; - auto a = make_shared(ngraph::element::i32, shape_nhwc); - auto ng_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose_1 = make_shared(a, ng_order); - auto transpose_2 = make_shared(transpose_1, ng_order); - transpose_2->output(0).set_names({"out_name"}); - auto res = make_shared(transpose_2); - auto func = make_shared(ngraph::OutputVector{res}, ngraph::ParameterVector{a}); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - EXPECT_EQ(new_transpose->output(0).get_names(), std::unordered_set({"out_name"})); - size_t transpose_cnt = count_ops_of_type(func); - EXPECT_EQ(transpose_cnt, 1); -} - -TEST(TransposeSinkingTest, EdgeSplitting) { - // checks if Transpose is pushed through Abs, but stopped by - // ReduceSum - ngraph::Shape shape_nhwc{16, 28, 28, 1}; - ngraph::Shape shape_nchw{16, 1, 28, 28}; - - auto a = make_shared(ngraph::element::i32, shape_nhwc); - auto ng_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose = make_shared(a, ng_order); - auto absn = make_shared(transpose); - auto absn2 = make_shared(absn); - - auto axes = make_shared(ngraph::element::i64, ngraph::Shape{4}, vector{0, 1, 2, 3}); - auto sum = make_shared(transpose, axes, true); - - auto func = make_shared(ngraph::OutputVector{absn2, sum}, ngraph::ParameterVector{a}); - size_t before_count = count_ops_of_type(func); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - ASSERT_EQ(before_count, 1); - size_t after_count = count_ops_of_type(func); - ASSERT_EQ(after_count, 2); - ASSERT_EQ(func->get_results().at(1)->input_value(0), sum); - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - ASSERT_EQ(new_transpose->get_output_shape(0), shape_nchw); -} - -// X (NHWC) -// | -// Transpose -// | -// AvgPool (NCHW) -// | -// Transpose -// | Const (NHWC) -// | / -// | / -// | / -// Add (NHWC) -// | -// Result -TEST(TransposeSinkingTest, PoolAdd1) { - ngraph::Shape input_shape{1, 3, 3, 1}; // NHWC (N=1, H=3, W=3, C=1) - - auto input_type = ngraph::element::f32; - - auto X = make_shared(input_type, input_shape); // NHWC - - auto ng_order1 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose1 = make_shared(X, ng_order1); // NCHW (1,1,3,3) - - auto avgpool = make_shared(transpose1, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - true, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - - auto ng_order2 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose2 = make_shared(avgpool, ng_order2); // NHWC (1,3,3,1) - - auto const1 = Constant::create(input_type, ngraph::Shape{1, 3, 3, 1}, {3}); // NHWC (1,3,3,1) - auto add1 = make_shared(transpose2, const1); - auto func = make_shared(add1, ngraph::ParameterVector{X}); - - ov::pass::Manager pass_manager; - size_t before_count = count_ops_of_type(func); - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); - ASSERT_LE(before_count, after_count); - ASSERT_EQ(3, after_count); - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - ASSERT_EQ(new_transpose->get_output_shape(0), (ngraph::Shape{1, 3, 3, 1})); -} - -TEST(TransposeSinkingTest, PoolAdd2) { - ngraph::Shape input_shape{1, 3, 3, 1}; // NHWC (N=1, H=3, W=3, C=1) - - auto input_type = ngraph::element::f32; - - auto X = make_shared(input_type, input_shape); // NHWC - - auto ng_order1 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose1 = make_shared(X, ng_order1); // NCHW (1,1,3,3) - - auto avgpool = make_shared(transpose1, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - true, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - - auto ng_order2 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose2 = make_shared(avgpool, ng_order2); // NHWC (1,3,3,1) - auto maxpool = make_shared(transpose1, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - - auto ng_order3 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose3 = make_shared(maxpool, ng_order3); - - auto const1 = Constant::create(input_type, ngraph::Shape{1, 3, 3, 1}, {3}); // NHWC (1,3,3,1) - auto add1 = make_shared(transpose3, const1); - auto add2 = make_shared(add1, transpose2); - auto func = make_shared(add2, ngraph::ParameterVector{X}); - - ov::pass::Manager pass_manager; - size_t before_count = count_ops_of_type(func); // 3 - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); // 4 - ASSERT_LE(before_count, after_count); - ASSERT_EQ(4, after_count); - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - ASSERT_EQ(new_transpose->get_output_shape(0), (ngraph::Shape{1, 3, 3, 1})); -} - -// Different rank constant input to Add1. After TransposeSinking the const -// would need a Reshape to have the same order as the other input to -// Add1. -TEST(TransposeSinkingTest, PoolAdd3) { - ngraph::Shape input_shape{1, 3, 3, 1}; // NHWC (N=1, H=3, W=3, C=1) - - auto input_type = ngraph::element::f32; - - auto X = make_shared(input_type, input_shape); // NHWC - - auto ng_order1 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose1 = make_shared(X, ng_order1); // NCHW (1,1,3,3) - - auto avgpool = make_shared(transpose1, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - true, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - - auto ng_order2 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose2 = make_shared(avgpool, ng_order2); // NHWC (1,3,3,1) - - auto const1 = Constant::create(input_type, ngraph::Shape{1}, {1}); // NHWC (1,3,3,1) - auto add1 = make_shared(transpose2, const1); - auto func = make_shared(add1, ngraph::ParameterVector{X}); - - ov::pass::Manager pass_manager; - size_t before_count = count_ops_of_type(func); - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); - ASSERT_LE(after_count, before_count); - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - ASSERT_EQ(new_transpose->get_output_shape(0), (ngraph::Shape{1, 3, 3, 1})); -} - -TEST(TransposeSinkingTest, Concat) { - // checks if Transpose is pushed through Concat - ngraph::Shape shape_nhwc{16, 28, 28, 1}; - auto a = make_shared(ngraph::element::i32, shape_nhwc); - auto b = make_shared(ngraph::element::i32, shape_nhwc); - auto to_nchw = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto a_transpose = make_shared(a, to_nchw); - auto b_transpose = make_shared(b, to_nchw); - auto concat = make_shared(ngraph::OutputVector{a_transpose, b_transpose}, 0); - auto to_nhwc = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto c = make_shared(concat, to_nhwc); - auto func = make_shared(ngraph::OutputVector{c}, ngraph::ParameterVector{a, b}); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t transpose_count = count_ops_of_type(func); - ASSERT_EQ(0, transpose_count); - auto result = func->get_results().at(0)->input_value(0).get_node_shared_ptr(); - ngraph::Shape expected_shape{32, 28, 28, 1}; - ASSERT_EQ(result->get_output_shape(0), expected_shape); -} - -TEST(TransposeSinkingTest, Concat_DummyShape) { - // checks if Transpose is pushed through Concat - ngraph::Shape shape1{4, 3, 3, 1}; - ngraph::Shape shape2{4, 3, 3, 2}; - ngraph::Shape shape3{4, 3, 3, 3}; - ngraph::Shape shape4{4, 3, 3, 4}; - auto to_nchw = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto to_nhwc = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - - auto a1 = make_shared(ngraph::element::i32, shape1); - auto a2 = make_shared(ngraph::element::i32, shape2); - auto a3 = make_shared(ngraph::element::i32, shape3); - auto a4 = make_shared(ngraph::element::i32, shape4); - auto a1_transpose = make_shared(a1, to_nchw); - auto a2_transpose = make_shared(a2, to_nchw); - auto a3_transpose = make_shared(a3, to_nchw); - auto a4_transpose = make_shared(a4, to_nchw); - auto concat = make_shared(ngraph::NodeVector{a1_transpose, a2_transpose, a3_transpose, a4_transpose}, 1); - auto out = make_shared(concat, to_nchw); - auto func = make_shared(ngraph::OutputVector{out}, ngraph::ParameterVector{a1, a2, a3, a4}); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t transpose_count = count_ops_of_type(func); // 1 - ASSERT_EQ(1, transpose_count); - auto result = func->get_results().at(0)->input_value(0).get_node_shared_ptr(); - ngraph::Shape expected_shape{4, 3, 10, 3}; - ASSERT_EQ(result->get_output_shape(0), expected_shape); -} - -// The Transpose should sink through Pad op but stopped by ReduceSum -TEST(TransposeSinkingTest, Pad) { - ngraph::Shape shape_nhwc{100, 8, 8, 1}; - - auto a = make_shared(ngraph::element::f32, shape_nhwc); - auto pad_value = Constant::create(ngraph::element::f32, ngraph::Shape{}, std::vector{0.0f}); - - ngraph::CoordinateDiff pad_end{0, 0, 0, 0}; - ngraph::CoordinateDiff pad_begin{0, 1, 1, 0}; - - auto a_to_nchw = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto a_transpose = make_shared(a, a_to_nchw); - - auto maxpool = make_shared(a_transpose, - ngraph::Strides{2, 2}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - - auto m_to_nhwc = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto m_transpose = make_shared(maxpool, m_to_nhwc); - - shared_ptr pads_begin_node, pads_end_node; - pads_begin_node = make_shared(ngraph::element::i64, ngraph::Shape{pad_begin.size()}, pad_begin); - pads_end_node = make_shared(ngraph::element::i64, ngraph::Shape{pad_end.size()}, pad_end); - auto pad = make_shared(m_transpose, pads_begin_node, pads_end_node, pad_value, ngraph::op::PadMode::CONSTANT); - - auto axes = make_shared(ngraph::element::i64, ngraph::Shape{4}, vector{0, 1, 2, 3}); - auto sum = make_shared(pad, axes, true); - - auto func = make_shared(ngraph::OutputVector{sum}, ngraph::ParameterVector{a}); - - ov::pass::Manager pass_manager; - size_t before_count = count_ops_of_type(func); // 2 - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); // 2 - ASSERT_EQ(after_count, before_count); - auto result = func->get_results().at(0)->input_value(0).get_node_shared_ptr(); - ngraph::Shape expected_shape{1, 1, 1, 1}; - ASSERT_EQ(result->get_output_shape(0), expected_shape); - auto out = ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(out); -} - -TEST(TransposeSinkingTest, SimpleUnary) { - ngraph::Shape shape_nhwc{16, 28, 28, 1}; - ngraph::Shape shape_nchw{16, 1, 28, 28}; - auto a = make_shared(ngraph::element::i32, shape_nhwc); - - auto ng_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose = make_shared(a, ng_order); - - auto a_t = make_shared(a, ng_order); - auto absn = make_shared(a_t); - auto absn2 = make_shared(absn); - - auto tf_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto absn2_t = make_shared(absn2, tf_order); - - auto func = make_shared(ngraph::OutputVector{absn2_t}, ngraph::ParameterVector{a}); - size_t before_count = count_ops_of_type(func); // 2 - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); // 0 - ASSERT_EQ(func->get_results().at(0)->input_value(0), absn2); - EXPECT_NE(before_count, after_count); - EXPECT_EQ(after_count, 0); -} - -TEST(TransposeSinkingTest, SinkingThroughPreLUWithScalarSlope) { - auto input = make_shared(ov::element::f32, ov::Shape{1, 105, 30, 30}); - auto transpose_before = - make_shared(input, - make_shared(ov::element::i64, ov::Shape{4}, std::vector{0, 2, 3, 1})); - - auto prelu = make_shared(transpose_before, - make_shared(ov::element::f32, ov::Shape{1}, std::vector{0.8f})); - auto transpose_after = - make_shared(prelu, - make_shared(ov::element::i64, ov::Shape{4}, std::vector{0, 3, 1, 2})); - - auto model = make_shared(ov::OutputVector{transpose_after}, ov::ParameterVector{input}); - size_t before_count = count_ops_of_type(model); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(model); - - size_t after_count = count_ops_of_type(model); - - EXPECT_EQ(before_count, 2); - EXPECT_EQ(after_count, 0); -} - -TEST(TransposeSinkingTest, SinkingThroughPreLUWithNonScalarSlope) { - auto input = make_shared(ov::element::f32, ov::Shape{1, 3, 3, 3}); - auto transpose_before = - make_shared(input, - make_shared(ov::element::i64, ov::Shape{4}, std::vector{0, 2, 3, 1})); - - auto prelu = - make_shared(transpose_before, - make_shared(ov::element::f32, ov::Shape{3}, std::vector{0.8f, 0.7f, 0.1f})); - auto transpose_after = - make_shared(prelu, - make_shared(ov::element::i64, ov::Shape{4}, std::vector{0, 3, 1, 2})); - - auto model = make_shared(ov::OutputVector{transpose_after}, ov::ParameterVector{input}); - size_t before_count = count_ops_of_type(model); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(model); - - size_t after_count = count_ops_of_type(model); - - EXPECT_EQ(before_count, 2); - // Now Transpose Sinking is not applied to Prelu with non-scalar slope - EXPECT_EQ(after_count, 2); -} - -/* X (NCHW) - * | - * Transpose1 - * | - * Split (NHWC) - * / \ - * / \ - * Transpose2 Transpose3 (NCHW) - * | | - * Const | | Const (NCHW) - * \ | | / - * \ | | / - * \ | | / - * Add Add (NCHW) - * \ / - * \ / - * \ / - * Add (NCHW) - * | - * Result (NCHW) - */ -TEST(TransposeSinkingTest, MultiOutput) { - ngraph::Shape shape_nhwc{1, 4, 4, 1}; - ngraph::Shape shape_nchw{1, 1, 4, 6}; - - auto input_type = ngraph::element::f32; - - auto X = make_shared(input_type, shape_nchw); // NCHW - - auto ng_order1 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose1 = make_shared(X, ng_order1); // NHWC (1, 4, 6, 1) - auto ng_split_dim = std::make_shared(ngraph::element::u64, ngraph::Shape{}, 2); - - auto split = make_shared(transpose1, ng_split_dim, 2); // (1, 4, 3, 1) - - auto ng_order2 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose2 = make_shared(split, ng_order2); // (1, 1, 4, 3) NCHW - - auto ng_order3 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose3 = make_shared(split, ng_order3); // (1, 1, 4, 3) NCHW - - auto const1 = Constant::create(input_type, ngraph::Shape{1, 1, 4, 3}, {3}); // NCHW - auto add1 = make_shared(transpose2, const1); - auto const2 = Constant::create(input_type, ngraph::Shape{1, 1, 4, 3}, {3}); // NCHW - auto add2 = make_shared(transpose3, const2); - auto add3 = make_shared(add1, add2); - auto func = make_shared(add3, ngraph::ParameterVector{X}); - - ov::pass::Manager pass_manager; - size_t before_count = count_ops_of_type(func); // 3 - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); // 4 - ASSERT_LE(before_count, after_count); - ASSERT_EQ(4, after_count); - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - ASSERT_EQ(new_transpose->get_output_shape(0), (ngraph::Shape{1, 1, 4, 3})); -} - -/* X (NHWC) - * | - * Transpose (NCHW) - * | - * AvgPool0 - * | - * Transpose0 (NHWC) - * | - * Split (NHWC) - * / \ - * / \ - * Transpose1 Transpose2 (NCHW) - * | | - * AvgPool1 AvgPool2 - * | | - * Transpose3 Transpose4 (NHWC) - * \ / - * \ / - * \ / - * Concat (NHWC) - * Const / - * \ / - * \ / - * \ / - * \ / - * Add (NHWC) - * | - * Result - */ -TEST(TransposeSinkingTest, AlexnetPattern) { - ngraph::Shape shape_nhwc{1, 55, 55, 96}; - ngraph::Shape shape_nchw{1, 96, 55, 55}; - - auto input_type = ngraph::element::f32; - - // X - auto X = make_shared(input_type, shape_nhwc); // NHWC - - // T -> AvgPool0 -> T0 - auto ng_order = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose = make_shared(X, ng_order); // NCHW - auto avgpool0 = make_shared(transpose, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - true, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - auto ng_order0 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose0 = make_shared(avgpool0, ng_order0); // NHWC - - // Split - auto ng_split_dim = std::make_shared(ngraph::element::u64, ngraph::Shape{}, 3); - auto split = make_shared(transpose0, ng_split_dim, 2); // NHWC - - // T1 -> AvgPool1 -> T2 - auto ng_order1 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose1 = make_shared(split, ng_order1); // NCHW - auto avgpool1 = make_shared(transpose1, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - true, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - auto ng_order2 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose2 = make_shared(avgpool1, ng_order2); // NHWC - - // T3 -> AvgPool2 -> T4 - auto ng_order3 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 3, 1, 2}); - auto transpose3 = make_shared(split, ng_order1); // NCHW - auto avgpool2 = make_shared(transpose3, - ngraph::Strides{1, 1}, - ngraph::Shape{0, 0}, - ngraph::Shape{0, 0}, - ngraph::Shape{1, 1}, - true, - ngraph::op::RoundingType::FLOOR, - ngraph::op::PadType::VALID); - auto ng_order4 = std::make_shared(ngraph::element::u64, ngraph::Shape{4}, ngraph::Shape{0, 2, 3, 1}); - auto transpose4 = make_shared(avgpool2, ng_order4); // NHWC - - // Concat - auto concat = make_shared(ngraph::OutputVector{transpose2, transpose4}, 3); // NHWC - - // Add - auto const1 = Constant::create(input_type, ngraph::Shape{96}, {1}); // NCHW - auto add1 = make_shared(concat, const1); - - auto func = make_shared(add1, ngraph::ParameterVector{X}); - - ov::pass::Manager pass_manager; - size_t before_count = count_ops_of_type(func); - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t after_count = count_ops_of_type(func); - ASSERT_LE(after_count, before_count); - ASSERT_EQ(5, after_count); - auto new_transpose = - ngraph::as_type_ptr(func->get_results().at(0)->input_value(0).get_node_shared_ptr()); - ASSERT_TRUE(new_transpose); - ASSERT_EQ(new_transpose->get_output_shape(0), (ngraph::Shape{1, 55, 55, 96})); -} - -Output make_transpose(const Output& input, const vector& order) { - return std::make_shared(input, opset8::Constant::create(element::i64, {order.size()}, order)); -} - -TEST(TransposeSinkingTest, BinarySubTrickyShapes) { - auto a = make_shared(ngraph::element::i32, ngraph::Shape{1, 17}); - auto a_t = make_transpose(a, {0, 1}); - auto b = make_shared(ngraph::element::i32, ngraph::Shape{48, 48, 17}); - auto b_t = make_transpose(b, {0, 1, 2}); - auto binary = make_shared(a, b); - - auto res = make_shared(binary); - auto func = make_shared(ngraph::OutputVector{res}, ngraph::ParameterVector{a, b}); - - ov::pass::Manager pass_manager; - pass_manager.register_pass(); - pass_manager.run_passes(func); - - size_t transpose_cnt = count_ops_of_type(func); - EXPECT_EQ(transpose_cnt, 0); -} \ No newline at end of file diff --git a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp index fe5bc351adacd8..74a7398c28fd34 100644 --- a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp +++ b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp @@ -17,6 +17,7 @@ size_t DecoderFlatBuffer::get_input_size() const { void DecoderFlatBuffer::get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const { const auto inputs = m_node_def->inputs(); FRONT_END_GENERAL_CHECK(inputs->size() > input_port_idx, @@ -33,13 +34,6 @@ void DecoderFlatBuffer::get_input_node(size_t input_port_idx, producer_output_port_index = input_tensor_idx; } -void DecoderFlatBuffer::get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const { - FRONT_END_NOT_IMPLEMENTED("get_input_node method with op_type_by_name map is not implemented for TFL FE."); -} - const std::string& DecoderFlatBuffer::get_op_type() const { return m_type; } diff --git a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h index 92325596b70656..65381fcbad3af6 100644 --- a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h +++ b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h @@ -52,11 +52,8 @@ class DecoderFlatBuffer : public ov::frontend::DecoderBase { void get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const override; - void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const override; std::string get_output_tensor_name(size_t idx) const; element::Type get_output_tensor_type(size_t idx) const; diff --git a/src/frontends/tensorflow_lite/src/decoder_map.hpp b/src/frontends/tensorflow_lite/src/decoder_map.hpp index 99923b89bef639..476d5c788e1f56 100644 --- a/src/frontends/tensorflow_lite/src/decoder_map.hpp +++ b/src/frontends/tensorflow_lite/src/decoder_map.hpp @@ -57,21 +57,9 @@ class DecoderMap : public ov::frontend::DecoderBase { /// \return producer_output_port_index Output port index from which data is generated void get_input_node(size_t input_port_idx, std::string& producer_name, + std::string& producer_output_port_name, size_t& producer_output_port_index) const override { - m_decoder->get_input_node(input_port_idx, producer_name, producer_output_port_index); - } - - /// \brief Get a producer name and its output port index - /// - /// \param input_port_idx Input port index by which data is consumed - /// \param producer_name A producer name - /// \param producer_output_port_index Output port index from which data is generated - /// \param op_type_by_name Map of operation name to their types - void get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index, - const OpTypeByName& op_type_by_name) const override { - FRONT_END_NOT_IMPLEMENTED("get_input_node method with op_type_by_name map is not implemented for TFL FE."); + m_decoder->get_input_node(input_port_idx, producer_name, producer_output_port_name, producer_output_port_index); } /// \brief Get operation type diff --git a/src/frontends/tensorflow_lite/src/frontend.cpp b/src/frontends/tensorflow_lite/src/frontend.cpp index b708d711ec9f76..52931f8986104a 100644 --- a/src/frontends/tensorflow_lite/src/frontend.cpp +++ b/src/frontends/tensorflow_lite/src/frontend.cpp @@ -10,7 +10,6 @@ #include "op_table.hpp" #include "openvino/frontend/tensorflow_lite/extension/op.hpp" #include "openvino/util/common_util.hpp" -#include "pass/transpose_sinking.hpp" #include "so_extension.hpp" #include "tensor_lite_place.hpp" #include "tf_framework_node.hpp" @@ -48,7 +47,9 @@ FrontEnd::FrontEnd() { /// \brief Check if FrontEndTensorflowLite can recognize model from given parts bool FrontEnd::supported_impl(const std::vector& variants) const { - if (variants.size() != 1) + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + if (variants.size() != 1 + extra_variants_num) return false; if (variants[0].is()) { @@ -71,7 +72,9 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { } ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { - if (variants.size() == 1) { + // Last boolean flag in `variants` (if presented) is reserved for FE configuration + size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is() ? 1 : 0; + if (variants.size() == 1 + extra_variants_num) { if (variants[0].is()) { std::string suffix = ".tflite"; std::string model_path = variants[0].as(); @@ -268,7 +271,7 @@ void FrontEnd::normalize(const std::shared_ptr& function) const { manager.register_pass(); manager.register_pass(); manager.register_pass(); - manager.register_pass(); + manager.register_pass(); manager.run_passes(function); } diff --git a/src/frontends/tensorflow_lite/src/op/op_translation_utils.cpp b/src/frontends/tensorflow_lite/src/op/op_translation_utils.cpp index b849c06f2594c6..7c51b45e7c9f5b 100644 --- a/src/frontends/tensorflow_lite/src/op/op_translation_utils.cpp +++ b/src/frontends/tensorflow_lite/src/op/op_translation_utils.cpp @@ -10,7 +10,6 @@ #include "openvino/core/node_vector.hpp" #include "openvino/frontend/tensorflow_lite/node_context.hpp" -#include "openvino_conversions.hpp" #include "utils.hpp" namespace ov { @@ -132,7 +131,7 @@ std::shared_ptr get_pool_decoder_map( OutputVector attribute_helper(const ov::frontend::tensorflow_lite::NodeContext& node, const std::map& attrs, - ov::OutputVector (*converter)(const ov::frontend::NodeContext&), + ov::frontend::CreatorFunction converter, std::string new_op_type, bool empty_name, ov::OutputVector inputs) { @@ -153,6 +152,23 @@ OutputVector attribute_helper(const ov::frontend::tensorflow_lite::NodeContext& return outputs; } +OutputVector attribute_helper(const ov::frontend::tensorflow_lite::NodeContext& node, + const std::map& attrs, + ov::frontend::CreatorFunctionNamedAndIndexed converter, + std::string new_op_type, + bool empty_name, + ov::OutputVector inputs) { + return attribute_helper( + node, + attrs, + [&](const ov::frontend::NodeContext& ctx) { + return indexed_from_named(converter(ctx)); + }, + new_op_type, + empty_name, + inputs); +} + std::shared_ptr get_decoder(const ov::frontend::tensorflow_lite::NodeContext& node) { const auto& decoder = std::dynamic_pointer_cast(node.get_decoder()); FRONT_END_GENERAL_CHECK(decoder != nullptr, diff --git a/src/frontends/tensorflow_lite/src/op/op_translation_utils.hpp b/src/frontends/tensorflow_lite/src/op/op_translation_utils.hpp index e3a6f62ffc43d6..16ee477f292de5 100644 --- a/src/frontends/tensorflow_lite/src/op/op_translation_utils.hpp +++ b/src/frontends/tensorflow_lite/src/op/op_translation_utils.hpp @@ -13,7 +13,7 @@ #include "openvino/core/node_vector.hpp" #include "openvino/frontend/tensorflow_lite/node_context.hpp" #include "openvino/opsets/opset10.hpp" -#include "openvino_conversions.hpp" +#include "openvino/opsets/opset8.hpp" #include "utils.hpp" namespace ov { @@ -91,7 +91,14 @@ template OutputVector translate_binary_op_with_activation& attrs, - ov::OutputVector (*converter)(const ov::frontend::NodeContext&), + ov::frontend::CreatorFunction converter, + std::string new_op_type = "", + bool empty_name = false, + ov::OutputVector inputs = {}); + +OutputVector attribute_helper(const ov::frontend::tensorflow_lite::NodeContext& node, + const std::map& attrs, + ov::frontend::CreatorFunctionNamedAndIndexed converter, std::string new_op_type = "", bool empty_name = false, ov::OutputVector inputs = {}); diff --git a/src/frontends/tensorflow_lite/src/op_table.cpp b/src/frontends/tensorflow_lite/src/op_table.cpp index f64005de6b148d..e5cf074034d151 100644 --- a/src/frontends/tensorflow_lite/src/op_table.cpp +++ b/src/frontends/tensorflow_lite/src/op_table.cpp @@ -6,6 +6,7 @@ #include "decoder_map.hpp" #include "openvino/opsets/opset10.hpp" +#include "openvino/opsets/opset8.hpp" #include "utils.hpp" using namespace std; @@ -27,7 +28,7 @@ using namespace ov::frontend::tensorflow::op; auto inputs = node.get_inputs(); \ ov::frontend::tensorflow_lite::dequantize_inputs(inputs); \ auto context = frontend::tensorflow_lite::NodeContext(decoder, inputs); \ - return func(context); \ + return get_indexed_outputs(func(context)); \ } namespace ov { diff --git a/src/frontends/tensorflow_lite/src/op_table.hpp b/src/frontends/tensorflow_lite/src/op_table.hpp index 9b92e69cb293d8..797e02c7722249 100644 --- a/src/frontends/tensorflow_lite/src/op_table.hpp +++ b/src/frontends/tensorflow_lite/src/op_table.hpp @@ -13,7 +13,6 @@ #include "openvino/core/node_vector.hpp" #include "openvino/frontend/tensorflow_lite/node_context.hpp" #include "openvino/opsets/opset10.hpp" -#include "openvino_conversions.hpp" #include "utils.hpp" namespace ov { diff --git a/src/frontends/tensorflow_lite/src/tflite_transformations/tflite_quantize_resolver.cpp b/src/frontends/tensorflow_lite/src/tflite_transformations/tflite_quantize_resolver.cpp index ccd203e84d8a8f..e1aff2366c8fc4 100644 --- a/src/frontends/tensorflow_lite/src/tflite_transformations/tflite_quantize_resolver.cpp +++ b/src/frontends/tensorflow_lite/src/tflite_transformations/tflite_quantize_resolver.cpp @@ -78,10 +78,13 @@ void fuse_zp_to_weights(ov::Output& output, std::vector& zero auto check_in_bounds = [&](ov::Output& value) -> bool { shared_ptr constant; - if (rank == 0) + if (rank == 0) { constant = ov::as_type_ptr(output.get_node_shared_ptr()); - else + } else { + OPENVINO_SUPPRESS_DEPRECATED_START constant = ov::get_constant_from_source(value); + OPENVINO_SUPPRESS_DEPRECATED_END + } if (!constant) return false; auto weight = constant->cast_vector()[0]; @@ -96,7 +99,9 @@ void fuse_zp_to_weights(ov::Output& output, std::vector& zero auto zp_node = ov::opset10::Constant::create(ov::element::i32, zp_shape, zero_point); output = std::make_shared(output, zp_node); output = std::make_shared(output, ov::element::i8); + OPENVINO_SUPPRESS_DEPRECATED_START output = ov::get_constant_from_source(output); // TODO: Check Me + OPENVINO_SUPPRESS_DEPRECATED_END zero_point = {0}; } @@ -176,10 +181,12 @@ pass::TFLQuantizeReplacer::TFLQuantizeReplacer() { output_low = ov::opset10::Constant::create(element::f32, {}, {low}); output_high = ov::opset10::Constant::create(element::f32, {}, {high}); } + OPENVINO_SUPPRESS_DEPRECATED_START input_low = get_constant_from_source(input_low); input_high = get_constant_from_source(input_high); output_low = get_constant_from_source(output_low); output_high = get_constant_from_source(output_high); + OPENVINO_SUPPRESS_DEPRECATED_END output = std::make_shared(output, input_low, input_high, output_low, output_high, levels); if (out_type != element::f32) { diff --git a/src/frontends/tensorflow_lite/src/utils.cpp b/src/frontends/tensorflow_lite/src/utils.cpp index e69b0d056a5a02..b89c7e0f659fda 100644 --- a/src/frontends/tensorflow_lite/src/utils.cpp +++ b/src/frontends/tensorflow_lite/src/utils.cpp @@ -99,4 +99,21 @@ void ov::frontend::tensorflow_lite::dequantize_inputs(OutputVector& deq_inputs) continue; deq_input = std::make_shared(deq_input, element::f32); } -} \ No newline at end of file +} + +namespace ov { +namespace frontend { +namespace tensorflow_lite { +// namespace required by arm compiler to specify template +template <> +OutputVector get_indexed_outputs(const OutputVector& outputs) { + return outputs; +}; + +template <> +OutputVector get_indexed_outputs(const frontend::NamedOutputVector& outputs) { + return indexed_from_named(outputs); +}; +} // namespace tensorflow_lite +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow_lite/src/utils.hpp b/src/frontends/tensorflow_lite/src/utils.hpp index 7d4f3ecdd2003f..640c7683bf1bff 100644 --- a/src/frontends/tensorflow_lite/src/utils.hpp +++ b/src/frontends/tensorflow_lite/src/utils.hpp @@ -23,6 +23,9 @@ std::shared_ptr get_quantization(const tflite::QuantizationPar void apply_quantization(ov::Output& output, ov::element::Type type); void dequantize_inputs(OutputVector& deq_inputs); +template +OutputVector get_indexed_outputs(const T& outputs); + } // namespace tensorflow_lite } // namespace frontend } // namespace ov \ No newline at end of file diff --git a/src/frontends/tensorflow_lite/tests/requirements.txt b/src/frontends/tensorflow_lite/tests/requirements.txt index cf70fd3cf83ba9..85cc64cbcc4f51 100644 --- a/src/frontends/tensorflow_lite/tests/requirements.txt +++ b/src/frontends/tensorflow_lite/tests/requirements.txt @@ -1,2 +1,3 @@ +-c ../../../bindings/python/constraints.txt numpy>=1.16.6,<1.25.0 -tensorflow>=1.15.5,<=2.11.0 +tensorflow diff --git a/src/frontends/tests/CMakeLists.txt b/src/frontends/tests/CMakeLists.txt index eca0bd8f5623c7..d7dd08e195b79b 100644 --- a/src/frontends/tests/CMakeLists.txt +++ b/src/frontends/tests/CMakeLists.txt @@ -2,6 +2,4 @@ # SPDX-License-Identifier: Apache-2.0 # -set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF) - add_subdirectory(frontend/shared) diff --git a/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp b/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp index 8760cce41eae6d..0e3a7e9999880b 100644 --- a/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp +++ b/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp @@ -38,8 +38,8 @@ void FrontEndFuzzyOpTest::doLoadFromFile() { } template -inline void addInputOutput(cnpy::NpyArray& npy_array, test::TestCase& test_case, bool is_input = true) { - T* npy_begin = npy_array.data(); +inline void addInputOutput(const cnpy::NpyArray& npy_array, test::TestCase& test_case, bool is_input = true) { + const T* npy_begin = npy_array.data(); std::vector data(npy_begin, npy_begin + npy_array.num_vals); if (is_input) test_case.add_input(npy_array.shape, data); diff --git a/src/frontends/tests/frontend/shared/test_builtin_extensions/builtin_extensions.cpp b/src/frontends/tests/frontend/shared/test_builtin_extensions/builtin_extensions.cpp index a80f1488342404..a397af1345eea5 100644 --- a/src/frontends/tests/frontend/shared/test_builtin_extensions/builtin_extensions.cpp +++ b/src/frontends/tests/frontend/shared/test_builtin_extensions/builtin_extensions.cpp @@ -17,9 +17,11 @@ #ifdef ENABLE_OV_PADDLE_FRONTEND # include -# define PADDLE_EXT \ - std::make_shared("NewCustomOp_4", CustomTranslatorPaddle), \ - std::make_shared("relu", ReluToSwishTranslatorPDPD), +# define PADDLE_EXT \ + std::make_shared("NewCustomOp_4", CustomTranslatorPaddle), \ + std::make_shared("relu", ReluToSwishTranslatorPDPD), \ + std::make_shared("NewCustomOp_4", CustomTranslatorPaddle), \ + std::make_shared("relu6", Relu6ToReluTranslatorPaddle), #else # define PADDLE_EXT #endif @@ -71,7 +73,88 @@ std::map CustomTranslatorPaddle(const ov::fronten return std::map(); } +std::map Relu6ToReluTranslatorPaddle(const ov::frontend::NodeContext& node) { + auto relu = std::make_shared(node.get_input("X")); + std::map ret; + ret["Out"] = {relu}; + return ret; +} + +class CustomElu : public ov::op::Op { +public: + OPENVINO_OP("CustomElu"); + + CustomElu() = default; + CustomElu(const ov::Output& input, float alpha, float beta) : m_alpha{alpha}, m_beta{beta} { + set_argument(0, input); + constructor_validate_and_infer_types(); + } + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& inputs) const override { + return std::make_shared(inputs[0], m_alpha, m_beta); + } + + void validate_and_infer_types() override { + set_output_size(1); + set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); + } + + bool visit_attributes(ov::AttributeVisitor& visitor) override { + visitor.on_attribute("m_alpha", m_alpha); + visitor.on_attribute("m_beta", m_beta); + return true; + } + + bool has_evaluate() const override { + switch (get_input_element_type(0)) { + case ov::element::f32: + return true; + default: + return false; + } + return false; + } + + template + void elu(const T* input, T* output, size_t n) const { + for (size_t i = 0; i < n; i++) { + if (input[i] > 0) + output[i] = m_beta * input[i]; + else + output[i] = m_alpha * (std::exp(input[i]) - 1); + } + } + + bool evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const override { + switch (get_input_element_type(0)) { + case ov::element::f32: + elu(inputs[0].data(), outputs[0].data(), ov::shape_size(get_output_shape(0))); + break; + default: + return false; + } + return true; + } + +private: + float m_alpha; + float m_beta; +}; + +#ifdef ENABLE_OV_PYTORCH_FRONTEND +# include +# include +# define PT_EXT \ + std::make_shared>( \ + "aten::elu", \ + std::map{{"m_alpha", 1}}, \ + std::map{{"m_beta", 1.0f}}), \ + std::make_shared("Relu", ReluToSwishTranslator), +#else +# define PT_EXT +#endif + OPENVINO_CREATE_EXTENSIONS(std::vector( {std::make_shared("NewCustomOp_1", CustomTranslatorCommon_1), std::make_shared("NewCustomOp_2", CustomTranslatorCommon_2), - ONNX_EXT PADDLE_EXT TF_EXT TF_LITE_EXT})); + ONNX_EXT PADDLE_EXT TF_EXT TF_LITE_EXT PT_EXT})); diff --git a/src/inference/CMakeLists.txt b/src/inference/CMakeLists.txt index ca27952acad912..6bc88dc8ea309c 100644 --- a/src/inference/CMakeLists.txt +++ b/src/inference/CMakeLists.txt @@ -79,8 +79,6 @@ if(ENABLE_SSE42) endif() endif() -addVersionDefines(src/ie_version.cpp CI_BUILD_NUMBER) - set (PUBLIC_HEADERS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") file (GLOB_RECURSE PUBLIC_HEADERS @@ -130,6 +128,8 @@ add_library(${TARGET_NAME}_obj OBJECT ${LIBRARY_HEADERS} ${PUBLIC_HEADERS}) +ov_add_version_defines(src/ie_version.cpp ${TARGET_NAME}_obj) + if(NOT BUILD_SHARED_LIBS) target_compile_definitions(${TARGET_NAME}_obj PUBLIC OPENVINO_STATIC_LIBRARY) endif() @@ -152,7 +152,7 @@ target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE target_include_directories(${TARGET_NAME}_obj PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" - # for static ie_plugins.hpp + # for static ov_plugins.hpp "${CMAKE_CURRENT_BINARY_DIR}" # for ie_ir_version.hpp $<$:$> diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp index d94aa978ffcfb9..f3d67bd98cfc59 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp @@ -47,32 +47,35 @@ DECLARE_CONFIG_KEY(LP_TRANSFORMS_MODE); DECLARE_CONFIG_KEY(CPU_THREADS_PER_STREAM); /** - * @brief Number of streams in Performance-core(big core) in HYBRID_AWARE and throughput - * The value is calculated in loadExeNetwork and is used by CPU Executor Streams to execute `parallel_for` calls + * @brief Number of streams in Performance-core(big core) * @ingroup ie_dev_api_plugin_api * @brief Shortcut for defining internal configuration values */ DECLARE_CONFIG_KEY(BIG_CORE_STREAMS); /** - * @brief Number of streams in Efficient-core(small core) in HYBRID_AWARE and throughput - * The value is calculated in loadExeNetwork and is used by CPU Executor Streams to execute `parallel_for` calls + * @brief Number of streams in Performance-logical core(big core) + * @ingroup ie_dev_api_plugin_api + * @brief Shortcut for defining internal configuration values + */ +DECLARE_CONFIG_KEY(BIG_CORE_LOGIC_STREAMS); + +/** + * @brief Number of streams in Efficient-core(small core) on hybrid cores machine * @ingroup ie_dev_api_plugin_api * @brief Shortcut for defining internal configuration values */ DECLARE_CONFIG_KEY(SMALL_CORE_STREAMS); /** - * @brief Threads per stream in big cores in HYBRID_AWARE and throughput - * The value is calculated in loadExeNetwork and is used by CPU Executor Streams to execute `parallel_for` calls + * @brief Number of threads per stream in big cores * @ingroup ie_dev_api_plugin_api * @brief Shortcut for defining internal configuration values */ DECLARE_CONFIG_KEY(THREADS_PER_STREAM_BIG); /** - * @brief Threads per stream in small cores in HYBRID_AWARE and throughput - * The value is calculated in loadExeNetwork and is used by CPU Executor Streams to execute `parallel_for` calls + * @brief Number of threads per stream in small cores on hybrid cores machine * @ingroup ie_dev_api_plugin_api * @brief Shortcut for defining internal configuration values */ diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp index 23898911263565..1a35764c109573 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_iplugin_internal.hpp @@ -316,7 +316,7 @@ class INFERENCE_ENGINE_API_CLASS(IInferencePlugin) : public std::enable_shared_f protected: IInferencePlugin(); - ~IInferencePlugin() = default; + virtual ~IInferencePlugin() = default; /** * @brief Creates an executable network from a parsed network object, users can create as many networks as they need diff --git a/src/inference/dev_api/ie_icore.hpp b/src/inference/dev_api/ie_icore.hpp index 32f08028d3f012..8852c1f4ecd8c9 100644 --- a/src/inference/dev_api/ie_icore.hpp +++ b/src/inference/dev_api/ie_icore.hpp @@ -173,13 +173,13 @@ class ICore : public ov::ICore { virtual std::vector GetAvailableDevices() const = 0; /** - * @brief Checks whether device supports Export & Import functionality of network + * @brief Checks whether device supports model caching feature * * @param deviceName - A name of a device to get a metric value. - * @return True if device has IMPORT_EXPORT_SUPPORT metric in SUPPORTED_METRICS and + * @return True if device has IMPORT_EXPORT_SUPPORT and CACHING_PROPERTIES metric in SUPPORTED_METRICS and * this metric returns 'true', False otherwise. */ - virtual bool DeviceSupportsImportExport(const std::string& deviceName) const = 0; + virtual bool DeviceSupportsModelCaching(const std::string& deviceName) const = 0; /** * @brief Create a new shared context object on specified accelerator device @@ -209,22 +209,4 @@ class ICore : public ov::ICore { virtual RemoteContext::Ptr GetDefaultContext(const std::string& deviceName) = 0; }; -/** - * @private - */ -class INFERENCE_ENGINE_API_CLASS(DeviceIDParser) { - std::string deviceName; - std::string deviceID; - -public: - explicit DeviceIDParser(const std::string& deviceNameWithID); - - std::string getDeviceID() const; - std::string getDeviceName() const; - - static std::vector getHeteroDevices(std::string fallbackDevice); - static std::vector getMultiDevices(std::string devicesList); - static std::string getBatchDevice(std::string devicesList); -}; - } // namespace InferenceEngine diff --git a/src/inference/dev_api/ie_ngraph_utils.hpp b/src/inference/dev_api/ie_ngraph_utils.hpp index f5204dbc4454ca..e3f4542c7035ec 100644 --- a/src/inference/dev_api/ie_ngraph_utils.hpp +++ b/src/inference/dev_api/ie_ngraph_utils.hpp @@ -62,43 +62,7 @@ inline ::ngraph::element::Type convertPrecision(const Precision& precision) { } inline ::ngraph::element::Type convertPrecision(const std::string& precision) { - if (precision == "f16" || precision == "FP16") { - return ::ngraph::element::Type(::ngraph::element::Type_t::f16); - } else if (precision == "f32" || precision == "FP32") { - return ::ngraph::element::Type(::ngraph::element::Type_t::f32); - } else if (precision == "bf16" || precision == "BF16") { - return ::ngraph::element::Type(::ngraph::element::Type_t::bf16); - } else if (precision == "f64" || precision == "FP64") { - return ::ngraph::element::Type(::ngraph::element::Type_t::f64); - } else if (precision == "i4" || precision == "I4") { - return ::ngraph::element::Type(::ngraph::element::Type_t::i4); - } else if (precision == "i8" || precision == "I8") { - return ::ngraph::element::Type(::ngraph::element::Type_t::i8); - } else if (precision == "i16" || precision == "I16") { - return ::ngraph::element::Type(::ngraph::element::Type_t::i16); - } else if (precision == "i32" || precision == "I32") { - return ::ngraph::element::Type(::ngraph::element::Type_t::i32); - } else if (precision == "i64" || precision == "I64") { - return ::ngraph::element::Type(::ngraph::element::Type_t::i64); - } else if (precision == "u1" || precision == "U1" || precision == "BIN" || precision == "bin") { - return ::ngraph::element::Type(::ngraph::element::Type_t::u1); - } else if (precision == "u4" || precision == "U4") { - return ::ngraph::element::Type(::ngraph::element::Type_t::u4); - } else if (precision == "u8" || precision == "U8") { - return ::ngraph::element::Type(::ngraph::element::Type_t::u8); - } else if (precision == "u16" || precision == "U16") { - return ::ngraph::element::Type(::ngraph::element::Type_t::u16); - } else if (precision == "u32" || precision == "U32") { - return ::ngraph::element::Type(::ngraph::element::Type_t::u32); - } else if (precision == "u64" || precision == "U64") { - return ::ngraph::element::Type(::ngraph::element::Type_t::u64); - } else if (precision == "boolean" || precision == "BOOL") { - return ::ngraph::element::Type(::ngraph::element::Type_t::boolean); - } else if (precision == "undefined") { - return ::ngraph::element::Type(::ngraph::element::Type_t::undefined); - } else { - IE_THROW() << "Incorrect precision: " << precision; - } + return ::ov::element::Type(precision); } inline Precision convertPrecision(const ::ngraph::element::Type& precision) { diff --git a/src/inference/dev_api/ie_system_conf.h b/src/inference/dev_api/ie_system_conf.h index 408c626accf436..fa4055538c2061 100644 --- a/src/inference/dev_api/ie_system_conf.h +++ b/src/inference/dev_api/ie_system_conf.h @@ -143,7 +143,48 @@ using ov::with_cpu_x86_avx512_core_amx_bf16; using ov::with_cpu_x86_avx512_core_amx; /** - * @brief This enum contains defination of each columns in processor type table which bases on cpu core types. Will + * @brief Checks whether CPU mapping Available + * @ingroup ie_dev_api_system_conf + * @return `True` is CPU mapping is available, `false` otherwise + */ +using ov::is_cpu_map_available; + +/** + * @brief Set flag bit 'Used' of CPU + * @ingroup ie_dev_api_system_conf + * @param[in] cpu_ids cpus in cup_mapping. + * @param[in] used flag bit + */ +using ov::set_cpu_used; + +/** + * @brief Returns number of CPU cores on Linux/Windows + * @ingroup ie_dev_api_system_conf + * @param[in] plugin_task plugin task. + * @return Number of CPU cores with core_type. + */ +using ov::get_num_available_cpu_cores; + +/** + * @brief Returns corresponding logical cores + * @ingroup ie_dev_api_system_conf + * @param[in] cpu_ids physical cores + * @return logical cores corresponding to physical core. + */ +using ov::get_logic_cores; + +/** + * @brief Returns available cpu ids + * @ingroup ie_dev_api_system_conf + * @param[in] core_type core type. + * @param[in] num_cpus number of cpus. + * @param[in] cpu_task is cpu task, not other plugin tasks + * @return Array of available cpu ids. + */ +using ov::reserve_available_cpus; + +/** + * @brief This enum contains definition of each columns in processor type table which bases on cpu core types. Will * extend to support other CPU core type like ARM. * * The following are two example of processor type table. @@ -162,7 +203,7 @@ using ov::with_cpu_x86_avx512_core_amx; using ov::ColumnOfProcessorTypeTable; /** - * @brief This enum contains defination of each columns in CPU mapping table which use processor id as index. + * @brief This enum contains definition of each columns in CPU mapping table which use processor id as index. * * GROUP_ID is generated according to the following rules. * 1. If one MAIN_CORE_PROC and one HYPER_THREADING_PROC are based on same Performance-cores, they are in one group. @@ -185,4 +226,9 @@ using ov::ColumnOfProcessorTypeTable; */ using ov::ColumnOfCPUMappingTable; +/** + * @brief definition of CPU_MAP_USED_FLAG column in CPU mapping table. + */ +using ov::ProcessorUseStatus; + } // namespace InferenceEngine diff --git a/src/inference/dev_api/openvino/runtime/device_id_parser.hpp b/src/inference/dev_api/openvino/runtime/device_id_parser.hpp new file mode 100644 index 00000000000000..ecd95c6a6a7859 --- /dev/null +++ b/src/inference/dev_api/openvino/runtime/device_id_parser.hpp @@ -0,0 +1,36 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief Provides parser for device name + * @file openvino/runtime/device_id_paeser.hpp + */ + +#pragma once + +#include + +#include "openvino/runtime/common.hpp" + +namespace ov { + +/** + * @brief Class parses device name and id + */ +class OPENVINO_RUNTIME_API DeviceIDParser { + std::string m_device_name; + std::string m_device_id; + +public: + explicit DeviceIDParser(const std::string& device_name_with_id); + + const std::string& get_device_id() const; + const std::string& get_device_name() const; + + static std::vector get_hetero_devices(const std::string& fallbackDevice); + static std::vector get_multi_devices(const std::string& devicesList); + static std::string get_batch_device(const std::string& devicesList); +}; + +} // namespace ov diff --git a/src/inference/dev_api/openvino/runtime/iplugin.hpp b/src/inference/dev_api/openvino/runtime/iplugin.hpp index 5d752ab5b15d08..a64073b3ebe0d1 100644 --- a/src/inference/dev_api/openvino/runtime/iplugin.hpp +++ b/src/inference/dev_api/openvino/runtime/iplugin.hpp @@ -20,6 +20,7 @@ #include "openvino/runtime/icore.hpp" #include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/threading/executor_manager.hpp" +#include "openvino/util/pp.hpp" namespace InferenceEngine { @@ -256,7 +257,11 @@ OPENVINO_RUNTIME_API std::unordered_set get_supported_nodes( std::function&)> transform, std::function)> is_node_supported); -} // namespace ov +/** + * @private + */ +using CreatePluginFunc = void(std::shared_ptr<::ov::IPlugin>&); + /** * @def OV_CREATE_PLUGIN * @brief Defines a name of a function creating plugin instance @@ -266,6 +271,13 @@ OPENVINO_RUNTIME_API std::unordered_set get_supported_nodes( # define OV_CREATE_PLUGIN CreatePluginEngine #endif +/** + * @private + */ +constexpr static const auto create_plugin_function = OV_PP_TOSTRING(OV_CREATE_PLUGIN); + +} // namespace ov + /** * @def OV_DEFINE_PLUGIN_CREATE_FUNCTION(PluginType, version) * @brief Defines the exported `OV_CREATE_PLUGIN` function which is used to create a plugin instance diff --git a/src/inference/dev_api/openvino/runtime/system_conf.hpp b/src/inference/dev_api/openvino/runtime/system_conf.hpp index 216d059ed357d2..8277a9cd7cac64 100644 --- a/src/inference/dev_api/openvino/runtime/system_conf.hpp +++ b/src/inference/dev_api/openvino/runtime/system_conf.hpp @@ -131,9 +131,41 @@ OPENVINO_RUNTIME_API bool with_cpu_x86_avx512_core_amx_bf16(); */ OPENVINO_RUNTIME_API bool with_cpu_x86_avx512_core_amx(); +/** + * @brief Checks whether cpu_mapping Available + * @ingroup ie_dev_api_system_conf + * @return `True` is CPU mapping is available, `false` otherwise + */ +OPENVINO_RUNTIME_API bool is_cpu_map_available(); + +/** + * @brief Returns number of CPU cores on Linux/Windows + * @ingroup ie_dev_api_system_conf + * @return A table about number of CPU cores of different types defined with ColumnOfProcessorTypeTable + * The following are two example of processor type table. + * 1. Processor table of two socket CPUs XEON server + * ALL_PROC | MAIN_CORE_PROC | EFFICIENT_CORE_PROC | HYPER_THREADING_PROC + * 96 48 0 48 // Total number of two sockets + * 48 24 0 24 // Number of socket one + * 48 24 0 24 // Number of socket two + * + * 2. Processor table of one socket CPU desktop + * ALL_PROC | MAIN_CORE_PROC | EFFICIENT_CORE_PROC | HYPER_THREADING_PROC + * 32 8 16 8 // Total number of one socket + */ +OPENVINO_RUNTIME_API std::vector> get_num_available_cpu_cores(); + +/** + * @brief Returns corresponding logical cores + * @ingroup ie_dev_api_system_conf + * @param[in] cpu_ids physical cores + * @return logical cores corresponding to physical core. + */ +OPENVINO_RUNTIME_API std::vector get_logic_cores(const std::vector cpu_ids); + /** * @enum ColumnOfProcessorTypeTable - * @brief This enum contains defination of each columns in processor type table which bases on cpu core types. Will + * @brief This enum contains definition of each columns in processor type table which bases on cpu core types. Will * extend to support other CPU core type like ARM. * * The following are two example of processor type table. @@ -157,9 +189,43 @@ enum ColumnOfProcessorTypeTable { PROC_TYPE_TABLE_SIZE = 4 //!< Size of processor type table }; +/** + * @enum ProcessorUseStatus + * @brief Definition of CPU_MAP_USED_FLAG column in CPU mapping table. + */ +enum ProcessorUseStatus { + NOT_USED = -1, //!< Processor is not bound to thread + CPU_USED = 1, //!< CPU is in using + PLUGIN_USED_START = 100 //!< Plugin other than CPU needs to use. If more GPUs use CPUs, the CPU_MAP_USED_FLAG is + //!< accumulated from PLUGIN_USED_START. For example: GPU.0:100, GPU.1:101 +}; + +/** + * @brief Get and reserve available cpu ids + * @ingroup ie_dev_api_system_conf + * @param[in] core_type core type. + * @param[in] num_cpus number of cpus. + * @param[in] seek_status look for CPU_MAP_USED_FLAG of seek_status in CPU mapping table + * @param[in] reset_status reset CPU_MAP_USED_FLAG with reset_status. + * @return Array of available cpu ids. + */ +OPENVINO_RUNTIME_API std::vector reserve_available_cpus(const ColumnOfProcessorTypeTable core_type, + const int num_cpus, + const int seek_status = NOT_USED, + const int reset_status = CPU_USED, + const bool reserve_logic_core = false); + +/** + * @brief Set CPU_MAP_USED_FLAG of cpu_mapping + * @ingroup ie_dev_api_system_conf + * @param[in] cpu_ids cpus in cpu_mapping. + * @param[in] used update CPU_MAP_USED_FLAG of cpu_mapping with this flag bit + */ +OPENVINO_RUNTIME_API void set_cpu_used(std::vector& cpu_ids, int used); + /** * @enum ColumnOfCPUMappingTable - * @brief This enum contains defination of each columns in CPU mapping table which use processor id as index. + * @brief This enum contains definition of each columns in CPU mapping table which use processor id as index. * * GROUP_ID is generated according to the following rules. * 1. If one MAIN_CORE_PROC and one HYPER_THREADING_PROC are based on same Performance-cores, they are in one group. diff --git a/src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp b/src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp new file mode 100644 index 00000000000000..438a730565dedf --- /dev/null +++ b/src/inference/dev_api/openvino/runtime/threading/cpu_map_scheduling.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file for CPU map scheduling + * @file cpu_map_scheduling.hpp + */ + +#pragma once + +#include + +#include "openvino/runtime/properties.hpp" + +namespace ov { + +/** + * @brief Limit available CPU resource in processors type table according to scheduling core type property + * @param[in] input_type input value of core type property. + * @param[in] proc_type_table candidate processors available at this time + * @return updated proc_type_table which removed unmatched processors + */ +std::vector> apply_scheduling_core_type(const ov::hint::SchedulingCoreType input_type, + const std::vector>& proc_type_table); + +/** + * @brief Limit available CPU resource in processors type table according to hyper threading property + * @param[in] input_type indicate value of property enable_hyper_threading. + * @param[in] input_changed indicate if value is set by user. + * @param[in] proc_type_table candidate processors available at this time + * @return updated proc_type_table which removed unmatched processors + */ +std::vector> apply_hyper_threading(bool input_type, + const bool input_changed, + const std::vector>& proc_type_table); + +} // namespace ov \ No newline at end of file diff --git a/src/inference/dev_api/openvino/runtime/threading/istreams_executor.hpp b/src/inference/dev_api/openvino/runtime/threading/istreams_executor.hpp index 2a64b69856d699..58ac9803043a6d 100644 --- a/src/inference/dev_api/openvino/runtime/threading/istreams_executor.hpp +++ b/src/inference/dev_api/openvino/runtime/threading/istreams_executor.hpp @@ -14,6 +14,7 @@ #include #include "openvino/runtime/common.hpp" +#include "openvino/runtime/system_conf.hpp" #include "openvino/runtime/threading/itask_executor.hpp" namespace ov { @@ -95,11 +96,14 @@ class OPENVINO_RUNTIME_API IStreamsExecutor : virtual public ITaskExecutor { int _threads = 0; //!< Number of threads distributed between streams. //!< Reserved. Should not be used. int _big_core_streams = 0; //!< Number of streams in Performance-core(big core) + int _big_core_logic_streams = 0; //!< Number of streams in Logical-core int _small_core_streams = 0; //!< Number of streams in Efficient-core(small core) int _threads_per_stream_big = 0; //!< Threads per stream in big cores int _threads_per_stream_small = 0; //!< Threads per stream in small cores int _small_core_offset = 0; //!< Calculate small core start offset when binding cpu cores bool _enable_hyper_thread = true; //!< enable hyper thread + bool _set_streams = false; + int _plugin_task = NOT_USED; enum StreamMode { DEFAULT, AGGRESSIVE, LESSAGGRESSIVE }; enum PreferredCoreType { ANY, diff --git a/src/inference/dev_api/performance_heuristics.hpp b/src/inference/dev_api/performance_heuristics.hpp index 563d7627393701..41a3da4489b189 100644 --- a/src/inference/dev_api/performance_heuristics.hpp +++ b/src/inference/dev_api/performance_heuristics.hpp @@ -67,7 +67,9 @@ static MemBandwidthPressure MemBandwidthPressureTolerance( output.get_partial_shape().is_static()) { const auto& shapeInput0 = input0.get_shape(); const auto& shapeInput1 = input1.get_shape(); + OPENVINO_SUPPRESS_DEPRECATED_START const auto non_const = !get_constant_from_source(node->input_value(1)); + OPENVINO_SUPPRESS_DEPRECATED_END const auto& shapeOutput = output.get_shape(); const auto dataSizeInput0 = std::accumulate(shapeInput0.begin(), shapeInput0.end(), size_t(1), std::multiplies()); diff --git a/src/inference/dev_api/threading/ie_cpu_streams_info.hpp b/src/inference/dev_api/threading/ie_cpu_streams_info.hpp index 6bd3d8aad96f2e..119bc89a5cf263 100644 --- a/src/inference/dev_api/threading/ie_cpu_streams_info.hpp +++ b/src/inference/dev_api/threading/ie_cpu_streams_info.hpp @@ -37,4 +37,4 @@ typedef enum { CPU_STREAMS_TABLE_SIZE = 3 //!< Size of streams info table } column_of_cpu_streams_info_table; -} // namespace InferenceEngine +} // namespace InferenceEngine \ No newline at end of file diff --git a/src/inference/include/ie/ie_iexecutable_network.hpp b/src/inference/include/ie/ie_iexecutable_network.hpp index 73106ef89900a8..c1299fea0c1a12 100644 --- a/src/inference/include/ie/ie_iexecutable_network.hpp +++ b/src/inference/include/ie/ie_iexecutable_network.hpp @@ -159,7 +159,7 @@ class INFERENCE_ENGINE_DEPRECATED("Use InferenceEngine::ExecutableNetwork instea virtual StatusCode GetContext(RemoteContext::Ptr& pContext, ResponseDesc* resp) const noexcept = 0; protected: - ~IExecutableNetwork() = default; + virtual ~IExecutableNetwork() = default; }; _IE_SUPPRESS_DEPRECATED_END_GCC diff --git a/src/inference/include/ie/ie_iinfer_request.hpp b/src/inference/include/ie/ie_iinfer_request.hpp index 04ff00d94a94c1..05da3ed1442bbb 100644 --- a/src/inference/include/ie/ie_iinfer_request.hpp +++ b/src/inference/include/ie/ie_iinfer_request.hpp @@ -202,7 +202,7 @@ class INFERENCE_ENGINE_DEPRECATED("Use InferenceEngine::InferRequest C++ wrapper virtual InferenceEngine::StatusCode SetBatch(int batch_size, ResponseDesc* resp) noexcept = 0; protected: - ~IInferRequest() = default; + virtual ~IInferRequest() = default; }; _IE_SUPPRESS_DEPRECATED_END_GCC diff --git a/src/inference/include/openvino/runtime/core.hpp b/src/inference/include/openvino/runtime/core.hpp index d761d8a2c52c33..3de1aad5ac0689 100644 --- a/src/inference/include/openvino/runtime/core.hpp +++ b/src/inference/include/openvino/runtime/core.hpp @@ -196,22 +196,29 @@ class OPENVINO_RUNTIME_API Core { * operation. * * @return A compiled model. + * @{ */ CompiledModel compile_model(const std::string& model_path, const AnyMap& properties = {}); +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + CompiledModel compile_model(const std::wstring& model_path, const AnyMap& properties = {}); +#endif + /// @} + /** - * @brief Reads and loads a compiled model from IR / ONNX / PDPD file to the default OpenVINI device selected by + * @brief Reads and loads a compiled model from IR / ONNX / PDPD file to the default OpenVINO device selected by * AUTO plugin. * * This can be more efficient than using read_model + compile_model(Model) flow * especially for cases when caching is enabled and cached model is available * * @tparam Properties Should be the pack of `std::pair` types - * @param model_path path to model + * @param model_path path to model with string or wstring * @param properties Optional pack of pairs: (property name, property value) relevant only for this * load operation * * @return A compiled model + * @{ */ template util::EnableIfAllStringAny compile_model(const std::string& model_path, @@ -219,6 +226,15 @@ class OPENVINO_RUNTIME_API Core { return compile_model(model_path, AnyMap{std::forward(properties)...}); } +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + template + util::EnableIfAllStringAny compile_model(const std::wstring& model_path, + Properties&&... properties) { + return compile_model(model_path, AnyMap{std::forward(properties)...}); + } +#endif + /// @} + /** * @brief Reads a model and creates a compiled model from the IR/ONNX/PDPD file. * @@ -231,11 +247,19 @@ class OPENVINO_RUNTIME_API Core { * operation. * * @return A compiled model. + * @{ */ CompiledModel compile_model(const std::string& model_path, const std::string& device_name, const AnyMap& properties = {}); +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + CompiledModel compile_model(const std::wstring& model_path, + const std::string& device_name, + const AnyMap& properties = {}); +#endif + /// @} + /** * @brief Reads a model and creates a compiled model from the IR/ONNX/PDPD file. * @@ -249,6 +273,7 @@ class OPENVINO_RUNTIME_API Core { * load operation. * * @return A compiled model. + * @{ */ template util::EnableIfAllStringAny compile_model(const std::string& model_path, @@ -257,6 +282,16 @@ class OPENVINO_RUNTIME_API Core { return compile_model(model_path, device_name, AnyMap{std::forward(properties)...}); } +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + template + util::EnableIfAllStringAny compile_model(const std::wstring& model_path, + const std::string& device_name, + Properties&&... properties) { + return compile_model(model_path, device_name, AnyMap{std::forward(properties)...}); + } +#endif + /// @} + /** * @brief Reads a model and creates a compiled model from the IR/ONNX/PDPD memory. * @param model String with a model in IR/ONNX/PDPD format. diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index d5a1853cb5fe11..b2b81b51c2465c 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -233,22 +233,16 @@ static constexpr Property model_name{"NETWO static constexpr Property optimal_number_of_infer_requests{ "OPTIMAL_NUMBER_OF_INFER_REQUESTS"}; -/** - * @brief Hint for device to use specified precision for inference - * @ingroup ov_runtime_cpp_prop_api - */ -static constexpr Property inference_precision{"INFERENCE_PRECISION_HINT"}; - /** * @brief Namespace with hint properties */ namespace hint { /** - * @brief An alias for inference_precision property for backward compatibility + * @brief Hint for device to use specified precision for inference * @ingroup ov_runtime_cpp_prop_api */ -using ov::inference_precision; +static constexpr Property inference_precision{"INFERENCE_PRECISION_HINT"}; /** * @brief Enum to define possible priorities hints @@ -271,7 +265,7 @@ inline std::ostream& operator<<(std::ostream& os, const Priority& priority) { case Priority::HIGH: return os << "HIGH"; default: - OPENVINO_THROW("Unsupported performance measure hint"); + OPENVINO_THROW("Unsupported model priority value"); } } @@ -284,6 +278,8 @@ inline std::istream& operator>>(std::istream& is, Priority& priority) { priority = Priority::MEDIUM; } else if (str == "HIGH") { priority = Priority::HIGH; + } else if (str == "DEFAULT") { + priority = Priority::DEFAULT; } else { OPENVINO_THROW("Unsupported model priority: ", str); } @@ -303,7 +299,8 @@ static constexpr Property model_priority{"MODEL_PRIORITY"}; * @ingroup ov_runtime_cpp_prop_api */ enum class PerformanceMode { - UNDEFINED = -1, //!< Undefined value, performance setting may vary from device to device + UNDEFINED OPENVINO_ENUM_DEPRECATED("Please use actual value instead. Will be removed in 2024.0") = + -1, //!< Undefined value, performance setting may vary from device to device LATENCY = 1, //!< Optimize for latency THROUGHPUT = 2, //!< Optimize for throughput CUMULATIVE_THROUGHPUT = 3, //!< Optimize for cumulative throughput @@ -312,8 +309,10 @@ enum class PerformanceMode { /** @cond INTERNAL */ inline std::ostream& operator<<(std::ostream& os, const PerformanceMode& performance_mode) { switch (performance_mode) { + OPENVINO_SUPPRESS_DEPRECATED_START case PerformanceMode::UNDEFINED: return os << "UNDEFINED"; + OPENVINO_SUPPRESS_DEPRECATED_END case PerformanceMode::LATENCY: return os << "LATENCY"; case PerformanceMode::THROUGHPUT: @@ -335,7 +334,9 @@ inline std::istream& operator>>(std::istream& is, PerformanceMode& performance_m } else if (str == "CUMULATIVE_THROUGHPUT") { performance_mode = PerformanceMode::CUMULATIVE_THROUGHPUT; } else if (str == "UNDEFINED") { + OPENVINO_SUPPRESS_DEPRECATED_START performance_mode = PerformanceMode::UNDEFINED; + OPENVINO_SUPPRESS_DEPRECATED_END } else { OPENVINO_THROW("Unsupported performance mode: ", str); } @@ -351,6 +352,94 @@ inline std::istream& operator>>(std::istream& is, PerformanceMode& performance_m */ static constexpr Property performance_mode{"PERFORMANCE_HINT"}; +/** + * @enum SchedulingCoreType + * @brief This enum contains definition of core type can be used for CPU tasks on different devices. + */ +enum class SchedulingCoreType { + ANY_CORE = 0, //!< Any processors can be used. + PCORE_ONLY = 1, //!< Only processors of performance-cores can be used. + ECORE_ONLY = 2, //!< Only processors of efficient-cores can be used. +}; + +/** @cond INTERNAL */ +inline std::ostream& operator<<(std::ostream& os, const SchedulingCoreType& core_type) { + switch (core_type) { + case SchedulingCoreType::ANY_CORE: + return os << "ANY_CORE"; + case SchedulingCoreType::PCORE_ONLY: + return os << "PCORE_ONLY"; + case SchedulingCoreType::ECORE_ONLY: + return os << "ECORE_ONLY"; + default: + throw ov::Exception{"Unsupported core type!"}; + } +} + +inline std::istream& operator>>(std::istream& is, SchedulingCoreType& core_type) { + std::string str; + is >> str; + if (str == "ANY_CORE") { + core_type = SchedulingCoreType::ANY_CORE; + } else if (str == "PCORE_ONLY") { + core_type = SchedulingCoreType::PCORE_ONLY; + } else if (str == "ECORE_ONLY") { + core_type = SchedulingCoreType::ECORE_ONLY; + } else { + throw ov::Exception{"Unsupported core type: " + str}; + } + return is; +} +/** @endcond */ + +/** + * @brief This property defines CPU core type which can be used during inference. + * @ingroup ov_runtime_cpp_prop_api + * + * Developer can use this property to select specific CPU cores for inference. Please refer SchedulingCoreType for + * all definition of core type. + * + * The following code is an example to only use efficient-cores for inference on hybrid CPU. If user sets this + * configuration on a platform with only performance-cores, CPU inference will still run on the performance-cores. + * + * @code + * ie.set_property(ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ECORE_ONLY)); + * @endcode + */ +static constexpr Property scheduling_core_type{"SCHEDULING_CORE_TYPE"}; + +/** + * @brief This property allows CPU threads pinning during inference. + * @ingroup ov_runtime_cpp_prop_api + * + * Developer can use this property to use or not use CPU threads pinning during inference. If user does not explicitly + * set value for this property, OpenVINO may choose any desired value based on internal logic. + * + * The following code is example to use this property. + * + * @code + * ie.set_property(ov::hint::enable_cpu_pinning(true)); + * ie.set_property(ov::hint::enable_cpu_pinning(false)); + * @endcode + */ +static constexpr Property enable_cpu_pinning{"ENABLE_CPU_PINNING"}; + +/** + * @brief This property define if using hyper threading during inference. + * @ingroup ov_runtime_cpp_prop_api + * + * Developer can use this property to use or not use CPU pinning during inference. If user does not explicitly set + * value for this property, OpenVINO may choose any desired value based on internal logic. + * + * The following code is example to use this property. + * + * @code + * ie.set_property(ov::hint::enable_hyper_threading(true)); + * ie.set_property(ov::hint::enable_hyper_threading(false)); + * @endcode + */ +static constexpr Property enable_hyper_threading{"ENABLE_HYPER_THREADING"}; + /** * @brief (Optional) property that backs the (above) Performance Hints * by giving additional information on how many inference requests the application will be keeping in flight @@ -377,16 +466,13 @@ static constexpr Property allow_auto_batching{"ALL * @ingroup ov_runtime_cpp_prop_api */ enum class ExecutionMode { - UNDEFINED = -1, //!< Undefined value, settings may vary from device to device - PERFORMANCE = 1, //!< Optimize for max performance + PERFORMANCE = 1, //!< Optimize for max performance, may apply properties which slightly affect accuracy ACCURACY = 2, //!< Optimize for max accuracy }; /** @cond INTERNAL */ inline std::ostream& operator<<(std::ostream& os, const ExecutionMode& mode) { switch (mode) { - case ExecutionMode::UNDEFINED: - return os << "UNDEFINED"; case ExecutionMode::PERFORMANCE: return os << "PERFORMANCE"; case ExecutionMode::ACCURACY: @@ -403,8 +489,6 @@ inline std::istream& operator>>(std::istream& is, ExecutionMode& mode) { mode = ExecutionMode::PERFORMANCE; } else if (str == "ACCURACY") { mode = ExecutionMode::ACCURACY; - } else if (str == "UNDEFINED") { - mode = ExecutionMode::UNDEFINED; } else { OPENVINO_THROW("Unsupported execution mode: ", str); } @@ -594,6 +678,17 @@ static constexpr Property, */ static constexpr Property force_tbb_terminate{"FORCE_TBB_TERMINATE"}; +/** + * @brief Read-write property to configure `mmap()` use for model read. Enabled by default. + * For the moment only IR Frontend supports the property. + * + * value type: boolean + * - True enable `mmap()` use and map model + * - False disable `mmap()` use and read model + * @ingroup ov_runtime_cpp_prop_api + */ +static constexpr Property enable_mmap{"ENABLE_MMAP"}; + /** * @brief Namespace with device properties */ diff --git a/src/inference/src/core.cpp b/src/inference/src/core.cpp index 01454656e7f3ad..19008d60ac9120 100644 --- a/src/inference/src/core.cpp +++ b/src/inference/src/core.cpp @@ -9,12 +9,9 @@ #include "dev/converter_utils.hpp" #include "dev/core_impl.hpp" #include "ie_itt.hpp" +#include "openvino/runtime/device_id_parser.hpp" #include "so_extension.hpp" -#ifdef OPENVINO_STATIC_LIBRARY -# include "ie_plugins.hpp" -#endif - namespace { std::string resolve_extension_path(const std::string& path) { std::string retvalue; @@ -31,8 +28,6 @@ std::string resolve_extension_path(const std::string& path) { namespace ov { -#ifndef OPENVINO_STATIC_LIBRARY - std::string findPluginXML(const std::string& xmlFile) { std::string xmlConfigFile_ = xmlFile; if (xmlConfigFile_.empty()) { @@ -55,14 +50,10 @@ std::string findPluginXML(const std::string& xmlFile) { xmlConfigFileDefault = FileUtils::makePath(ielibraryDir, ov::util::to_file_path("plugins.xml")); if (FileUtils::fileExist(xmlConfigFileDefault)) return xmlConfigFile_ = ov::util::from_file_path(xmlConfigFileDefault); - - OPENVINO_THROW("Failed to find plugins.xml file"); } return xmlConfigFile_; } -#endif // OPENVINO_STATIC_LIBRARY - #define OV_CORE_CALL_STATEMENT(...) \ try { \ __VA_ARGS__; \ @@ -80,13 +71,13 @@ class Core::Impl : public CoreImpl { Core::Core(const std::string& xml_config_file) { _impl = std::make_shared(); -#ifdef OPENVINO_STATIC_LIBRARY - OV_CORE_CALL_STATEMENT(_impl->register_plugins_in_registry(::getStaticPluginsRegistry());) -#else - OV_CORE_CALL_STATEMENT( - // If XML is default, load default plugins by absolute paths - _impl->register_plugins_in_registry(findPluginXML(xml_config_file), xml_config_file.empty());) -#endif + std::string xmlConfigFile = ov::findPluginXML(xml_config_file); + if (!xmlConfigFile.empty()) + OV_CORE_CALL_STATEMENT( + // If XML is default, load default plugins by absolute paths + _impl->register_plugins_in_registry(xmlConfigFile, xml_config_file.empty());) + // Load plugins from the pre-compiled list + OV_CORE_CALL_STATEMENT(_impl->register_compile_time_plugins();) } std::map Core::get_versions(const std::string& device_name) const { @@ -130,6 +121,12 @@ CompiledModel Core::compile_model(const std::string& model_path, const AnyMap& c return compile_model(model_path, ov::DEFAULT_DEVICE_NAME, config); } +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +CompiledModel Core::compile_model(const std::wstring& model_path, const AnyMap& config) { + return compile_model(ov::util::wstring_to_string(model_path), config); +} +#endif + CompiledModel Core::compile_model(const std::string& model_path, const std::string& device_name, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ auto exec = _impl->compile_model(model_path, device_name, config); @@ -137,6 +134,14 @@ CompiledModel Core::compile_model(const std::string& model_path, const std::stri }); } +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +CompiledModel Core::compile_model(const std::wstring& model_path, + const std::string& device_name, + const AnyMap& config) { + return compile_model(ov::util::wstring_to_string(model_path), device_name, config); +} +#endif + CompiledModel Core::compile_model(const std::string& model, const ov::Tensor& weights, const std::string& device_name, @@ -171,8 +176,11 @@ void Core::add_extension(const std::string& library_path) { OPENVINO_SUPPRESS_DEPRECATED_START add_extension(extension_ptr); OPENVINO_SUPPRESS_DEPRECATED_END - } catch (const std::runtime_error&) { - OPENVINO_THROW("Cannot add extension. Cannot find entry point to the extension library"); + } catch (const std::runtime_error& e) { + OPENVINO_THROW( + std::string( + "Cannot add extension. Cannot find entry point to the extension library. This error happened: ") + + e.what()); } } } @@ -252,8 +260,8 @@ void Core::register_plugin(const std::string& plugin, const std::string& device_ void Core::unload_plugin(const std::string& device_name) { OV_CORE_CALL_STATEMENT({ - ie::DeviceIDParser parser(device_name); - std::string devName = parser.getDeviceName(); + ov::DeviceIDParser parser(device_name); + std::string devName = parser.get_device_name(); _impl->unload_plugin(devName); }); diff --git a/src/inference/src/dev/converter_utils.cpp b/src/inference/src/dev/converter_utils.cpp index a8c1999a3fb1ba..e54b8f64d1e550 100644 --- a/src/inference/src/dev/converter_utils.cpp +++ b/src/inference/src/dev/converter_utils.cpp @@ -227,6 +227,9 @@ class IInferencePluginWrapper : public InferenceEngine::IInferencePlugin { _isNewAPI = plugin->is_new_api(); _executorManager = InferenceEngine::create_old_manager(plugin->get_executor_manager()); } + + virtual ~IInferencePluginWrapper() = default; + std::string GetName() const noexcept override { return m_plugin->get_device_name(); } @@ -840,6 +843,7 @@ class IRemoteContextWrapper : public ov::IRemoteContext { public: IRemoteContextWrapper(const std::shared_ptr& context) : m_context(context) {} + virtual ~IRemoteContextWrapper() = default; const std::shared_ptr& get_context() { return m_context; } diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 44fc79a4987f59..1f1a88c39e4837 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -28,12 +28,15 @@ #include "openvino/core/preprocess/pre_post_process.hpp" #include "openvino/core/version.hpp" #include "openvino/pass/manager.hpp" +#include "openvino/runtime/device_id_parser.hpp" #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/itensor.hpp" #include "openvino/runtime/remote_context.hpp" #include "openvino/runtime/threading/executor_manager.hpp" #include "openvino/util/common_util.hpp" +#include "openvino/util/file_util.hpp" #include "openvino/util/shared_object.hpp" +#include "ov_plugins.hpp" #include "preprocessing/preprocessing.hpp" #include "xml_parse_utils.h" @@ -276,9 +279,9 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, const An updated_device_name = deviceName.substr(0, pos); parsed_device_priority = deviceName.substr(pos + 1); } else { - InferenceEngine::DeviceIDParser parser(deviceName); - updated_device_name = parser.getDeviceName(); - parsed_device_priority = parser.getDeviceID(); + ov::DeviceIDParser parser(deviceName); + updated_device_name = parser.get_device_name(); + parsed_device_priority = parser.get_device_id(); } // checks and updates device priority @@ -296,8 +299,8 @@ ov::Parsed ov::parseDeviceNameIntoConfig(const std::string& deviceName, const An }; // clean-up auto-batch related properties - clean_batch_properties(updated_device_name, updated_config, ov::hint::allow_auto_batching.name()); - clean_batch_properties(updated_device_name, updated_config, ov::auto_batch_timeout.name()); + clean_batch_properties(updated_device_name, updated_config, ov::hint::allow_auto_batching); + clean_batch_properties(updated_device_name, updated_config, ov::auto_batch_timeout); return {updated_device_name, updated_config}; } @@ -310,6 +313,35 @@ ov::CoreImpl::CoreImpl(bool _newAPI) : m_new_api(_newAPI) { } } +void ov::CoreImpl::register_compile_time_plugins() { + std::lock_guard lock(get_mutex()); + + const decltype(::getCompiledPluginsRegistry())& plugins = getCompiledPluginsRegistry(); + for (const auto& plugin : plugins) { + const auto& deviceName = plugin.first; + if (deviceName.find('.') != std::string::npos) { + OPENVINO_THROW("Device name must not contain dot '.' symbol"); + } +#ifdef OPENVINO_STATIC_LIBRARY + if (pluginRegistry.find(deviceName) == pluginRegistry.end()) { + const auto& value = plugin.second; + ov::AnyMap config = any_copy(value.m_default_config); + PluginDescriptor desc{value.m_create_plugin_func, config, value.m_create_extension_func}; + pluginRegistry[deviceName] = desc; + add_mutex(deviceName); + } +#else + const auto& pluginPath = ov::util::get_compiled_plugin_path(plugin.second.m_plugin_path); + if (pluginRegistry.find(deviceName) == pluginRegistry.end() && ov::util::file_exists(pluginPath)) { + ov::AnyMap config = any_copy(plugin.second.m_default_config); + PluginDescriptor desc{pluginPath, config}; + pluginRegistry[deviceName] = desc; + add_mutex(deviceName); + } +#endif + } +} + void ov::CoreImpl::register_plugins_in_registry(const std::string& xml_config_file, const bool& by_abs_path) { std::lock_guard lock(get_mutex()); @@ -416,8 +448,7 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { } else { so = ov::util::load_shared_object(desc.libraryLocation.c_str()); std::shared_ptr plugin_impl; - reinterpret_cast( - ov::util::get_symbol(so, InferenceEngine::create_plugin_function))(plugin_impl); + reinterpret_cast(ov::util::get_symbol(so, ov::create_plugin_function))(plugin_impl); plugin = Plugin{plugin_impl, so}; } @@ -425,8 +456,8 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { plugin.set_name(deviceName); // Set Core class reference to plugins - std::weak_ptr mutableCore = - std::const_pointer_cast(shared_from_this()); + std::weak_ptr mutableCore = + std::const_pointer_cast(std::dynamic_pointer_cast(shared_from_this())); plugin.set_core(mutableCore); } @@ -461,10 +492,8 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { { auto supportedConfigKeys = plugin.get_property(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); - auto config_iter = std::find(supportedConfigKeys.begin(), - supportedConfigKeys.end(), - CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); - const bool supportsConfigDeviceID = config_iter != supportedConfigKeys.end(); + const bool supportsConfigDeviceID = + ov::util::contains(supportedConfigKeys, CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); const std::string deviceKey = supportsConfigDeviceID ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) : CONFIG_KEY(DEVICE_ID); @@ -472,9 +501,9 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { // for each such .0, .1, .# device to make sure plugin can handle different settings for different // device IDs for (auto pluginDesc : pluginRegistry) { - InferenceEngine::DeviceIDParser parser(pluginDesc.first); - if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.getDeviceID().empty()) { - pluginDesc.second.defaultConfig[deviceKey] = parser.getDeviceID(); + ov::DeviceIDParser parser(pluginDesc.first); + if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.get_device_id().empty()) { + pluginDesc.second.defaultConfig[deviceKey] = parser.get_device_id(); plugin.set_property(pluginDesc.second.defaultConfig); } } @@ -527,7 +556,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto plugin = get_plugin(parsed._deviceName); ov::SoPtr res; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; - if (cacheManager && device_supports_import_export(plugin)) { + if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); @@ -556,7 +585,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto plugin = get_plugin(parsed._deviceName); ov::SoPtr res; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; - if (cacheManager && device_supports_import_export(plugin)) { + if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager}; cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); @@ -598,7 +627,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod ov::SoPtr compiled_model; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; - if (cacheManager && device_supports_import_export(plugin)) { + if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager, model_path}; cacheContent.blobId = ov::ModelCache::compute_hash(model_path, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); @@ -629,7 +658,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod ov::SoPtr compiled_model; auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; - if (cacheManager && device_supports_import_export(plugin)) { + if (cacheManager && device_supports_model_caching(plugin)) { CacheContent cacheContent{cacheManager}; cacheContent.blobId = ov::ModelCache::compute_hash(model_str, weights, create_compile_config(plugin, parsed._config)); @@ -708,6 +737,26 @@ ov::RemoteContext ov::CoreImpl::create_context(const std::string& device_name, c ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name, const ov::AnyMap& user_properties) const { + if (is_virtual_device(full_device_name)) { + // Considerations: + // 1. in case of virtual devices all the magic will happen on the level when + // virtual device calls ICore::get_supported_property for real HW devices + // so, for now we can returns user properties almost as is without any + // filtering / flattening + // 2. The only exception here: while common properties like ov::num::streams or + // ov::hint::performance_mode are shared across all the devices, the + // ov::device::priority cannot be shared, because it's specific for current virtual + // plugin. So, we need to remove ov::device::priorities from the list, because it's + // supposed to be set for current virtual plugin and cannot be propogated down + ov::AnyMap return_properties = clone_map(user_properties); + auto device_priorities_it = return_properties.find(ov::device::priorities.name()); + if (device_priorities_it != return_properties.end()) { + return_properties.erase(device_priorities_it); + } + + return return_properties; + } + static const std::vector core_level_properties = { ov::cache_dir.name(), ov::force_tbb_terminate.name(), @@ -719,28 +768,6 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n const auto flattened = ov::parseDeviceNameIntoConfig(full_device_name, user_properties); const std::string& device_name = flattened._deviceName; const auto& flattened_config = flattened._config; - ov::AnyMap supported_config, options; - - // fill 'options' to provide more information to ICore::get_property calls - { - auto priority_prop_name = get_device_priority_property(device_name).prop_name; - auto it = flattened_config.find(priority_prop_name); - if (it != flattened_config.end()) - options[it->first] = it->second; - else if (device_name == "HETERO") { - // TODO: remove together with API 1.0 - priority_prop_name = "TARGET_FALLBACK"; - it = flattened_config.find(priority_prop_name); - if (it != flattened_config.end()) - options[it->first] = it->second; - } else if (device_name == "BATCH") { - // TODO: remove together with API 1.0 - priority_prop_name = CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG); - it = flattened_config.find(priority_prop_name); - if (it != flattened_config.end()) - options[it->first] = it->second; - } - } // virtual plugins should bypass core-level properties to HW plugins // so, we need to report them as supported @@ -749,16 +776,16 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n // try to search against IE API 1.0' SUPPORTED_CONFIG_KEYS try { const auto supported_keys = - GetMetric(device_name, METRIC_KEY(SUPPORTED_CONFIG_KEYS), options).as>(); + GetMetric(device_name, METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); for (auto&& config_key : supported_keys) { supported_config_keys.emplace_back(config_key); } } catch (ov::Exception&) { } - // try to search against OV API 2.0' supported_properties + // try to search against OV API 2.0' mutable supported_properties try { - for (auto&& property : ICore::get_property(device_name, ov::supported_properties, options)) { + for (auto&& property : ICore::get_property(device_name, ov::supported_properties, {})) { if (property.is_mutable()) { supported_config_keys.emplace_back(std::move(property)); } @@ -766,11 +793,14 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n } catch (ov::Exception&) { } + // collect supported properties for HW device + AnyMap supported_config; for (auto&& kvp : flattened_config) { if (util::contains(supported_config_keys, kvp.first)) { supported_config[kvp.first] = kvp.second; } } + return supported_config; } @@ -795,7 +825,7 @@ void ov::CoreImpl::apply_auto_batching(const std::shared_ptr& m if (pos == std::string::npos) return; // BATCH device is already configured via the config deviceNameWithBatchSize = deviceName.substr(pos + 1); - deviceNameWithoutBatch = InferenceEngine::DeviceIDParser::getBatchDevice(deviceNameWithBatchSize); + deviceNameWithoutBatch = ov::DeviceIDParser::get_batch_device(deviceNameWithBatchSize); // when user sets the BATCH device explicitly, we may check the dims less strictly // as the result is being checked by the user strictly_check_dims = false; @@ -877,10 +907,10 @@ void ov::CoreImpl::set_property(const std::string& device_name, const AnyMap& pr // unsupport to set ov::device::properties to HW device through this function auto devices = get_registered_devices(); for (auto&& config : properties) { - auto is_secondary_config_for_hw_device = config.first.find(ov::device::properties.name()) != std::string::npos; - OPENVINO_ASSERT(!is_secondary_config_for_hw_device, + const auto is_secondary_property = config.first.find(ov::device::properties.name()) != std::string::npos; + OPENVINO_ASSERT(!is_secondary_property, "set_property do not support ov::device::propreties. " - "You can configure the devices through the compile_model()/loadNetwork() API."); + "You can configure the devices through the compile_model()/query_model() API."); } set_property_for_device(properties, device_name); } @@ -894,6 +924,9 @@ ov::Any ov::CoreImpl::get_property_for_core(const std::string& name) const { } else if (name == ov::hint::allow_auto_batching.name()) { const auto flag = coreConfig.get_allow_auto_batch(); return decltype(ov::hint::allow_auto_batching)::value_type(flag); + } else if (name == ov::enable_mmap.name()) { + const auto flag = coreConfig.get_enable_mmap(); + return decltype(ov::enable_mmap)::value_type(flag); } OPENVINO_THROW("Exception is thrown while trying to call get_property with unsupported property: '", name, "'"); @@ -982,8 +1015,8 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st return; } - InferenceEngine::DeviceIDParser parser(deviceName); - std::string clearDeviceName = parser.getDeviceName(); + ov::DeviceIDParser parser(deviceName); + std::string clearDeviceName = parser.get_device_name(); std::vector> created_plugins; { @@ -1056,17 +1089,12 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st } // Add device specific value to support device_name.device_id cases { - auto supportedConfigKeys = - plugin.second.get_property(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); - auto config_iter = std::find(supportedConfigKeys.begin(), - supportedConfigKeys.end(), - CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); - const bool supportsConfigDeviceID = config_iter != supportedConfigKeys.end(); - const std::string deviceKey = - supportsConfigDeviceID ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) : CONFIG_KEY(DEVICE_ID); - - if (!parser.getDeviceID().empty()) { - configCopy[deviceKey] = parser.getDeviceID(); + if (!parser.get_device_id().empty()) { + const std::string deviceKey = + device_supports_property(plugin.second, CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)) + ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) + : CONFIG_KEY(DEVICE_ID); + configCopy[deviceKey] = parser.get_device_id(); } } plugin.second.set_property(configCopy); @@ -1090,25 +1118,26 @@ const std::vector& ov::CoreImpl::GetExtensions() return extensions; } -bool ov::CoreImpl::device_supports_import_export(const std::string& deviceName) const { +bool ov::CoreImpl::device_supports_model_caching(const std::string& deviceName) const { auto parsed = parseDeviceNameIntoConfig(deviceName); - return device_supports_import_export(get_plugin(parsed._deviceName)); + return device_supports_model_caching(get_plugin(parsed._deviceName)); } -bool ov::CoreImpl::device_supports_property(const ov::Plugin& plugin, const std::string& key) const { +bool ov::CoreImpl::device_supports_property(const ov::Plugin& plugin, const ov::PropertyName& key) const { return util::contains(plugin.get_property(ov::supported_properties), key); } -bool ov::CoreImpl::device_supports_import_export(const ov::Plugin& plugin) const { +bool ov::CoreImpl::device_supports_model_caching(const ov::Plugin& plugin) const { auto supportedMetricKeys = plugin.get_property(METRIC_KEY(SUPPORTED_METRICS), {}).as>(); - auto it = std::find(supportedMetricKeys.begin(), supportedMetricKeys.end(), METRIC_KEY(IMPORT_EXPORT_SUPPORT)); - auto supported = - (it != supportedMetricKeys.end()) && plugin.get_property(METRIC_KEY(IMPORT_EXPORT_SUPPORT), {}).as(); + auto supported = util::contains(supportedMetricKeys, METRIC_KEY(IMPORT_EXPORT_SUPPORT)) && + plugin.get_property(METRIC_KEY(IMPORT_EXPORT_SUPPORT), {}).as(); if (!supported) { - if (device_supports_property(plugin, ov::device::capabilities.name())) { - supported = - util::contains(plugin.get_property(ov::device::capabilities), ov::device::capability::EXPORT_IMPORT); - } + supported = + device_supports_property(plugin, ov::device::capabilities) && + util::contains(plugin.get_property(ov::device::capabilities), ov::device::capability::EXPORT_IMPORT); + } + if (supported) { + supported = device_supports_property(plugin, ov::caching_properties); } return supported; } @@ -1125,7 +1154,7 @@ ov::SoPtr ov::CoreImpl::compile_model_and_cache(const std::s OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "CoreImpl::compile_model_and_cache"); ov::SoPtr execNetwork; execNetwork = compile_model_with_preprocess(plugin, model, context, parsedConfig); - if (cacheContent.cacheManager && device_supports_import_export(plugin)) { + if (cacheContent.cacheManager && device_supports_model_caching(plugin)) { try { // need to export network for further import from "cache" OV_ITT_SCOPE(FIRST_INFERENCE, InferenceEngine::itt::domains::IE_LT, "Core::compile_model::Export"); @@ -1196,45 +1225,36 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache( ov::AnyMap ov::CoreImpl::create_compile_config(const ov::Plugin& plugin, const ov::AnyMap& user_config) const { ov::AnyMap property_config; - ov::AnyMap compile_config; - // 0. Move TARGET_FALLBACK key to property_config - auto targetFallbackIt = user_config.find("TARGET_FALLBACK"); - if (targetFallbackIt == user_config.end()) { - targetFallbackIt = user_config.find(ov::device::priorities.name()); + // 0. Move ov::device::priorities / TARGET_FALLBACK key to property_config + auto device_priorities_it = user_config.find("TARGET_FALLBACK"); + if (device_priorities_it == user_config.end()) { + device_priorities_it = user_config.find(ov::device::priorities.name()); } - if (targetFallbackIt != user_config.end()) { - property_config[targetFallbackIt->first] = targetFallbackIt->second.as(); + if (device_priorities_it != user_config.end()) { + property_config[device_priorities_it->first] = device_priorities_it->second.as(); } // 1. Move DEVICE_ID key to property_config + const bool supports_device_id = device_supports_property(plugin, ov::device::id); auto deviceIt = user_config.find(ov::device::id.name()); if (deviceIt != user_config.end()) { property_config[deviceIt->first] = deviceIt->second.as(); - } else { - // we likely need to extract default device_id from the plugin, - // but we suppose when we call plugin.get_property it will provide the answer - // for the default device (e.g. DEVICE_ID = 0 for GPU) + } else if (supports_device_id) { + property_config[ov::device::id.name()] = plugin.get_property(ov::device::id, {}); } - // 2. Replace DEVICE_ID with DEVICE_ARCHITECTURE value to identify device - if (device_supports_property(plugin, ov::device::architecture.name())) { - compile_config[ov::device::architecture.name()] = - plugin.get_property(ov::device::architecture, property_config); - } else { - // Take device name if device does not support DEVICE_ARCHITECTURE metric - compile_config[ov::device::architecture.name()] = plugin.get_name(); - } + // 2. Extract config keys which affect compilation process + auto caching_props = plugin.get_property(ov::caching_properties, property_config); + OPENVINO_ASSERT(!caching_props.empty(), "ov::caching_properties returned by ", plugin.get_name(), " are empty"); - // 3. Extract config keys which affect compilation process - if (device_supports_property(plugin, ov::caching_properties.name())) { - auto cachingProps = plugin.get_property(ov::caching_properties); - for (const auto& prop : cachingProps) { - // user_config values have higher priority than plugin parameters - auto it = user_config.find(prop); - compile_config[prop] = it == user_config.end() ? plugin.get_property(prop, property_config) : it->second; - } + ov::AnyMap compile_config; + for (const auto& prop : caching_props) { + // user_config values have higher priority than plugin parameters + auto it = user_config.find(prop); + compile_config[prop] = it == user_config.end() ? plugin.get_property(prop, property_config) : it->second; } + return compile_config; } @@ -1261,7 +1281,7 @@ void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { std::lock_guard lock(_cacheConfigMutex); // fill global cache config _cacheConfig = CoreConfig::CacheConfig::create(it->second.as()); - // sets cache config per-device if it's set explicitly before + // sets cache config per-device if it's not set explicitly before for (auto& deviceCfg : _cacheConfigPerDevice) { deviceCfg.second = CoreConfig::CacheConfig::create(it->second.as()); } @@ -1281,6 +1301,13 @@ void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { _flag_allow_auto_batching = flag; config.erase(it); } + + it = config.find(ov::enable_mmap.name()); + if (it != config.end()) { + auto flag = it->second.as(); + _flag_enable_mmap = flag; + config.erase(it); + } } void ov::CoreImpl::CoreConfig::set_cache_dir_for_device(const std::string& dir, const std::string& name) { @@ -1297,6 +1324,10 @@ bool ov::CoreImpl::CoreConfig::get_allow_auto_batch() const { return _flag_allow_auto_batching; } +bool ov::CoreImpl::CoreConfig::get_enable_mmap() const { + return _flag_enable_mmap; +} + // Creating thread-safe copy of config including shared_ptr to ICacheManager // Passing empty or not-existing name will return global cache config ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::get_cache_config_for_device( diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 2277d70b9d0acd..4f8ab8e38dda1b 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -23,10 +23,6 @@ #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/threading/executor_manager.hpp" -#ifdef OPENVINO_STATIC_LIBRARY -# include "ie_plugins.hpp" -#endif - namespace ov { const std::string DEFAULT_DEVICE_NAME = "DEFAULT_DEVICE"; @@ -48,16 +44,13 @@ Parsed parseDeviceNameIntoConfig(const std::string& deviceName, const AnyMap& co * * @param device_name Target device * @param device_name_to_parse Device ID of property - * @return true if ov::device::properties(, ...) is applicable for device identified by 'device_name + * @return true if ov::device::properties(, ...) is applicable for device identified by + * 'device_name */ bool is_config_applicable(const std::string& device_name, const std::string& device_name_to_parse); -#ifndef OPENVINO_STATIC_LIBRARY - std::string findPluginXML(const std::string& xmlFile); -#endif - class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_this { private: mutable std::map plugins; @@ -92,16 +85,18 @@ class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_t bool get_allow_auto_batch() const; + bool get_enable_mmap() const; + // Creating thread-safe copy of config including shared_ptr to ICacheManager // Passing empty or not-existing name will return global cache config - CacheConfig get_cache_config_for_device(const ov::Plugin& plugin, - ov::AnyMap& parsedConfig) const; + CacheConfig get_cache_config_for_device(const ov::Plugin& plugin, ov::AnyMap& parsedConfig) const; private: mutable std::mutex _cacheConfigMutex; CacheConfig _cacheConfig; std::map _cacheConfigPerDevice; bool _flag_allow_auto_batching = true; + bool _flag_enable_mmap = true; }; struct CacheContent { @@ -158,31 +153,31 @@ class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_t const bool m_new_api; ov::SoPtr compile_model_and_cache(const std::shared_ptr& model, - ov::Plugin& plugin, - const ov::AnyMap& parsedConfig, - const ov::RemoteContext& context, - const CacheContent& cacheContent) const; + ov::Plugin& plugin, + const ov::AnyMap& parsedConfig, + const ov::RemoteContext& context, + const CacheContent& cacheContent) const; - static ov::SoPtr load_model_from_cache(const CacheContent& cacheContent, - ov::Plugin& plugin, - const ov::AnyMap& config, - const ov::RemoteContext& context, - std::function()> compile_model_lambda); + static ov::SoPtr load_model_from_cache( + const CacheContent& cacheContent, + ov::Plugin& plugin, + const ov::AnyMap& config, + const ov::RemoteContext& context, + std::function()> compile_model_lambda); - bool device_supports_import_export(const ov::Plugin& plugin) const; + bool device_supports_model_caching(const ov::Plugin& plugin) const; - bool device_supports_property(const ov::Plugin& plugin, const std::string& key) const; + bool device_supports_property(const ov::Plugin& plugin, const ov::PropertyName& key) const; OPENVINO_DEPRECATED("Don't use this method, it will be removed soon") bool device_supports_cache_dir(const ov::Plugin& plugin) const; ov::SoPtr compile_model_with_preprocess(ov::Plugin& plugin, - const std::shared_ptr& model, - const ov::RemoteContext& context, - const ov::AnyMap& config) const; + const std::shared_ptr& model, + const ov::RemoteContext& context, + const ov::AnyMap& config) const; - ov::AnyMap create_compile_config(const ov::Plugin& plugin, - const ov::AnyMap& origConfig) const; + ov::AnyMap create_compile_config(const ov::Plugin& plugin, const ov::AnyMap& origConfig) const; // Legacy API void AddExtensionUnsafe(const InferenceEngine::IExtensionPtr& extension) const; @@ -218,30 +213,10 @@ class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_t std::string& deviceName, ov::AnyMap& config) const; -#ifdef OPENVINO_STATIC_LIBRARY - - /** - * @brief Register plugins for devices using statically defined configuration - * @note The function supports UNICODE path - * @param static_registry a statically defined configuration with device / plugin information + /* + * @brief Register plugins according to the build configuration */ - void register_plugins_in_registry(const decltype(::getStaticPluginsRegistry())& static_registry) { - std::lock_guard lock(get_mutex()); - - for (const auto& plugin : static_registry) { - const auto& deviceName = plugin.first; - if (deviceName.find('.') != std::string::npos) { - IE_THROW() << "Device name must not contain dot '.' symbol"; - } - const auto& value = plugin.second; - ov::AnyMap config = any_copy(value.m_default_config); - PluginDescriptor desc{value.m_create_plugin_func, config, value.m_create_extension_func}; - pluginRegistry[deviceName] = desc; - add_mutex(deviceName); - } - } - -#endif + void register_compile_time_plugins(); // // ICore public API @@ -325,7 +300,7 @@ class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_t */ const std::vector& GetExtensions() const; - bool DeviceSupportsImportExport(const std::string& deviceName) const override; + bool DeviceSupportsModelCaching(const std::string& deviceName) const override; std::map GetVersions(const std::string& deviceName) const; @@ -369,7 +344,7 @@ class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_t void add_extension(const std::vector& extensions); - bool device_supports_import_export(const std::string& deviceName) const; + bool device_supports_model_caching(const std::string& deviceName) const; // ov::ICore std::shared_ptr read_model(const std::string& model, diff --git a/src/inference/src/dev/core_impl_ie.cpp b/src/inference/src/dev/core_impl_ie.cpp index cbf60f265ba209..1103fb274d07f2 100644 --- a/src/inference/src/dev/core_impl_ie.cpp +++ b/src/inference/src/dev/core_impl_ie.cpp @@ -18,6 +18,7 @@ #include "ngraph/op/constant.hpp" #include "ngraph/pass/constant_folding.hpp" #include "openvino/itt.hpp" +#include "openvino/runtime/device_id_parser.hpp" #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/itensor.hpp" #include "openvino/util/common_util.hpp" @@ -48,7 +49,12 @@ InferenceEngine::RemoteContext::Ptr ov::CoreImpl::GetDefaultContext(const std::s InferenceEngine::CNNNetwork ov::CoreImpl::ReadNetwork(const std::string& modelPath, const std::string& binPath) const { OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::IE_RT, "CoreImpl::ReadNetwork from file"); - return InferenceEngine::details::ReadNetwork(modelPath, binPath, extensions, ov_extensions, is_new_api()); + return InferenceEngine::details::ReadNetwork(modelPath, + binPath, + extensions, + ov_extensions, + is_new_api(), + coreConfig.get_enable_mmap()); } InferenceEngine::CNNNetwork ov::CoreImpl::ReadNetwork(const std::string& model, @@ -213,8 +219,8 @@ void ov::CoreImpl::AddExtension(const InferenceEngine::IExtensionPtr& extension) AddExtensionUnsafe(extension); } -bool ov::CoreImpl::DeviceSupportsImportExport(const std::string& deviceName) const { - return device_supports_import_export(deviceName); +bool ov::CoreImpl::DeviceSupportsModelCaching(const std::string& deviceName) const { + return device_supports_model_caching(deviceName); } std::map ov::CoreImpl::GetSupportedConfig(const std::string& deviceName, @@ -231,25 +237,25 @@ std::map ov::CoreImpl::GetVersions(const if (deviceName.find("HETERO") == 0) { auto pos = deviceName.find_first_of(":"); if (pos != std::string::npos) { - deviceNames = InferenceEngine::DeviceIDParser::getHeteroDevices(deviceName.substr(pos + 1)); + deviceNames = ov::DeviceIDParser::get_hetero_devices(deviceName.substr(pos + 1)); } deviceNames.push_back("HETERO"); } else if (deviceName.find("MULTI") == 0) { auto pos = deviceName.find_first_of(":"); if (pos != std::string::npos) { - deviceNames = InferenceEngine::DeviceIDParser::getMultiDevices(deviceName.substr(pos + 1)); + deviceNames = ov::DeviceIDParser::get_multi_devices(deviceName.substr(pos + 1)); } deviceNames.push_back("MULTI"); } else if (deviceName.find("AUTO") == 0) { auto pos = deviceName.find_first_of(":"); if (pos != std::string::npos) { - deviceNames = InferenceEngine::DeviceIDParser::getMultiDevices(deviceName.substr(pos + 1)); + deviceNames = ov::DeviceIDParser::get_multi_devices(deviceName.substr(pos + 1)); } deviceNames.emplace_back("AUTO"); } else if (deviceName.find("BATCH") == 0) { auto pos = deviceName.find_first_of(":"); if (pos != std::string::npos) { - deviceNames = {InferenceEngine::DeviceIDParser::getBatchDevice(deviceName.substr(pos + 1))}; + deviceNames = {ov::DeviceIDParser::get_batch_device(deviceName.substr(pos + 1))}; } deviceNames.push_back("BATCH"); } else { @@ -258,8 +264,8 @@ std::map ov::CoreImpl::GetVersions(const } for (auto&& deviceName_ : deviceNames) { - ie::DeviceIDParser parser(deviceName_); - std::string deviceNameLocal = parser.getDeviceName(); + ov::DeviceIDParser parser(deviceName_); + std::string deviceNameLocal = parser.get_device_name(); ov::Plugin cppPlugin = get_plugin(deviceNameLocal); diff --git a/src/inference/src/dev/device_id_parser.cpp b/src/inference/src/dev/device_id_parser.cpp index 73309ee1cc9fc8..ba64a1cf08b831 100644 --- a/src/inference/src/dev/device_id_parser.cpp +++ b/src/inference/src/dev/device_id_parser.cpp @@ -2,47 +2,51 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ie_icore.hpp" +#include "openvino/runtime/device_id_parser.hpp" -namespace InferenceEngine { +#include + +namespace ov { DeviceIDParser::DeviceIDParser(const std::string& deviceNameWithID) { - deviceName = deviceNameWithID; + m_device_name = deviceNameWithID; - auto pos = deviceName.find('.'); + auto pos = m_device_name.find('.'); if (pos != std::string::npos) { - deviceName = deviceNameWithID.substr(0, pos); - deviceID = deviceNameWithID.substr(pos + 1, deviceNameWithID.size()); + m_device_name = deviceNameWithID.substr(0, pos); + m_device_id = deviceNameWithID.substr(pos + 1, deviceNameWithID.size()); } } -std::string DeviceIDParser::getDeviceID() const { - return deviceID; +const std::string& DeviceIDParser::get_device_id() const { + return m_device_id; } -std::string DeviceIDParser::getDeviceName() const { - return deviceName; +const std::string& DeviceIDParser::get_device_name() const { + return m_device_name; } -std::vector DeviceIDParser::getHeteroDevices(std::string fallbackDevice) { +std::vector DeviceIDParser::get_hetero_devices(const std::string& fallbackDevice) { std::vector deviceNames; + std::string fallback_dev = fallbackDevice; std::string cdevice; char delimiter = ','; size_t pos = 0; - while ((pos = fallbackDevice.find(delimiter)) != std::string::npos) { - deviceNames.push_back(fallbackDevice.substr(0, pos)); - fallbackDevice.erase(0, pos + 1); + while ((pos = fallback_dev.find(delimiter)) != std::string::npos) { + deviceNames.push_back(fallback_dev.substr(0, pos)); + fallback_dev.erase(0, pos + 1); } - if (!fallbackDevice.empty()) - deviceNames.push_back(fallbackDevice); + if (!fallback_dev.empty()) + deviceNames.push_back(fallback_dev); return deviceNames; } -std::vector DeviceIDParser::getMultiDevices(std::string devicesList) { +std::vector DeviceIDParser::get_multi_devices(const std::string& devicesList) { + std::string dev_list = devicesList; std::set deviceNames; auto trim_request_info = [](const std::string& device_with_requests) { auto opening_bracket = device_with_requests.find_first_of('('); @@ -53,37 +57,37 @@ std::vector DeviceIDParser::getMultiDevices(std::string devicesList size_t pos = 0; // in addition to the list of devices, every device can have a #requests in the brackets e.g. "CPU(100)" // we skip the #requests info here - while ((pos = devicesList.find(delimiter)) != std::string::npos) { - auto d = devicesList.substr(0, pos); + while ((pos = dev_list.find(delimiter)) != std::string::npos) { + auto d = dev_list.substr(0, pos); if (d.find("BATCH") == 0) { deviceNames.insert("BATCH"); auto p = d.find_first_of(":"); if (p != std::string::npos) - deviceNames.insert(DeviceIDParser::getBatchDevice(d.substr(p + 1))); + deviceNames.insert(DeviceIDParser::get_batch_device(d.substr(p + 1))); } else { deviceNames.insert(trim_request_info(d)); } - devicesList.erase(0, pos + 1); + dev_list.erase(0, pos + 1); } - if (!devicesList.empty()) { - if (devicesList.find("BATCH") == 0) { + if (!dev_list.empty()) { + if (dev_list.find("BATCH") == 0) { deviceNames.insert("BATCH"); - auto p = devicesList.find_first_of(":"); + auto p = dev_list.find_first_of(":"); if (p != std::string::npos) - deviceNames.insert(DeviceIDParser::getBatchDevice(devicesList.substr(p + 1))); + deviceNames.insert(DeviceIDParser::get_batch_device(dev_list.substr(p + 1))); } else { - deviceNames.insert(trim_request_info(devicesList)); + deviceNames.insert(trim_request_info(dev_list)); } } return std::vector(deviceNames.begin(), deviceNames.end()); } -std::string DeviceIDParser::getBatchDevice(std::string device) { +std::string DeviceIDParser::get_batch_device(const std::string& device) { auto trim_request_info = [](const std::string& device_with_requests) { auto opening_bracket = device_with_requests.find_first_of('('); return device_with_requests.substr(0, opening_bracket); }; return trim_request_info(device); } -} // namespace InferenceEngine +} // namespace ov diff --git a/src/inference/src/dev/icompiled_model_wrapper.hpp b/src/inference/src/dev/icompiled_model_wrapper.hpp index 27d77724e93738..09c4fbb8dafd92 100644 --- a/src/inference/src/dev/icompiled_model_wrapper.hpp +++ b/src/inference/src/dev/icompiled_model_wrapper.hpp @@ -13,6 +13,7 @@ namespace InferenceEngine { class ICompiledModelWrapper : public ov::ICompiledModel { public: ICompiledModelWrapper(const std::shared_ptr& model); + virtual ~ICompiledModelWrapper() = default; std::shared_ptr create_infer_request() const override; void export_model(std::ostream& model) const override; diff --git a/src/inference/src/dev/iplugin_wrapper.hpp b/src/inference/src/dev/iplugin_wrapper.hpp index 403c48ce4478e8..0d40b75da55c60 100644 --- a/src/inference/src/dev/iplugin_wrapper.hpp +++ b/src/inference/src/dev/iplugin_wrapper.hpp @@ -21,6 +21,11 @@ class IPluginWrapper : public ov::IPlugin { */ IPluginWrapper(const std::shared_ptr& ptr); + /** + * @brief Destructor + */ + virtual ~IPluginWrapper() = default; + /** * @brief Create compiled model based on model and properties * diff --git a/src/inference/src/dev/threading/cpu_map_scheduling.cpp b/src/inference/src/dev/threading/cpu_map_scheduling.cpp new file mode 100644 index 00000000000000..1ba22fb3090d9a --- /dev/null +++ b/src/inference/src/dev/threading/cpu_map_scheduling.cpp @@ -0,0 +1,60 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/runtime/threading/cpu_map_scheduling.hpp" + +#include "ie_system_conf.h" + +namespace ov { + +std::vector> apply_scheduling_core_type(const ov::hint::SchedulingCoreType input_type, + const std::vector>& proc_type_table) { + std::vector> result_table = proc_type_table; + + switch (input_type) { + case ov::hint::SchedulingCoreType::ANY_CORE: + break; + case ov::hint::SchedulingCoreType::PCORE_ONLY: + if (proc_type_table[0][EFFICIENT_CORE_PROC] > 0) { + for (auto& i : result_table) { + i[ALL_PROC] -= i[EFFICIENT_CORE_PROC]; + i[EFFICIENT_CORE_PROC] = 0; + } + } + break; + case ov::hint::SchedulingCoreType::ECORE_ONLY: + if ((proc_type_table[0][EFFICIENT_CORE_PROC] > 0) && + (proc_type_table[0][EFFICIENT_CORE_PROC] != proc_type_table[0][ALL_PROC])) { + for (auto& i : result_table) { + i[ALL_PROC] -= i[MAIN_CORE_PROC] + i[HYPER_THREADING_PROC]; + i[MAIN_CORE_PROC] = 0; + i[HYPER_THREADING_PROC] = 0; + } + } + break; + default: + throw ov::Exception{"Unsupported core type!"}; + } + + return result_table; +} + +std::vector> apply_hyper_threading(bool input_value, + const bool input_changed, + const std::vector>& proc_type_table) { + std::vector> result_table = proc_type_table; + + if ((proc_type_table[0][HYPER_THREADING_PROC] > 0) && + (((!input_value) && input_changed) || ((!input_changed) && (proc_type_table.size() > 1)))) { + for (auto& i : result_table) { + i[ALL_PROC] -= i[HYPER_THREADING_PROC]; + i[HYPER_THREADING_PROC] = 0; + } + input_value = false; + } + + return result_table; +} + +} // namespace ov \ No newline at end of file diff --git a/src/inference/src/dev/threading/cpu_streams_executor.cpp b/src/inference/src/dev/threading/cpu_streams_executor.cpp index 26789a570fb44f..b1f588b7c0149b 100644 --- a/src/inference/src/dev/threading/cpu_streams_executor.cpp +++ b/src/inference/src/dev/threading/cpu_streams_executor.cpp @@ -29,6 +29,7 @@ struct CPUStreamsExecutor::Impl { int _threadBindingStep = 0; int _offset = 0; int _cpuIdxOffset = 0; + std::vector _cpu_ids; Observer(custom::task_arena& arena, CpuSet mask, int ncpus, @@ -36,18 +37,21 @@ struct CPUStreamsExecutor::Impl { const int threadsPerStream, const int threadBindingStep, const int threadBindingOffset, - const int cpuIdxOffset = 0) + const int cpuIdxOffset = 0, + const std::vector cpu_ids = {}) : custom::task_scheduler_observer(arena), _mask{std::move(mask)}, _ncpus(ncpus), _threadBindingStep(threadBindingStep), _offset{streamId * threadsPerStream + threadBindingOffset}, - _cpuIdxOffset(cpuIdxOffset) {} + _cpuIdxOffset(cpuIdxOffset), + _cpu_ids(cpu_ids) {} void on_scheduler_entry(bool) override { pin_thread_to_vacant_core(_offset + tbb::this_task_arena::current_thread_index(), _threadBindingStep, _ncpus, _mask, + _cpu_ids, _cpuIdxOffset); } void on_scheduler_exit(bool) override { @@ -72,6 +76,121 @@ struct CPUStreamsExecutor::Impl { _impl->_usedNumaNodes.size())) : _impl->_usedNumaNodes.at(_streamId % _impl->_usedNumaNodes.size()); #if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO + if (is_cpu_map_available()) { + init_stream(); + } else { + init_stream_legacy(); + } +#elif OV_THREAD == OV_THREAD_OMP + omp_set_num_threads(_impl->_config._threadsPerStream); + if (!check_open_mp_env_vars(false) && (ThreadBindingType::NONE != _impl->_config._threadBindingType)) { + CpuSet processMask; + int ncpus = 0; + std::tie(processMask, ncpus) = get_process_mask(); + if (nullptr != processMask) { + parallel_nt(_impl->_config._threadsPerStream, [&](int threadIndex, int threadsPerStream) { + int thrIdx = _streamId * _impl->_config._threadsPerStream + threadIndex + + _impl->_config._threadBindingOffset; + pin_thread_to_vacant_core(thrIdx, _impl->_config._threadBindingStep, ncpus, processMask); + }); + } + } +#elif OV_THREAD == OV_THREAD_SEQ + if (ThreadBindingType::NUMA == _impl->_config._threadBindingType) { + pin_current_thread_to_socket(_numaNodeId); + } else if (ThreadBindingType::CORES == _impl->_config._threadBindingType) { + CpuSet processMask; + int ncpus = 0; + std::tie(processMask, ncpus) = get_process_mask(); + if (nullptr != processMask) { + pin_thread_to_vacant_core(_streamId + _impl->_config._threadBindingOffset, + _impl->_config._threadBindingStep, + ncpus, + processMask); + } + } +#endif + } + ~Stream() { + { + std::lock_guard lock{_impl->_streamIdMutex}; + _impl->_streamIdQueue.push(_streamId); + } +#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO + set_cpu_used(_cpu_ids, NOT_USED); + if (nullptr != _observer) { + _observer->observe(false); + } +#endif + } + +#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO + void init_stream() { + std::lock_guard lock{_impl->_cpumap_mutex}; + const auto stream_id = _streamId >= _impl->_config._streams ? _impl->_config._streams - 1 : _streamId; + const auto concurrency = + _impl->any_cores + ? _impl->_config._threads + : (stream_id < _impl->_config._big_core_streams + _impl->_config._big_core_logic_streams + ? _impl->_config._threads_per_stream_big + : _impl->_config._threads_per_stream_small); + if (concurrency > 0 && (ThreadBindingType::CORES == _impl->_config._threadBindingType || + ThreadBindingType::NONE == _impl->_config._threadBindingType || _impl->any_cores || + _streamId >= _impl->_config._streams)) { + _taskArena.reset(new custom::task_arena{concurrency}); + } else if (ThreadBindingType::NUMA == _impl->_config._threadBindingType) { + _taskArena.reset(new custom::task_arena{custom::task_arena::constraints{_numaNodeId, concurrency}}); + } else if (ThreadBindingType::HYBRID_AWARE == _impl->_config._threadBindingType) { + const auto selected_core_type = + (stream_id < _impl->_config._big_core_streams + _impl->_config._big_core_logic_streams) + ? custom::info::core_types().back() + : custom::info::core_types().front(); +# ifdef _WIN32 + _taskArena.reset(new custom::task_arena{custom::task_arena::constraints{} + .set_core_type(selected_core_type) + .set_max_concurrency(concurrency)}); +# else + if (_impl->bind_cores) { + _taskArena.reset(new custom::task_arena{concurrency}); + } else { + _taskArena.reset(new custom::task_arena{custom::task_arena::constraints{} + .set_core_type(selected_core_type) + .set_max_concurrency(concurrency)}); + } +# endif + } + if (_impl->bind_cores && _streamId < _impl->_config._streams) { + const auto cpu_core_type = + _streamId < _impl->_config._big_core_streams + ? MAIN_CORE_PROC + : (_streamId < _impl->_config._big_core_streams + _impl->_config._big_core_logic_streams + ? HYPER_THREADING_PROC + : EFFICIENT_CORE_PROC); + const auto small_core_threads_3 = + cpu_core_type == EFFICIENT_CORE_PROC && concurrency == 3 && _impl->_config._small_core_streams > 1; + const auto num_cpus = small_core_threads_3 ? concurrency + 1 : concurrency; + _cpu_ids = reserve_available_cpus(cpu_core_type, num_cpus, _impl->_config._plugin_task); + if (_cpu_ids.size() > 0) { + CpuSet processMask; + int ncpus = 0; + std::tie(processMask, ncpus) = get_process_mask(); + if (nullptr != processMask) { + _observer.reset(new Observer{*_taskArena, + std::move(processMask), + ncpus, + 0, + concurrency, + 0, + 0, + 0, + _cpu_ids}); + _observer->observe(true); + } + } + } + } + + void init_stream_legacy() { const auto concurrency = (0 == _impl->_config._threadsPerStream) ? custom::task_arena::automatic : _impl->_config._threadsPerStream; if (ThreadBindingType::HYBRID_AWARE == _impl->_config._threadBindingType) { @@ -122,8 +241,8 @@ struct CPUStreamsExecutor::Impl { : streamId_wrapped; const auto thread_binding_step = hybrid_core ? (small_core ? _impl->_config._threadBindingStep : 2) : _impl->_config._threadBindingStep; - // Special handling of _threads_per_stream_small == 3, need to skip 4 (Four cores share one L2 cache - // on the small core), stream_id = 0, cpu_idx_offset cumulative plus 4 + // Special handling of _threads_per_stream_small == 3, need to skip 4 (Four cores share one L2 + // cache on the small core), stream_id = 0, cpu_idx_offset cumulative plus 4 const auto small_core_offset = small_core_skip ? _impl->_config._small_core_offset + (streamId_wrapped - big_core_streams) * 4 : _impl->_config._small_core_offset; @@ -132,10 +251,13 @@ struct CPUStreamsExecutor::Impl { // Prevent conflicts with system scheduling, so default cpu id on big core starts from 1 ? (small_core ? small_core_offset : (logic_core ? 0 : 1)) : 0; - +# ifdef _WIN32 _taskArena.reset(new custom::task_arena{custom::task_arena::constraints{} .set_core_type(selected_core_type) .set_max_concurrency(max_concurrency)}); +# else + _taskArena.reset(new custom::task_arena{max_concurrency}); +# endif CpuSet processMask; int ncpus = 0; std::tie(processMask, ncpus) = get_process_mask(); @@ -172,47 +294,8 @@ struct CPUStreamsExecutor::Impl { } } } -#elif OV_THREAD == OV_THREAD_OMP - omp_set_num_threads(_impl->_config._threadsPerStream); - if (!check_open_mp_env_vars(false) && (ThreadBindingType::NONE != _impl->_config._threadBindingType)) { - CpuSet processMask; - int ncpus = 0; - std::tie(processMask, ncpus) = get_process_mask(); - if (nullptr != processMask) { - parallel_nt(_impl->_config._threadsPerStream, [&](int threadIndex, int threadsPerStream) { - int thrIdx = _streamId * _impl->_config._threadsPerStream + threadIndex + - _impl->_config._threadBindingOffset; - pin_thread_to_vacant_core(thrIdx, _impl->_config._threadBindingStep, ncpus, processMask); - }); - } - } -#elif OV_THREAD == OV_THREAD_SEQ - if (ThreadBindingType::NUMA == _impl->_config._threadBindingType) { - pin_current_thread_to_socket(_numaNodeId); - } else if (ThreadBindingType::CORES == _impl->_config._threadBindingType) { - CpuSet processMask; - int ncpus = 0; - std::tie(processMask, ncpus) = get_process_mask(); - if (nullptr != processMask) { - pin_thread_to_vacant_core(_streamId + _impl->_config._threadBindingOffset, - _impl->_config._threadBindingStep, - ncpus, - processMask); - } - } -#endif } - ~Stream() { - { - std::lock_guard lock{_impl->_streamIdMutex}; - _impl->_streamIdQueue.push(_streamId); - } -#if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO - if (nullptr != _observer) { - _observer->observe(false); - } #endif - } Impl* _impl = nullptr; int _streamId = 0; @@ -222,6 +305,7 @@ struct CPUStreamsExecutor::Impl { #if OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO std::unique_ptr _taskArena; std::unique_ptr _observer; + std::vector _cpu_ids; #endif }; @@ -240,7 +324,24 @@ struct CPUStreamsExecutor::Impl { _usedNumaNodes = numaNodes; } #if (OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO) + if ((config._streams > static_cast(numaNodes.size()) && + ThreadBindingType::HYBRID_AWARE == config._threadBindingType) || + ThreadBindingType::CORES == config._threadBindingType) { + bind_cores = true; + if (_config._plugin_task >= PLUGIN_USED_START) { + _config._threads = _config._streams; + if (_config._threadPreferredCoreType == Config::PreferredCoreType::LITTLE) { + _config._small_core_streams = config._small_core_streams; + _config._threads_per_stream_small = 1; + } else { + _config._big_core_streams = config._small_core_streams; + _config._threads_per_stream_big = 1; + } + } + } if (ThreadBindingType::HYBRID_AWARE == config._threadBindingType) { + any_cores = config._streams > + config._big_core_streams + config._big_core_logic_streams + config._small_core_streams; const auto core_types = custom::info::core_types(); const auto num_core_phys = get_number_of_cpu_cores(); num_big_core_phys = get_number_of_cpu_cores(true); @@ -335,6 +436,7 @@ struct CPUStreamsExecutor::Impl { std::queue _streamIdQueue; std::vector _threads; std::mutex _mutex; + std::mutex _cpumap_mutex; std::condition_variable _queueCondVar; std::queue _taskQueue; bool _isStopped = false; @@ -348,6 +450,8 @@ struct CPUStreamsExecutor::Impl { using StreamIdToCoreTypes = std::vector>; StreamIdToCoreTypes total_streams_on_core_types; int num_big_core_phys; + bool any_cores = false; + bool bind_cores = false; #endif std::shared_ptr _exectorMgr; }; diff --git a/src/inference/src/dev/threading/istreams_executor.cpp b/src/inference/src/dev/threading/istreams_executor.cpp index eb6dad48aa52e4..e9e5b70e2d6199 100644 --- a/src/inference/src/dev/threading/istreams_executor.cpp +++ b/src/inference/src/dev/threading/istreams_executor.cpp @@ -73,6 +73,7 @@ void IStreamsExecutor::Config::set_property(const ov::AnyMap& property) { } } else if (key == CONFIG_KEY(CPU_THROUGHPUT_STREAMS)) { if (value.as() == CONFIG_VALUE(CPU_THROUGHPUT_NUMA)) { + _set_streams = true; _streams = static_cast(get_available_numa_nodes().size()); } else if (value.as() == CONFIG_VALUE(CPU_THROUGHPUT_AUTO)) { // bare minimum of streams (that evenly divides available number of cores) @@ -90,16 +91,21 @@ void IStreamsExecutor::Config::set_property(const ov::AnyMap& property) { IE_THROW() << "Wrong value for property key " << CONFIG_KEY(CPU_THROUGHPUT_STREAMS) << ". Expected only positive numbers (#streams)"; } + _set_streams = true; _streams = val_i; } } else if (key == ov::num_streams) { auto streams = value.as(); if (streams == ov::streams::NUMA) { + _set_streams = true; _streams = static_cast(get_available_numa_nodes().size()); } else if (streams == ov::streams::AUTO) { // bare minimum of streams (that evenly divides available number of cores) - _streams = get_default_num_streams(); + if (!is_cpu_map_available()) { + _streams = get_default_num_streams(); + } } else if (streams.num >= 0) { + _set_streams = true; _streams = streams.num; } else { OPENVINO_THROW("Wrong value for property key ", @@ -147,6 +153,19 @@ void IStreamsExecutor::Config::set_property(const ov::AnyMap& property) { << ". Expected only non negative numbers (#streams)"; } _big_core_streams = val_i; + } else if (key == CONFIG_KEY_INTERNAL(BIG_CORE_LOGIC_STREAMS)) { + int val_i; + try { + val_i = value.as(); + } catch (const std::exception&) { + IE_THROW() << "Wrong value for HYBRID_AWARE key " << CONFIG_KEY_INTERNAL(BIG_CORE_LOGIC_STREAMS) + << ". Expected only non negative numbers (#streams)"; + } + if (val_i < 0) { + IE_THROW() << "Wrong value for HYBRID_AWARE key " << CONFIG_KEY_INTERNAL(BIG_CORE_LOGIC_STREAMS) + << ". Expected only non negative numbers (#streams)"; + } + _big_core_logic_streams = val_i; } else if (key == CONFIG_KEY_INTERNAL(SMALL_CORE_STREAMS)) { int val_i; try { @@ -221,6 +240,7 @@ ov::Any IStreamsExecutor::Config::get_property(const std::string& key) const { CONFIG_KEY(CPU_THREADS_NUM), CONFIG_KEY_INTERNAL(CPU_THREADS_PER_STREAM), CONFIG_KEY_INTERNAL(BIG_CORE_STREAMS), + CONFIG_KEY_INTERNAL(BIG_CORE_LOGIC_STREAMS), CONFIG_KEY_INTERNAL(SMALL_CORE_STREAMS), CONFIG_KEY_INTERNAL(THREADS_PER_STREAM_BIG), CONFIG_KEY_INTERNAL(THREADS_PER_STREAM_SMALL), @@ -265,6 +285,8 @@ ov::Any IStreamsExecutor::Config::get_property(const std::string& key) const { return {std::to_string(_threadsPerStream)}; } else if (key == CONFIG_KEY_INTERNAL(BIG_CORE_STREAMS)) { return {std::to_string(_big_core_streams)}; + } else if (key == CONFIG_KEY_INTERNAL(BIG_CORE_LOGIC_STREAMS)) { + return {std::to_string(_big_core_logic_streams)}; } else if (key == CONFIG_KEY_INTERNAL(SMALL_CORE_STREAMS)) { return {std::to_string(_small_core_streams)}; } else if (key == CONFIG_KEY_INTERNAL(THREADS_PER_STREAM_BIG)) { diff --git a/src/inference/src/dev/threading/thread_affinity.cpp b/src/inference/src/dev/threading/thread_affinity.cpp index 86b25c78f4774d..1e74c5cd055b63 100644 --- a/src/inference/src/dev/threading/thread_affinity.cpp +++ b/src/inference/src/dev/threading/thread_affinity.cpp @@ -47,26 +47,37 @@ bool pin_current_thread_by_mask(int ncores, const CpuSet& procMask) { return 0 == sched_setaffinity(0, CPU_ALLOC_SIZE(ncores), procMask.get()); } -bool pin_thread_to_vacant_core(int thrIdx, int hyperthreads, int ncores, const CpuSet& procMask, int cpuIdxOffset) { +bool pin_thread_to_vacant_core(int thrIdx, + int hyperthreads, + int ncores, + const CpuSet& procMask, + const std::vector& cpu_ids, + int cpuIdxOffset) { if (procMask == nullptr) return false; const size_t size = CPU_ALLOC_SIZE(ncores); const int num_cpus = CPU_COUNT_S(size, procMask.get()); thrIdx %= num_cpus; // To limit unique number in [; num_cpus-1] range - // Place threads with specified step - int cpu_idx = cpuIdxOffset; - for (int i = 0, offset = 0; i < thrIdx; ++i) { - cpu_idx += hyperthreads; - if (cpu_idx >= num_cpus) - cpu_idx = ++offset; - } - // Find index of 'cpu_idx'-th bit that equals to 1 - int mapped_idx = cpuIdxOffset - 1; - while (cpu_idx >= cpuIdxOffset) { - mapped_idx++; - if (CPU_ISSET_S(mapped_idx, size, procMask.get())) - --cpu_idx; + int mapped_idx; + if (cpu_ids.size() > 0) { + mapped_idx = cpu_ids[thrIdx]; + } else { + // Place threads with specified step + int cpu_idx = cpuIdxOffset; + for (int i = 0, offset = 0; i < thrIdx; ++i) { + cpu_idx += hyperthreads; + if (cpu_idx >= num_cpus) + cpu_idx = ++offset; + } + + // Find index of 'cpu_idx'-th bit that equals to 1 + mapped_idx = cpuIdxOffset - 1; + while (cpu_idx >= cpuIdxOffset) { + mapped_idx++; + if (CPU_ISSET_S(mapped_idx, size, procMask.get())) + --cpu_idx; + } } CpuSet targetMask{CPU_ALLOC(ncores)}; @@ -105,7 +116,12 @@ std::tuple get_process_mask() { } void release_process_mask(cpu_set_t*) {} -bool pin_thread_to_vacant_core(int thrIdx, int hyperthreads, int ncores, const CpuSet& procMask, int cpuIdxOffset) { +bool pin_thread_to_vacant_core(int thrIdx, + int hyperthreads, + int ncores, + const CpuSet& procMask, + const std::vector& cpu_ids, + int cpuIdxOffset) { return false; } bool pin_current_thread_by_mask(int ncores, const CpuSet& procMask) { diff --git a/src/inference/src/dev/threading/thread_affinity.hpp b/src/inference/src/dev/threading/thread_affinity.hpp index 562b128a787fd3..92f1bf9b1800e8 100644 --- a/src/inference/src/dev/threading/thread_affinity.hpp +++ b/src/inference/src/dev/threading/thread_affinity.hpp @@ -6,6 +6,7 @@ #include #include +#include #if !(defined(__APPLE__) || defined(__EMSCRIPTEN__) || defined(_WIN32)) # include @@ -68,6 +69,7 @@ bool pin_thread_to_vacant_core(int thrIdx, int hyperThreads, int ncores, const CpuSet& processMask, + const std::vector& cpu_ids = {}, int cpuIdxOffset = 0); /** diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 799e284f58ba38..139c12d763145c 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -40,16 +40,13 @@ #include "openvino/op/result.hpp" #include "openvino/runtime/compiled_model.hpp" #include "openvino/runtime/core.hpp" +#include "openvino/runtime/device_id_parser.hpp" #include "openvino/util/common_util.hpp" #include "openvino/util/file_util.hpp" #include "openvino/util/shared_object.hpp" #include "so_extension.hpp" #include "xml_parse_utils.h" -#ifdef OPENVINO_STATIC_LIBRARY -# include "ie_plugins.hpp" -#endif - using namespace InferenceEngine::PluginConfigParams; using namespace InferenceEngine; using namespace std::placeholders; @@ -90,13 +87,12 @@ class Core::Impl : public ov::CoreImpl { Core::Core(const std::string& xmlConfigFile) { _impl = std::make_shared(); -#ifdef OPENVINO_STATIC_LIBRARY - _impl->register_plugins_in_registry(::getStaticPluginsRegistry()); -#else - // If XML is default, load default plugins by absolute paths - auto loadByAbsPath = xmlConfigFile.empty(); - _impl->register_plugins_in_registry(ov::findPluginXML(xmlConfigFile), loadByAbsPath); -#endif + std::string xmlConfigFile_ = ov::findPluginXML(xmlConfigFile); + if (!xmlConfigFile_.empty()) + // If XML is default, load default plugins by absolute paths + _impl->register_plugins_in_registry(xmlConfigFile_, xmlConfigFile.empty()); + // Load plugins from pre-compiled list + _impl->register_compile_time_plugins(); } std::map Core::GetVersions(const std::string& deviceName) const { @@ -251,8 +247,8 @@ ExecutableNetwork Core::ImportNetwork(std::istream& networkModel, } std::string deviceName_ = context->getDeviceName(); - DeviceIDParser device(deviceName_); - std::string deviceName = device.getDeviceName(); + ov::DeviceIDParser device(deviceName_); + std::string deviceName = device.get_device_name(); auto parsed = ov::parseDeviceNameIntoConfig(deviceName, ov::any_copy(config)); auto exec = _impl->get_plugin(deviceName) @@ -350,8 +346,8 @@ void Core::RegisterPlugins(const std::string& xmlConfigFile) { } void Core::UnregisterPlugin(const std::string& deviceName_) { - DeviceIDParser parser(deviceName_); - std::string deviceName = parser.getDeviceName(); + ov::DeviceIDParser parser(deviceName_); + std::string deviceName = parser.get_device_name(); _impl->unload_plugin(deviceName); } diff --git a/src/inference/src/ie_network_reader.cpp b/src/inference/src/ie_network_reader.cpp index 6baf232df80f57..09b39ccaf0dd98 100644 --- a/src/inference/src/ie_network_reader.cpp +++ b/src/inference/src/ie_network_reader.cpp @@ -47,7 +47,7 @@ class ExtensionWrapper : public ov::LegacyOpExtension { : m_ext(ext), m_opset_name(opset), m_type(name), - m_ext_type(m_type.c_str(), 0, m_opset_name.c_str()) {} + m_ext_type(m_type.c_str(), m_opset_name.c_str()) {} const ov::DiscreteTypeInfo& get_type_info() const override { return m_ext_type; @@ -418,7 +418,8 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath, const std::string& binPath, const std::vector& exts, const std::vector& ov_exts, - bool newAPI) { + bool newAPI, + bool enable_mmap) { #ifdef ENABLE_IR_V7_READER // IR v7 obsolete code { @@ -457,6 +458,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath, #endif params.emplace_back(weights_path); } + params.emplace_back(enable_mmap); FE = manager.load_by_model(params); if (FE) { diff --git a/src/inference/src/ie_network_reader.hpp b/src/inference/src/ie_network_reader.hpp index 9b2aec6a4a240e..bdf3c5e7d8c72f 100644 --- a/src/inference/src/ie_network_reader.hpp +++ b/src/inference/src/ie_network_reader.hpp @@ -22,13 +22,15 @@ namespace details { * @param exts vector with extensions * @param ov_exts vector with OpenVINO extensions * @param newAPI Whether this function is called from OpenVINO 2.0 API + * @param enable_mmap boolean to enable/disable `mmap` use in Frontend * @return CNNNetwork */ CNNNetwork ReadNetwork(const std::string& modelPath, const std::string& binPath, const std::vector& exts, const std::vector& ov_exts, - bool newAPI); + bool newAPI, + bool enable_mmap); /** * @brief Reads IR xml and bin (with the same name) files * @param model string with IR diff --git a/src/inference/src/os/lin/lin_system_conf.cpp b/src/inference/src/os/lin/lin_system_conf.cpp index d60c0410660755..3fa4e8d659a2eb 100644 --- a/src/inference/src/os/lin/lin_system_conf.cpp +++ b/src/inference/src/os/lin/lin_system_conf.cpp @@ -8,8 +8,6 @@ #include #include -#include -#include #include #include @@ -20,82 +18,72 @@ namespace ov { -struct CPU { - int _processors = 0; - int _sockets = 0; - int _cores = 0; +void CPU::init_cpu(CPU& cpu) { + std::vector> system_info_table; - std::vector> _proc_type_table; - std::vector> _cpu_mapping_table; + auto GetCatchInfoLinux = [&]() { + cpu._processors = sysconf(_SC_NPROCESSORS_ONLN); + system_info_table.resize(cpu._processors, std::vector(3)); - CPU() { - std::vector> system_info_table; + for (int n = 0; n < cpu._processors; n++) { + for (int m = 0; m < 3; m++) { + int Ln = (m == 0) ? m : m + 1; - auto GetCatchInfoLinux = [&]() { - _processors = sysconf(_SC_NPROCESSORS_ONLN); - system_info_table.resize(_processors, std::vector(3)); - - for (int n = 0; n < _processors; n++) { - for (int m = 0; m < 3; m++) { - int Ln = (m == 0) ? m : m + 1; - - std::ifstream cache_file("/sys/devices/system/cpu/cpu" + std::to_string(n) + "/cache/index" + - std::to_string(Ln) + "/shared_cpu_list"); - if (!cache_file.is_open()) { - return -1; - } - std::string cache_info; - std::getline(cache_file, cache_info); - system_info_table[n][m] += cache_info; + std::ifstream cache_file("/sys/devices/system/cpu/cpu" + std::to_string(n) + "/cache/index" + + std::to_string(Ln) + "/shared_cpu_list"); + if (!cache_file.is_open()) { + return -1; } + std::string cache_info; + std::getline(cache_file, cache_info); + system_info_table[n][m] += cache_info; } - return 0; - }; - - if (!GetCatchInfoLinux()) { - parse_processor_info_linux(_processors, - system_info_table, - _sockets, - _cores, - _proc_type_table, - _cpu_mapping_table); - } else { - /*Previous CPU resource based on calculation*/ - std::ifstream cpuinfo("/proc/cpuinfo"); - std::vector processors; - std::map sockets; - int socketId = 0; - while (!cpuinfo.eof()) { - std::string line; - std::getline(cpuinfo, line); - if (line.empty()) - continue; - auto delimeter = line.find(':'); - auto key = line.substr(0, delimeter); - auto value = line.substr(delimeter + 1); - if (0 == key.find("processor")) { - processors.emplace_back(std::stoi(value)); - } - if (0 == key.find("physical id")) { - socketId = std::stoi(value); - } - if (0 == key.find("cpu cores")) { - sockets[socketId] = std::stoi(value); - } + } + return 0; + }; + + if (!GetCatchInfoLinux()) { + parse_processor_info_linux(cpu._processors, + system_info_table, + cpu._sockets, + cpu._cores, + cpu._proc_type_table, + cpu._cpu_mapping_table); + } else { + /*Previous CPU resource based on calculation*/ + std::ifstream cpuinfo("/proc/cpuinfo"); + std::vector processors; + std::map sockets; + int socketId = 0; + while (!cpuinfo.eof()) { + std::string line; + std::getline(cpuinfo, line); + if (line.empty()) + continue; + auto delimeter = line.find(':'); + auto key = line.substr(0, delimeter); + auto value = line.substr(delimeter + 1); + if (0 == key.find("processor")) { + processors.emplace_back(std::stoi(value)); } - _processors = processors.size(); - _sockets = sockets.size(); - for (auto&& socket : sockets) { - _cores += socket.second; + if (0 == key.find("physical id")) { + socketId = std::stoi(value); } - if (_cores == 0) { - _cores = _processors; + if (0 == key.find("cpu cores")) { + sockets[socketId] = std::stoi(value); } } - std::vector>().swap(system_info_table); + cpu._processors = processors.size(); + cpu._sockets = sockets.size(); + for (auto&& socket : sockets) { + cpu._cores += socket.second; + } + if (cpu._cores == 0) { + cpu._cores = cpu._processors; + } } -}; -static CPU cpu; + std::vector>().swap(system_info_table); +} void parse_processor_info_linux(const int _processors, const std::vector> system_info_table, @@ -242,42 +230,4 @@ void parse_processor_info_linux(const int _processors, } }; -#if !((OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO)) -std::vector get_available_numa_nodes() { - std::vector nodes((0 == cpu._sockets) ? 1 : cpu._sockets); - std::iota(std::begin(nodes), std::end(nodes), 0); - return nodes; -} -#endif -int get_number_of_cpu_cores(bool bigCoresOnly) { - unsigned numberOfProcessors = cpu._processors; - unsigned totalNumberOfCpuCores = cpu._cores; - OPENVINO_ASSERT(totalNumberOfCpuCores != 0); - cpu_set_t usedCoreSet, currentCoreSet, currentCpuSet; - CPU_ZERO(¤tCpuSet); - CPU_ZERO(&usedCoreSet); - CPU_ZERO(¤tCoreSet); - - sched_getaffinity(0, sizeof(currentCpuSet), ¤tCpuSet); - - for (unsigned processorId = 0u; processorId < numberOfProcessors; processorId++) { - if (CPU_ISSET(processorId, ¤tCpuSet)) { - unsigned coreId = processorId % totalNumberOfCpuCores; - if (!CPU_ISSET(coreId, &usedCoreSet)) { - CPU_SET(coreId, &usedCoreSet); - CPU_SET(processorId, ¤tCoreSet); - } - } - } - int phys_cores = CPU_COUNT(¤tCoreSet); -#if (OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO) - auto core_types = custom::info::core_types(); - if (bigCoresOnly && core_types.size() > 1) /*Hybrid CPU*/ { - phys_cores = custom::info::default_concurrency( - custom::task_arena::constraints{}.set_core_type(core_types.back()).set_max_threads_per_core(1)); - } -#endif - return phys_cores; -} - } // namespace ov diff --git a/src/inference/src/os/win/win_system_conf.cpp b/src/inference/src/os/win/win_system_conf.cpp index 58fee804cc02df..f2add2736da529 100644 --- a/src/inference/src/os/win/win_system_conf.cpp +++ b/src/inference/src/os/win/win_system_conf.cpp @@ -17,51 +17,53 @@ namespace ov { -struct CPU { - int _processors = 0; - int _sockets = 0; - int _cores = 0; - - std::vector> _proc_type_table; - std::vector> _cpu_mapping_table; - - CPU() { - DWORD len = 0; - if (GetLogicalProcessorInformationEx(RelationAll, nullptr, &len) || - GetLastError() != ERROR_INSUFFICIENT_BUFFER) { - return; - } - - std::shared_ptr base_shared_ptr(new char[len]); - char* base_ptr = base_shared_ptr.get(); - if (!GetLogicalProcessorInformationEx(RelationAll, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)base_ptr, &len)) { - return; - } - - _processors = GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS); +void CPU::init_cpu(CPU& cpu) { + DWORD len = 0; + if (GetLogicalProcessorInformationEx(RelationAll, nullptr, &len) || GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + return; + } - parse_processor_info_win(base_ptr, len, _processors, _sockets, _cores, _proc_type_table, _cpu_mapping_table); + std::unique_ptr base_shared_ptr(new char[len]); + char* base_ptr = base_shared_ptr.get(); + if (!GetLogicalProcessorInformationEx(RelationAll, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)base_ptr, &len)) { + return; } -}; -static CPU cpu; + + parse_processor_info_win(base_ptr, + len, + cpu._processors, + cpu._sockets, + cpu._cores, + cpu._proc_type_table, + cpu._cpu_mapping_table); +} void parse_processor_info_win(const char* base_ptr, const unsigned long len, - const int _processors, + int& _processors, int& _sockets, int& _cores, std::vector>& _proc_type_table, std::vector>& _cpu_mapping_table) { - _cpu_mapping_table.resize(_processors, std::vector(CPU_MAP_TABLE_SIZE, -1)); - std::vector list; + std::vector proc_info; + + std::vector proc_init_line(PROC_TYPE_TABLE_SIZE, 0); + std::vector cpu_init_line(CPU_MAP_TABLE_SIZE, -1); char* info_ptr = (char*)base_ptr; int list_len = 0; int base_proc = 0; - int proc_count = 0; int group = 0; + + int group_start = 0; + int group_end = 0; + int group_id = 0; + int group_type = 0; + + _processors = 0; _sockets = -1; + _cores = 0; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = NULL; @@ -82,8 +84,6 @@ void parse_processor_info_win(const char* base_ptr, return; }; - std::vector line_value_0(PROC_TYPE_TABLE_SIZE, 0); - for (; info_ptr < base_ptr + len; info_ptr += (DWORD)info->Size) { info = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)info_ptr; @@ -91,56 +91,66 @@ void parse_processor_info_win(const char* base_ptr, _sockets++; MaskToList(info->Processor.GroupMask->Mask); if (0 == _sockets) { - _proc_type_table.push_back(line_value_0); + _proc_type_table.push_back(proc_init_line); } else { _proc_type_table.push_back(_proc_type_table[0]); - _proc_type_table[0] = line_value_0; + _proc_type_table[0] = proc_init_line; } } else if (info->Relationship == RelationProcessorCore) { MaskToList(info->Processor.GroupMask->Mask); - if (proc_count >= _processors) { - break; - } - if (0 == list[0]) { - base_proc = proc_count; + base_proc = _processors; } if (2 == list_len) { - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_PROCESSOR_ID] = list[0] + base_proc; - _cpu_mapping_table[list[1] + base_proc][CPU_MAP_PROCESSOR_ID] = list[1] + base_proc; - - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_SOCKET_ID] = _sockets; - _cpu_mapping_table[list[1] + base_proc][CPU_MAP_SOCKET_ID] = _sockets; - - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_CORE_ID] = _cores; - _cpu_mapping_table[list[1] + base_proc][CPU_MAP_CORE_ID] = _cores; - - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_CORE_TYPE] = HYPER_THREADING_PROC; - _cpu_mapping_table[list[1] + base_proc][CPU_MAP_CORE_TYPE] = MAIN_CORE_PROC; - - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_GROUP_ID] = group; - _cpu_mapping_table[list[1] + base_proc][CPU_MAP_GROUP_ID] = group; + proc_info = cpu_init_line; + proc_info[CPU_MAP_PROCESSOR_ID] = list[0] + base_proc; + proc_info[CPU_MAP_SOCKET_ID] = _sockets; + proc_info[CPU_MAP_CORE_ID] = _cores; + proc_info[CPU_MAP_CORE_TYPE] = HYPER_THREADING_PROC; + proc_info[CPU_MAP_GROUP_ID] = group; + _cpu_mapping_table.push_back(proc_info); + + proc_info = cpu_init_line; + proc_info[CPU_MAP_PROCESSOR_ID] = list[1] + base_proc; + proc_info[CPU_MAP_SOCKET_ID] = _sockets; + proc_info[CPU_MAP_CORE_ID] = _cores; + proc_info[CPU_MAP_CORE_TYPE] = MAIN_CORE_PROC; + proc_info[CPU_MAP_GROUP_ID] = group; + _cpu_mapping_table.push_back(proc_info); _proc_type_table[0][MAIN_CORE_PROC]++; _proc_type_table[0][HYPER_THREADING_PROC]++; group++; } else { - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_PROCESSOR_ID] = list[0] + base_proc; - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_SOCKET_ID] = _sockets; - _cpu_mapping_table[list[0] + base_proc][CPU_MAP_CORE_ID] = _cores; + proc_info = cpu_init_line; + proc_info[CPU_MAP_PROCESSOR_ID] = list[0] + base_proc; + proc_info[CPU_MAP_SOCKET_ID] = _sockets; + proc_info[CPU_MAP_CORE_ID] = _cores; + if ((_processors > group_start) && (_processors <= group_end)) { + proc_info[CPU_MAP_CORE_TYPE] = group_type; + proc_info[CPU_MAP_GROUP_ID] = group_id; + _proc_type_table[0][group_type]++; + } + _cpu_mapping_table.push_back(proc_info); } _proc_type_table[0][ALL_PROC] += list_len; - proc_count += list_len; + _processors += list_len; _cores++; } else if ((info->Relationship == RelationCache) && (info->Cache.Level == 2)) { MaskToList(info->Cache.GroupMask.Mask); if (4 == list_len) { - for (int m = 0; m < list_len; m++) { + if (_processors < list[list_len - 1] + base_proc) { + group_start = list[0]; + group_end = list[list_len - 1]; + group_id = group; + group_type = EFFICIENT_CORE_PROC; + } + for (int m = 0; m < _processors - list[0]; m++) { _cpu_mapping_table[list[m] + base_proc][CPU_MAP_CORE_TYPE] = EFFICIENT_CORE_PROC; _cpu_mapping_table[list[m] + base_proc][CPU_MAP_GROUP_ID] = group; _proc_type_table[0][EFFICIENT_CORE_PROC]++; @@ -158,7 +168,7 @@ void parse_processor_info_win(const char* base_ptr, _sockets++; if (_sockets > 1) { _proc_type_table.push_back(_proc_type_table[0]); - _proc_type_table[0] = line_value_0; + _proc_type_table[0] = proc_init_line; for (int m = 1; m <= _sockets; m++) { for (int n = 0; n < PROC_TYPE_TABLE_SIZE; n++) { diff --git a/src/inference/src/streams_executor.hpp b/src/inference/src/streams_executor.hpp index 4bea102dbceb63..abe6c5ca5d2e5e 100644 --- a/src/inference/src/streams_executor.hpp +++ b/src/inference/src/streams_executor.hpp @@ -8,15 +8,36 @@ */ #pragma once +#include #include #include +#include "dev/threading/parallel_custom_arena.hpp" + namespace ov { +struct CPU { + int _processors = 0; + int _sockets = 0; + int _cores = 0; + std::vector> _proc_type_table; + std::vector> _cpu_mapping_table; + std::mutex _cpu_mutex; + std::mutex _task_mutex; + int _plugin_status = PLUGIN_USED_START; + int _socket_idx = 0; + int _num_threads = 0; + + CPU() { + _num_threads = parallel_get_max_threads(); + init_cpu(*this); + } + void init_cpu(CPU& cpu); +}; + #ifdef __linux__ /** * @brief Parse processors infomation on Linux - * @ingroup ie_dev_api_system_conf * @param[in] _processors total number for processors in system. * @param[in] _system_info_table system information for this platform. * @param[out] _sockets total number for sockets in system @@ -36,10 +57,9 @@ void parse_processor_info_linux(const int _processors, #if (defined(_WIN32) || defined(_WIN64)) /** * @brief Parse processors infomation on Windows - * @ingroup ie_dev_api_system_conf * @param[in] base_ptr buffer object pointer of Windows system infomation * @param[in] len buffer object length of Windows system infomation - * @param[in] _processors total number for processors in system. + * @param[out] _processors total number for processors in system. * @param[out] _sockets total number for sockets in system * @param[out] _cores total number for physical CPU cores in system * @param[out] _proc_type_table summary table of number of processors per type @@ -48,7 +68,7 @@ void parse_processor_info_linux(const int _processors, */ void parse_processor_info_win(const char* base_ptr, const unsigned long len, - const int _processors, + int& _processors, int& _sockets, int& _cores, std::vector>& _proc_type_table, diff --git a/src/inference/src/system_conf.cpp b/src/inference/src/system_conf.cpp index 3de87e2e86960d..47646e16e8ddf5 100644 --- a/src/inference/src/system_conf.cpp +++ b/src/inference/src/system_conf.cpp @@ -6,10 +6,17 @@ #include #include +#include +#include +#include +#include +#include #include #include "dev/threading/parallel_custom_arena.hpp" +#include "ie_common.h" #include "openvino/core/visibility.hpp" +#include "streams_executor.hpp" #define XBYAK_NO_OP_NAMES #define XBYAK_UNDEF_JNL @@ -155,7 +162,161 @@ std::vector get_available_numa_nodes() { int get_number_of_logical_cpu_cores(bool) { return parallel_get_max_threads(); } +std::vector> get_num_available_cpu_cores() { + return {{-1}}; +} +bool is_cpu_map_available() { + return false; +} +std::vector reserve_available_cpus(const ColumnOfProcessorTypeTable core_type, + const int num_cpus, + const int seek_status, + const int reset_status, + const bool reserve_logic_core) { + return {}; +} +std::vector get_logic_cores(const std::vector cpu_ids) { + return {}; +} +void set_cpu_used(std::vector& cpu_ids, int used) {} + #else + +static CPU cpu; + +# ifndef _WIN32 +int get_number_of_cpu_cores(bool bigCoresOnly) { + unsigned numberOfProcessors = cpu._processors; + unsigned totalNumberOfCpuCores = cpu._cores; + IE_ASSERT(totalNumberOfCpuCores != 0); + cpu_set_t usedCoreSet, currentCoreSet, currentCpuSet; + CPU_ZERO(¤tCpuSet); + CPU_ZERO(&usedCoreSet); + CPU_ZERO(¤tCoreSet); + + sched_getaffinity(0, sizeof(currentCpuSet), ¤tCpuSet); + + for (unsigned processorId = 0u; processorId < numberOfProcessors; processorId++) { + if (CPU_ISSET(processorId, ¤tCpuSet)) { + unsigned coreId = processorId % totalNumberOfCpuCores; + if (!CPU_ISSET(coreId, &usedCoreSet)) { + CPU_SET(coreId, &usedCoreSet); + CPU_SET(processorId, ¤tCoreSet); + } + } + } + int phys_cores = CPU_COUNT(¤tCoreSet); +# if (OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO) + auto core_types = custom::info::core_types(); + if (bigCoresOnly && core_types.size() > 1) /*Hybrid CPU*/ { + phys_cores = custom::info::default_concurrency( + custom::task_arena::constraints{}.set_core_type(core_types.back()).set_max_threads_per_core(1)); + } +# endif + return phys_cores; +} + +# if !((OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO)) +std::vector get_available_numa_nodes() { + std::vector nodes((0 == cpu._sockets) ? 1 : cpu._sockets); + std::iota(std::begin(nodes), std::end(nodes), 0); + return nodes; +} +# endif +# endif + +std::vector> get_num_available_cpu_cores() { + return cpu._proc_type_table; +} + +bool is_cpu_map_available() { + return cpu._proc_type_table.size() > 0 && cpu._num_threads == cpu._proc_type_table[0][ALL_PROC]; +} + +std::vector reserve_available_cpus(const ColumnOfProcessorTypeTable core_type, + const int num_cpus, + const int seek_status, + const int reset_status, + const bool reserve_logic_core) { + std::lock_guard lock{cpu._cpu_mutex}; + std::vector cpu_ids; + int socket = -1; + if (reset_status >= PLUGIN_USED_START && cpu._sockets > 1) { + socket = cpu._socket_idx; + cpu._socket_idx = (cpu._socket_idx + 1) % cpu._sockets; + } + if (core_type < PROC_TYPE_TABLE_SIZE && core_type >= ALL_PROC) { + for (int i = 0; i < cpu._processors; i++) { + if (cpu._cpu_mapping_table[i][CPU_MAP_CORE_TYPE] == core_type && + cpu._cpu_mapping_table[i][CPU_MAP_USED_FLAG] == seek_status && + (socket < 0 || (socket >= 0 && cpu._cpu_mapping_table[i][CPU_MAP_SOCKET_ID] == socket))) { + cpu_ids.push_back(cpu._cpu_mapping_table[i][CPU_MAP_PROCESSOR_ID]); + } + if (static_cast(cpu_ids.size()) == num_cpus) { + break; + } + } + if (reserve_logic_core) { + auto logic_ids = get_logic_cores(cpu_ids); + cpu_ids.insert(cpu_ids.end(), logic_ids.begin(), logic_ids.end()); + } + set_cpu_used(cpu_ids, reset_status); + } else { + IE_THROW() << "Wrong value for core_type " << core_type; + } + return cpu_ids; +} + +std::vector get_logic_cores(const std::vector cpu_ids) { + std::vector logic_cores; + if (cpu._proc_type_table[0][HYPER_THREADING_PROC] > 0) { + int cpu_size = static_cast(cpu_ids.size()); + for (int i = 0; i < cpu._processors; i++) { + for (int j = 0; j < cpu_size; j++) { + if (cpu._cpu_mapping_table[i][CPU_MAP_CORE_ID] == cpu._cpu_mapping_table[cpu_ids[j]][CPU_MAP_CORE_ID] && + cpu._cpu_mapping_table[i][CPU_MAP_CORE_TYPE] == HYPER_THREADING_PROC) { + logic_cores.push_back(cpu._cpu_mapping_table[i][CPU_MAP_PROCESSOR_ID]); + } + } + if (cpu_ids.size() == logic_cores.size()) { + break; + } + } + } + + return logic_cores; +} + +void set_cpu_used(std::vector& cpu_ids, int used) { + const auto cpu_size = static_cast(cpu_ids.size()); + for (int i = 0; i < cpu_size; i++) { + if (cpu_ids[i] < cpu._processors) { + cpu._cpu_mapping_table[cpu_ids[i]][CPU_MAP_USED_FLAG] = used; + } + } + // update _proc_type_table + if (used == NOT_USED || used >= PLUGIN_USED_START) { + std::vector all_table; + int start = cpu._sockets > 1 ? 1 : 0; + if (is_cpu_map_available()) { + cpu._proc_type_table.assign(cpu._proc_type_table.size(), std::vector(PROC_TYPE_TABLE_SIZE, 0)); + all_table.resize(PROC_TYPE_TABLE_SIZE, 0); + for (int i = 0; i < cpu._processors; i++) { + if (cpu._cpu_mapping_table[i][CPU_MAP_USED_FLAG] < PLUGIN_USED_START) { + cpu._proc_type_table[cpu._cpu_mapping_table[i][CPU_MAP_SOCKET_ID] + start] + [cpu._cpu_mapping_table[i][CPU_MAP_CORE_TYPE]]++; + cpu._proc_type_table[cpu._cpu_mapping_table[i][CPU_MAP_SOCKET_ID] + start][ALL_PROC]++; + all_table[cpu._cpu_mapping_table[i][CPU_MAP_CORE_TYPE]]++; + all_table[ALL_PROC]++; + } + } + if (cpu._sockets > 1) { + cpu._proc_type_table[0] = all_table; + } + } + } +} + int get_number_of_logical_cpu_cores(bool bigCoresOnly) { int logical_cores = parallel_get_max_threads(); # if (OV_THREAD == OV_THREAD_TBB || OV_THREAD == OV_THREAD_TBB_AUTO) diff --git a/src/inference/src/threading/ie_thread_affinity.cpp b/src/inference/src/threading/ie_thread_affinity.cpp index c0dc955cb30144..578775ac42ab57 100644 --- a/src/inference/src/threading/ie_thread_affinity.cpp +++ b/src/inference/src/threading/ie_thread_affinity.cpp @@ -16,8 +16,13 @@ void ReleaseProcessMask(cpu_set_t* mask) { ov::threading::release_process_mask(mask); } -bool PinThreadToVacantCore(int thrIdx, int hyperthreads, int ncores, const CpuSet& procMask, int cpuIdxOffset) { - return ov::threading::pin_thread_to_vacant_core(thrIdx, hyperthreads, ncores, procMask, cpuIdxOffset); +bool PinThreadToVacantCore(int thrIdx, + int hyperthreads, + int ncores, + const CpuSet& procMask, + const std::vector& cpu_ids, + int cpuIdxOffset) { + return ov::threading::pin_thread_to_vacant_core(thrIdx, hyperthreads, ncores, procMask, cpu_ids, cpuIdxOffset); } bool PinCurrentThreadByMask(int ncores, const CpuSet& procMask) { return ov::threading::pin_current_thread_by_mask(ncores, procMask); diff --git a/src/inference/src/threading/ie_thread_affinity.hpp b/src/inference/src/threading/ie_thread_affinity.hpp index 9b8322ddc130c1..53fb8219cf4245 100644 --- a/src/inference/src/threading/ie_thread_affinity.hpp +++ b/src/inference/src/threading/ie_thread_affinity.hpp @@ -6,6 +6,7 @@ #include #include +#include #include "dev/threading/thread_affinity.hpp" @@ -47,7 +48,12 @@ std::tuple GetProcessMask(); * @param[in] processMask The process mask * @return `True` in case of success, `false` otherwise */ -bool PinThreadToVacantCore(int thrIdx, int hyperThreads, int ncores, const CpuSet& processMask, int cpuIdxOffset = 0); +bool PinThreadToVacantCore(int thrIdx, + int hyperThreads, + int ncores, + const CpuSet& processMask, + const std::vector& cpu_ids = {}, + int cpuIdxOffset = 0); /** * @brief Pins thread to a spare core in the round-robin scheme, while respecting the given process mask. diff --git a/src/inference/tests/functional/caching_test.cpp b/src/inference/tests/functional/caching_test.cpp index e23c6c14d716f5..531c115a32fb91 100644 --- a/src/inference/tests/functional/caching_test.cpp +++ b/src/inference/tests/functional/caching_test.cpp @@ -371,6 +371,7 @@ class CachingTest : public ::testing::TestWithParam&) { std::vector res; res.emplace_back(METRIC_KEY(IMPORT_EXPORT_SUPPORT)); + res.emplace_back(ov::caching_properties.name()); res.emplace_back(METRIC_KEY(DEVICE_ARCHITECTURE)); return res; })); @@ -379,6 +380,7 @@ class CachingTest : public ::testing::TestWithParam{ov::supported_properties.name(), METRIC_KEY(IMPORT_EXPORT_SUPPORT), ov::device::capabilities.name(), + ov::caching_properties.name(), ov::device::architecture.name()}; })); @@ -404,6 +406,12 @@ class CachingTest : public ::testing::TestWithParam&) { + std::vector cachingProperties = {ov::device::architecture.name()}; + return decltype(ov::caching_properties)::value_type(cachingProperties); + })); + ON_CALL(plugin, ImportNetwork(_, _, _)) .WillByDefault(Invoke( [&](std::istream& istr, const RemoteContext::Ptr&, const std::map& config) { @@ -503,6 +511,7 @@ TEST_P(CachingTest, TestLoad) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -541,6 +550,7 @@ TEST_P(CachingTest, TestLoad_by_device_name) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -579,6 +589,7 @@ TEST_P(CachingTest, TestLoadCustomImportExport) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); ON_CALL(*mockPlugin, ImportNetwork(_, _, _)) .WillByDefault( Invoke([&](std::istream& s, const RemoteContext::Ptr&, const std::map&) { @@ -703,6 +714,7 @@ TEST_P(CachingTest, TestChangeLoadConfig_With_Cache_Dir_inline) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); ON_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS), _)) .WillByDefault(Invoke([&](const std::string&, const std::map&) { return std::vector{}; @@ -741,6 +753,7 @@ TEST_P(CachingTest, TestNoCacheEnabled) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(0); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -763,6 +776,7 @@ TEST_P(CachingTest, TestNoCacheSupported) { .Times(AnyNumber()) .WillRepeatedly(Return(false)); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(ov::device::capabilities.name(), _)) .Times(AnyNumber()) .WillRepeatedly(Return(decltype(ov::device::capabilities)::value_type{})); @@ -793,6 +807,7 @@ TEST_P(CachingTest, TestNoCacheMetricSupported) { .WillRepeatedly(Return(std::vector{})); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(0); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(0); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(0); EXPECT_CALL(*mockPlugin, GetMetric(ov::device::capabilities.name(), _)).Times(0); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); @@ -821,6 +836,7 @@ TEST_P(CachingTest, TestNoCacheMetricSupported_by_device_name) { .WillRepeatedly(Return(std::vector{})); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(0); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(0); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(0); EXPECT_CALL(*mockPlugin, GetMetric(ov::device::capabilities.name(), _)).Times(0); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); @@ -1008,6 +1024,7 @@ TEST_P(CachingTest, TestLoadChangeCacheDir) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1046,6 +1063,7 @@ TEST_P(CachingTest, TestLoadChangeCacheDirOneCore) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, SetConfig(_)) .Times(AnyNumber()) .WillRepeatedly(Invoke([](const std::map& config) { @@ -1082,6 +1100,7 @@ TEST_P(CachingTest, TestLoadChangeCacheDirOneCore_overwrite_device_dir) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, SetConfig(_)) .Times(AnyNumber()) .WillRepeatedly(Invoke([](const std::map& config) { @@ -1130,6 +1149,7 @@ TEST_P(CachingTest, TestLoadChangeCacheDirOneCore_SupportsCacheDir_NoImportExpor .Times(AnyNumber()) .WillRepeatedly(Return(decltype(ov::device::capabilities)::value_type{})); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); std::string set_cache_dir = {}; EXPECT_CALL(*mockPlugin, SetConfig(_)) .Times(AtLeast(2)) @@ -1167,6 +1187,7 @@ TEST_P(CachingTest, TestLoadChangeCacheDirOneCore_by_device_name) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, SetConfig(_)) .Times(AnyNumber()) .WillRepeatedly(Invoke([](const std::map& config) { @@ -1212,6 +1233,7 @@ TEST_P(CachingTest, TestLoadChangeCacheDirOneCore_by_device_name_supports_cache_ .Times(AnyNumber()) .WillRepeatedly(Return(decltype(ov::device::capabilities)::value_type{})); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, SetConfig(_)) .Times(AtLeast(2)) .WillRepeatedly(Invoke([](const std::map& config) { @@ -1247,6 +1269,7 @@ TEST_P(CachingTest, TestClearCacheDir) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(0); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1270,6 +1293,7 @@ TEST_P(CachingTest, TestChangeOtherConfig) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1294,6 +1318,7 @@ TEST_P(CachingTest, TestChangeCacheDirFailure) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1335,6 +1360,7 @@ TEST_P(CachingTest, TestCacheDirCreateRecursive) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1358,6 +1384,7 @@ TEST_P(CachingTest, TestDeviceArchitecture) { EXPECT_CALL(*mockPlugin, GetMetric(ov::supported_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)) .Times(AnyNumber()) .WillRepeatedly(Invoke([&](const std::string&, const std::map& options) { @@ -1433,7 +1460,12 @@ TEST_P(CachingTest, TestNoDeviceArchitecture) { EXPECT_CALL(*mockPlugin, GetMetric(ov::supported_properties.name(), _)) .Times(AnyNumber()) .WillRepeatedly(Invoke([&](const std::string&, const std::map&) { - return std::vector{ov::device::capabilities.name()}; + return std::vector{ov::device::capabilities.name(), ov::caching_properties.name()}; + })); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)) + .Times(AnyNumber()) + .WillRepeatedly(Invoke([&](const std::string&, const std::map&) { + return std::vector{ov::supported_properties}; })); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)) .Times(AnyNumber()) @@ -1476,12 +1508,48 @@ TEST_P(CachingTest, TestNoDeviceArchitecture) { } } +TEST_P(CachingTest, TestNoCachingProperties) { + EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::supported_properties.name(), _)) + .Times(AnyNumber()) + .WillRepeatedly(Invoke([&](const std::string&, const std::map&) { + return std::vector{ov::device::capabilities.name()}; + })); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(0); + EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)) + .Times(AnyNumber()) + .WillRepeatedly(Invoke([&](const std::string&, const std::map&) { + return std::vector{METRIC_KEY(IMPORT_EXPORT_SUPPORT)}; + })); + EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::device::capabilities.name(), _)) + .Times(AnyNumber()) + .WillRepeatedly(Return(decltype(ov::device::capabilities)::value_type{ov::device::capability::EXPORT_IMPORT})); + EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(0); + { + EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); + EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); + EXPECT_CALL(*mockPlugin, OnLoadNetworkFromFile()).Times(m_type == TestLoadType::EModelName ? 1 : 0); + EXPECT_CALL(*mockPlugin, ImportNetwork(_, _, _)).Times(0); + EXPECT_CALL(*mockPlugin, ImportNetwork(_, _)).Times(0); + m_post_mock_net_callbacks.emplace_back([&](MockExecutableNetwork& net) { + EXPECT_CALL(net, Export(_)).Times(0); + }); + testLoad([&](Core& ie) { + deviceToLoad = "mock.0"; + ie.SetConfig({{CONFIG_KEY(CACHE_DIR), m_cacheDir}}); + m_testFunction(ie); + }); + } +} + TEST_P(CachingTest, TestThrowOnExport) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(ov::supported_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1505,6 +1573,7 @@ TEST_P(CachingTest, TestThrowOnImport) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1558,6 +1627,7 @@ TEST_P(CachingTest, TestNetworkModified) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _)).Times(!m_remoteContext ? 1 : 0); @@ -1617,6 +1687,7 @@ TEST_P(CachingTest, TestCacheFileCorrupted) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); @@ -1673,6 +1744,7 @@ TEST_P(CachingTest, TestCacheFileOldVersion) { EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(SUPPORTED_METRICS), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(IMPORT_EXPORT_SUPPORT), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); { EXPECT_CALL(*mockPlugin, LoadExeNetworkImpl(_, _, _)).Times(m_remoteContext ? 1 : 0); @@ -1854,6 +1926,7 @@ TEST_P(CachingTest, LoadHetero_TargetFallbackFromCore) { TEST_P(CachingTest, LoadHetero_MultiArchs) { EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)) .Times(AnyNumber()) @@ -1938,6 +2011,7 @@ TEST_P(CachingTest, LoadHetero_MultiArchs) { TEST_P(CachingTest, LoadHetero_MultiArchs_TargetFallback_FromCore) { EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)) .Times(AnyNumber()) .WillRepeatedly(Invoke([&](const std::string&, const std::map& options) { @@ -2006,6 +2080,7 @@ TEST_P(CachingTest, LoadAUTO_OneDevice) { const auto TEST_COUNT = 2; EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Auto plugin @@ -2033,6 +2108,7 @@ TEST_P(CachingTest, LoadAUTOWithConfig) { const auto TEST_COUNT = 2; EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Auto plugin @@ -2064,6 +2140,7 @@ TEST_P(CachingTest, LoadAUTO_OneDeviceNoImportExport) { EXPECT_CALL(*mockPlugin, GetMetric(ov::device::capabilities.name(), _)) .Times(AnyNumber()) .WillRepeatedly(Return(decltype(ov::device::capabilities)::value_type{})); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Auto plugin @@ -2097,6 +2174,7 @@ TEST_P(CachingTest, LoadMulti_race) { EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Multi plugin } @@ -2137,6 +2215,7 @@ TEST_P(CachingTest, LoadMultiWithConfig_race) { EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Multi plugin } @@ -2174,6 +2253,7 @@ TEST_P(CachingTest, LoadMulti_Archs) { const auto TEST_DEVICE_MAX_COUNT = 30; // Shall be >= 2 EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)) .Times(AnyNumber()) .WillRepeatedly(Invoke([&](const std::string&, const std::map& options) { @@ -2230,6 +2310,7 @@ TEST_P(CachingTest, LoadMulti_NoCachingOnDevice) { .WillRepeatedly(Return(decltype(ov::device::capabilities)::value_type{})); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); DataPtr inData = std::make_shared("Param_1", Precision::FP32); InputInfo inpInfo; @@ -2284,6 +2365,7 @@ TEST_P(CachingTest, LoadBATCHWithConfig) { EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Auto plugin } @@ -2313,6 +2395,7 @@ TEST_P(CachingTest, Load_threads) { EXPECT_CALL(*mockPlugin, GetMetric(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, QueryNetwork(_, _)).Times(AnyNumber()); EXPECT_CALL(*mockPlugin, GetMetric(METRIC_KEY(DEVICE_ARCHITECTURE), _)).Times(AnyNumber()); + EXPECT_CALL(*mockPlugin, GetMetric(ov::caching_properties.name(), _)).Times(AnyNumber()); if (m_remoteContext) { return; // skip the remote Context test for Multi plugin } diff --git a/src/inference/tests/functional/matmul_sr_tests.cpp b/src/inference/tests/functional/matmul_sr_tests.cpp index 1bfccfba2b29be..9d5a92169f802a 100644 --- a/src/inference/tests/functional/matmul_sr_tests.cpp +++ b/src/inference/tests/functional/matmul_sr_tests.cpp @@ -34,6 +34,35 @@ struct ReshapeMatMulTestCase { class SmartReshapeMatMulTests : public CommonTestUtils::TestsCommon, public testing::WithParamInterface> { public: + static std::string getTestCaseName(testing::TestParamInfo> obj) { + std::ostringstream result; + const auto& value = std::get<0>(obj.param); + result << "reshape_is_A_input=" << value.reshape_is_A_input << "_"; + result << "A_shape=" << value.A_shape << "_"; + result << "B_shape=" << value.B_shape << "_"; + result << "reshape_pattern=["; + for (size_t i = 0; i < value.reshape_pattern.size(); i++) { + if (i) + result << ","; + result << value.reshape_pattern[i]; + } + result << "]_"; + result << "transpose_a=" << value.transpose_a << "_"; + result << "transpose_b=" << value.transpose_b << "_"; + result << "new_shapes={"; + for (const auto& it : value.new_shapes) { + result << it.first << "=["; + for (size_t i = 0; i < it.second.size(); i++) { + if (i) + result << ","; + result << it.second[i]; + } + result << "]"; + } + result << "}"; + return result.str(); + } + void SetUp() override { const auto& test_case = std::get<0>(GetParam()); @@ -86,7 +115,8 @@ INSTANTIATE_TEST_SUITE_P( ReshapeMatMulTestCase{false, {20, 30}, {1, 30, 40}, {-1, 40}, false, false, {{"input_B", {2, 30, 40}}}}, ReshapeMatMulTestCase{false, {20, 30}, {1, 40, 30}, {40, -1}, false, true, {{"input_B", {2, 40, 30}}}}, ReshapeMatMulTestCase{false, {30, 20}, {1, 30, 40}, {-1, 40}, true, false, {{"input_B", {2, 30, 40}}}}, - ReshapeMatMulTestCase{false, {30, 20}, {1, 40, 30}, {40, -1}, true, true, {{"input_B", {2, 40, 30}}}})); + ReshapeMatMulTestCase{false, {30, 20}, {1, 40, 30}, {40, -1}, true, true, {{"input_B", {2, 40, 30}}}}), + SmartReshapeMatMulTests::getTestCaseName); } // namespace TEST(SmartReshapeTransposeMatMulTests, TransposeAMatMulFuse) { diff --git a/src/inference/tests/unit/CMakeLists.txt b/src/inference/tests/unit/CMakeLists.txt index 36889d8548ede8..02fed6cfd7756a 100644 --- a/src/inference/tests/unit/CMakeLists.txt +++ b/src/inference/tests/unit/CMakeLists.txt @@ -11,9 +11,6 @@ ov_add_test_target( template_extension LINK_LIBRARIES unitTestUtils - INCLUDES - # for static ie_plugins.hpp - "${CMAKE_BINARY_DIR}/src/inference/" ADD_CLANG_FORMAT LABELS OV diff --git a/src/inference/tests/unit/cpu_map_parser.cpp b/src/inference/tests/unit/cpu_map_parser.cpp index 20f8ace1862eb7..514899b32146bb 100644 --- a/src/inference/tests/unit/cpu_map_parser.cpp +++ b/src/inference/tests/unit/cpu_map_parser.cpp @@ -618,11 +618,12 @@ class WinCpuMapParserTests : public CommonTestUtils::TestsCommon, const char* test_ptr = (char*)test_data.system_info.c_str(); std::size_t test_len = test_data.system_info.length(); - std::shared_ptr test_info(new char[test_len / 2]); + std::unique_ptr test_info(new char[test_len / 2]); char* test_info_ptr = test_info.get(); Hex2Bin(test_ptr, test_len, test_info_ptr); + int test_processors = 0; int test_sockets = 0; int test_cores = 0; unsigned long len = unsigned long(test_len / 2); @@ -631,12 +632,13 @@ class WinCpuMapParserTests : public CommonTestUtils::TestsCommon, ov::parse_processor_info_win(test_info_ptr, len, - test_data._processors, + test_processors, test_sockets, test_cores, test_proc_type_table, test_cpu_mapping_table); + ASSERT_EQ(test_data._processors, test_processors); ASSERT_EQ(test_data._sockets, test_sockets); ASSERT_EQ(test_data._cores, test_cores); ASSERT_EQ(test_data._proc_type_table, test_proc_type_table); @@ -1544,6 +1546,197 @@ WinCpuMapTestCase _2sockets_48cores_hyperthreading = { "fffff000030300000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffff0000"}, }; +WinCpuMapTestCase _2sockets_36cores_hyperthreading = { + 72, + 2, + 36, + {{72, 36, 0, 36}, {36, 18, 0, 18}, {36, 18, 0, 18}}, + { + {0, 0, 0, HYPER_THREADING_PROC, 0, -1}, {1, 0, 0, MAIN_CORE_PROC, 0, -1}, + {2, 0, 1, HYPER_THREADING_PROC, 1, -1}, {3, 0, 1, MAIN_CORE_PROC, 1, -1}, + {4, 0, 2, HYPER_THREADING_PROC, 2, -1}, {5, 0, 2, MAIN_CORE_PROC, 2, -1}, + {6, 0, 3, HYPER_THREADING_PROC, 3, -1}, {7, 0, 3, MAIN_CORE_PROC, 3, -1}, + {8, 0, 4, HYPER_THREADING_PROC, 4, -1}, {9, 0, 4, MAIN_CORE_PROC, 4, -1}, + {10, 0, 5, HYPER_THREADING_PROC, 5, -1}, {11, 0, 5, MAIN_CORE_PROC, 5, -1}, + {12, 0, 6, HYPER_THREADING_PROC, 6, -1}, {13, 0, 6, MAIN_CORE_PROC, 6, -1}, + {14, 0, 7, HYPER_THREADING_PROC, 7, -1}, {15, 0, 7, MAIN_CORE_PROC, 7, -1}, + {16, 0, 8, HYPER_THREADING_PROC, 8, -1}, {17, 0, 8, MAIN_CORE_PROC, 8, -1}, + {18, 0, 9, HYPER_THREADING_PROC, 9, -1}, {19, 0, 9, MAIN_CORE_PROC, 9, -1}, + {20, 0, 10, HYPER_THREADING_PROC, 10, -1}, {21, 0, 10, MAIN_CORE_PROC, 10, -1}, + {22, 0, 11, HYPER_THREADING_PROC, 11, -1}, {23, 0, 11, MAIN_CORE_PROC, 11, -1}, + {24, 0, 12, HYPER_THREADING_PROC, 12, -1}, {25, 0, 12, MAIN_CORE_PROC, 12, -1}, + {26, 0, 13, HYPER_THREADING_PROC, 13, -1}, {27, 0, 13, MAIN_CORE_PROC, 13, -1}, + {28, 0, 14, HYPER_THREADING_PROC, 14, -1}, {29, 0, 14, MAIN_CORE_PROC, 14, -1}, + {30, 0, 15, HYPER_THREADING_PROC, 15, -1}, {31, 0, 15, MAIN_CORE_PROC, 15, -1}, + {32, 0, 16, HYPER_THREADING_PROC, 16, -1}, {33, 0, 16, MAIN_CORE_PROC, 16, -1}, + {34, 0, 17, HYPER_THREADING_PROC, 17, -1}, {35, 0, 17, MAIN_CORE_PROC, 17, -1}, + {36, 1, 18, HYPER_THREADING_PROC, 18, -1}, {37, 1, 18, MAIN_CORE_PROC, 18, -1}, + {38, 1, 19, HYPER_THREADING_PROC, 19, -1}, {39, 1, 19, MAIN_CORE_PROC, 19, -1}, + {40, 1, 20, HYPER_THREADING_PROC, 20, -1}, {41, 1, 20, MAIN_CORE_PROC, 20, -1}, + {42, 1, 21, HYPER_THREADING_PROC, 21, -1}, {43, 1, 21, MAIN_CORE_PROC, 21, -1}, + {44, 1, 22, HYPER_THREADING_PROC, 22, -1}, {45, 1, 22, MAIN_CORE_PROC, 22, -1}, + {46, 1, 23, HYPER_THREADING_PROC, 23, -1}, {47, 1, 23, MAIN_CORE_PROC, 23, -1}, + {48, 1, 24, HYPER_THREADING_PROC, 24, -1}, {49, 1, 24, MAIN_CORE_PROC, 24, -1}, + {50, 1, 25, HYPER_THREADING_PROC, 25, -1}, {51, 1, 25, MAIN_CORE_PROC, 25, -1}, + {52, 1, 26, HYPER_THREADING_PROC, 26, -1}, {53, 1, 26, MAIN_CORE_PROC, 26, -1}, + {54, 1, 27, HYPER_THREADING_PROC, 27, -1}, {55, 1, 27, MAIN_CORE_PROC, 27, -1}, + {56, 1, 28, HYPER_THREADING_PROC, 28, -1}, {57, 1, 28, MAIN_CORE_PROC, 28, -1}, + {58, 1, 29, HYPER_THREADING_PROC, 29, -1}, {59, 1, 29, MAIN_CORE_PROC, 29, -1}, + {60, 1, 30, HYPER_THREADING_PROC, 30, -1}, {61, 1, 30, MAIN_CORE_PROC, 30, -1}, + {62, 1, 31, HYPER_THREADING_PROC, 31, -1}, {63, 1, 31, MAIN_CORE_PROC, 31, -1}, + {64, 1, 32, HYPER_THREADING_PROC, 32, -1}, {65, 1, 32, MAIN_CORE_PROC, 32, -1}, + {66, 1, 33, HYPER_THREADING_PROC, 33, -1}, {67, 1, 33, MAIN_CORE_PROC, 33, -1}, + {68, 1, 34, HYPER_THREADING_PROC, 34, -1}, {69, 1, 34, MAIN_CORE_PROC, 34, -1}, + {70, 1, 35, HYPER_THREADING_PROC, 35, -1}, {71, 1, 35, MAIN_CORE_PROC, 35, -1}, + }, + {"0300000030000000000000000000000000000000000000000000000000000100ffffffff0f000000000000000000000000000000300000000" + "10000000000000000000000000000000000000000000100030000000000000000000000000000000200000038000000010c400000c0000002" + "00000000000000000000000000000000000000000000000300000000000000000000000000000002000000380000000108400000800000010" + "00000000000000000000000000000000000000000000003000000000000000000000000000000020000003800000002144000000014000000" + "00000000000000000000000000000000000000000000030000000000000000000000000000000200000038000000030c40000000700200000" + "0000000000000000000000000000000000000000000ffffffff0f000000000000000000000000000000300000000100000000000000000000" + "000000000000000000000001000c0000000000000000000000000000000200000038000000010c400000c0000002000000000000000000000" + "00000000000000000000000000c00000000000000000000000000000002000000380000000108400000800000010000000000000000000000" + "0000000000000000000000000c000000000000000000000000000000020000003800000002144000000014000000000000000000000000000" + "000000000000000000000000c0000000000000000000000000000000000000030000000010000000000000000000000000000000000000000" + "000100300000000000000000000000000000000200000038000000010c400000c000000200000000000000000000000000000000000000000" + "00000300000000000000000000000000000000200000038000000010840000080000001000000000000000000000000000000000000000000" + "00003000000000000000000000000000000002000000380000000214400000001400000000000000000000000000000000000000000000000" + "000300000000000000000000000000000000000000030000000010000000000000000000000000000000000000000000100c0000000000000" + "0000000000000000000200000038000000010c400000c00000020000000000000000000000000000000000000000000000c00000000000000" + "0000000000000000002000000380000000108400000800000010000000000000000000000000000000000000000000000c000000000000000" + "000000000000000002000000380000000214400000001400000000000000000000000000000000000000000000000000c0000000000000000" + "00000000000000000000000300000000100000000000000000000000000000000000000000001000003000000000000000000000000000002" + "00000038000000010c400000c0000002000000000000000000000000000000000000000000000000030000000000000000000000000000020" + "00000380000000108400000800000010000000000000000000000000000000000000000000000000300000000000000000000000000000200" + "00003800000002144000000014000000000000000000000000000000000000000000000000000003000000000000000000000000000000000" + "00030000000010000000000000000000000000000000000000000000100000c00000000000000000000000000000200000038000000010c40" + "0000c00000020000000000000000000000000000000000000000000000000c000000000000000000000000000002000000380000000108400" + "000800000010000000000000000000000000000000000000000000000000c0000000000000000000000000000020000003800000002144000" + "00001400000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000030000000010000000" + "000000000000000000000000000000000000100003000000000000000000000000000000200000038000000010c400000c000000200000000" + "00000000000000000000000000000000000000003000000000000000000000000000000200000038000000010840000080000001000000000" + "00000000000000000000000000000000000000030000000000000000000000000000002000000380000000214400000001400000000000000" + "00000000000000000000000000000000000000300000000000000000000000000000000000003000000001000000000000000000000000000" + "000000000000000010000c000000000000000000000000000000200000038000000010c400000c00000020000000000000000000000000000" + "00000000000000000000c00000000000000000000000000000020000003800000001084000008000000100000000000000000000000000000" + "0000000000000000000c000000000000000000000000000000200000038000000021440000000140000000000000000000000000000000000" + "000000000000000000c0000000000000000000000000000000000000300000000100000000000000000000000000000000000000000001000" + "00003000000000000000000000000000200000038000000010c400000c0000002000000000000000000000000000000000000000000000000" + "00030000000000000000000000000002000000380000000108400000800000010000000000000000000000000000000000000000000000000" + "00300000000000000000000000000020000003800000002144000000014000000000000000000000000000000000000000000000000000000" + "0300000000000000000000000000000000003000000001000000000000000000000000000000000000000000010000000c000000000000000" + "000000000000200000038000000010c400000c0000002000000000000000000000000000000000000000000000000000c0000000000000000" + "00000000000200000038000000010840000080000001000000000000000000000000000000000000000000000000000c00000000000000000" + "0000000000200000038000000021440000000140000000000000000000000000000000000000000000000000000000c000000000000000000" + "00000000000000003000000001000000000000000000000000000000000000000000010000003000000000000000000000000000020000003" + "8000000010c400000c00000020000000000000000000000000000000000000000000000000030000000000000000000000000000200000038" + "00000001084000008000000100000000000000000000000000000000000000000000000000300000000000000000000000000002000000380" + "00000021440000000140000000000000000000000000000000000000000000000000000003000000000000000000000000000000000003000" + "00000100000000000000000000000000000000000000000001000000c0000000000000000000000000000200000038000000010c400000c00" + "0000200000000000000000000000000000000000000000000000000c000000000000000000000000000020000003800000001084000008000" + "000100000000000000000000000000000000000000000000000000c0000000000000000000000000000200000038000000021440000000140" + "00000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000300000000100000000000000" + "00000000000000000000000000000100000000030000000000000000000000000200000038000000010c400000c0000002000000000000000" + "00000000000000000000000000000000000000300000000000000000000000002000000380000000108400000800000010000000000000000" + "00000000000000000000000000000000000003000000000000000000000000020000003800000002144000000014000000000000000000000" + "00000000000000000000000000000000000030000000000000000000000000000000030000000010000000000000000000000000000000000" + "0000000001000000000c0000000000000000000000000200000038000000010c400000c000000200000000000000000000000000000000000" + "000000000000000000c0000000000000000000000000200000038000000010840000080000001000000000000000000000000000000000000" + "00000000000000000c00000000000000000000000002000000380000000214400000001400000000000000000000000000000000000000000" + "0000000000000000c000000000000000000000000000000003000000001000000000000000000000000000000000000000000010000000030" + "0000000000000000000000000200000038000000010c400000c00000020000000000000000000000000000000000000000000000000000300" + "00000000000000000000000020000003800000001084000008000000100000000000000000000000000000000000000000000000000003000" + "00000000000000000000000200000038000000021440000000140000000000000000000000000000000000000000000000000000000030000" + "0000000000000000000000000000030000000010000000000000000000000000000000000000000000100000000c000000000000000000000" + "00000200000038000000010c400000c00000020000000000000000000000000000000000000000000000000000c0000000000000000000000" + "00002000000380000000108400000800000010000000000000000000000000000000000000000000000000000c00000000000000000000000" + "0002000000380000000214400000001400000000000000000000000000000000000000000000000000000000c000000000000000000000000" + "00000000030000000010000000000000000000000000000000000000000000100000000000300000000000000000000000200000038000000" + "010c400000c000000200000000000000000000000000000000000000000000000000000003000000000000000000000002000000380000000" + "10840000080000001000000000000000000000000000000000000000000000000000000030000000000000000000000020000003800000002" + "14400000001400000000000000000000000000000000000000000000000000000000000300000000000000000000000000000030000000010" + "000000000000000000000000000000000000000000100000000000c00000000000000000000000200000038000000010c400000c000000200" + "00000000000000000000000000000000000000000000000000000c00000000000000000000000200000038000000010840000080000001000" + "0000000000000000000000000000000000000000000000000000c000000000000000000000002000000380000000214400000001400000000" + "000000000000000000000000000000000000000000000000000c0000000000000000000000030000003000000000000000000000000000000" + "0000000000000000000000100ffffffff0f000000010000000000000000000000300000000100000000000000000000000000000000000000" + "00000100030000000000000001000000000000000200000038000000010c400000c0000002000000000000000000000000000000000000000" + "00000000300000000000000010000000000000002000000380000000108400000800000010000000000000000000000000000000000000000" + "00000003000000000000000100000000000000020000003800000002144000000014000000000000000000000000000000000000000000000" + "00000030000000000000001000000000000000200000038000000030c40000000700200000000000000000000000000000000000000000000" + "0000ffffffff0f000000010000000000000000000000300000000100000000000000000000000000000000000000000001000c00000000000" + "00001000000000000000200000038000000010c400000c000000200000000000000000000000000000000000000000000000c000000000000" + "000100000000000000020000003800000001084000008000000100000000000000000000000000000000000000000000000c0000000000000" + "00100000000000000020000003800000002144000000014000000000000000000000000000000000000000000000000000c00000000000000" + "01000000000000000000000030000000010000000000000000000000000000000000000000000100300000000000000001000000000000000" + "200000038000000010c400000c000000200000000000000000000000000000000000000000000003000000000000000010000000000000002" + "00000038000000010840000080000001000000000000000000000000000000000000000000000030000000000000000100000000000000020" + "00000380000000214400000001400000000000000000000000000000000000000000000000000300000000000000001000000000000000000" + "000030000000010000000000000000000000000000000000000000000100c00000000000000001000000000000000200000038000000010c4" + "00000c00000020000000000000000000000000000000000000000000000c00000000000000001000000000000000200000038000000010840" + "0000800000010000000000000000000000000000000000000000000000c000000000000000010000000000000002000000380000000214400" + "000001400000000000000000000000000000000000000000000000000c0000000000000000100000000000000000000003000000001000000" + "0000000000000000000000000000000000000100000300000000000001000000000000000200000038000000010c400000c00000020000000" + "00000000000000000000000000000000000000000030000000000000100000000000000020000003800000001084000008000000100000000" + "00000000000000000000000000000000000000000300000000000001000000000000000200000038000000021440000000140000000000000" + "00000000000000000000000000000000000000003000000000000010000000000000000000000300000000100000000000000000000000000" + "00000000000000000100000c00000000000001000000000000000200000038000000010c400000c0000002000000000000000000000000000" + "0000000000000000000000c000000000000010000000000000002000000380000000108400000800000010000000000000000000000000000" + "000000000000000000000c0000000000000100000000000000020000003800000002144000000014000000000000000000000000000000000" + "00000000000000000000c00000000000001000000000000000000000030000000010000000000000000000000000000000000000000000100" + "003000000000000001000000000000000200000038000000010c400000c000000200000000000000000000000000000000000000000000000" + "03000000000000001000000000000000200000038000000010840000080000001000000000000000000000000000000000000000000000000" + "30000000000000010000000000000002000000380000000214400000001400000000000000000000000000000000000000000000000000003" + "00000000000000100000000000000000000003000000001000000000000000000000000000000000000000000010000c00000000000000100" + "0000000000000200000038000000010c400000c0000002000000000000000000000000000000000000000000000000c000000000000001000" + "000000000000200000038000000010840000080000001000000000000000000000000000000000000000000000000c0000000000000010000" + "00000000000200000038000000021440000000140000000000000000000000000000000000000000000000000000c00000000000000100000" + "00000000000000000300000000100000000000000000000000000000000000000000001000000030000000000010000000000000002000000" + "38000000010c400000c0000002000000000000000000000000000000000000000000000000000300000000000100000000000000020000003" + "80000000108400000800000010000000000000000000000000000000000000000000000000003000000000001000000000000000200000038" + "00000002144000000014000000000000000000000000000000000000000000000000000000030000000000010000000000000000000000300" + "0000001000000000000000000000000000000000000000000010000000c000000000001000000000000000200000038000000010c400000c0" + "000002000000000000000000000000000000000000000000000000000c0000000000010000000000000002000000380000000108400000800" + "00001000000000000000000000000000000000000000000000000000c00000000000100000000000000020000003800000002144000000014" + "0000000000000000000000000000000000000000000000000000000c000000000001000000000000000000000030000000010000000000000" + "000000000000000000000000000000100000030000000000001000000000000000200000038000000010c400000c000000200000000000000" + "00000000000000000000000000000000000030000000000001000000000000000200000038000000010840000080000001000000000000000" + "00000000000000000000000000000000000300000000000010000000000000002000000380000000214400000001400000000000000000000" + "00000000000000000000000000000000003000000000000100000000000000000000003000000001000000000000000000000000000000000" + "00000000001000000c0000000000001000000000000000200000038000000010c400000c00000020000000000000000000000000000000000" + "0000000000000000c000000000000100000000000000020000003800000001084000008000000100000000000000000000000000000000000" + "000000000000000c0000000000001000000000000000200000038000000021440000000140000000000000000000000000000000000000000" + "00000000000000c00000000000010000000000000000000000300000000100000000000000000000000000000000000000000001000000000" + "30000000001000000000000000200000038000000010c400000c0000002000000000000000000000000000000000000000000000000000003" + "00000000010000000000000002000000380000000108400000800000010000000000000000000000000000000000000000000000000000030" + "00000000100000000000000020000003800000002144000000014000000000000000000000000000000000000000000000000000000000300" + "000000010000000000000000000000300000000100000000000000000000000000000000000000000001000000000c0000000001000000000" + "000000200000038000000010c400000c000000200000000000000000000000000000000000000000000000000000c00000000010000000000" + "0000020000003800000001084000008000000100000000000000000000000000000000000000000000000000000c000000000100000000000" + "000020000003800000002144000000014000000000000000000000000000000000000000000000000000000000c0000000001000000000000" + "00000000003000000001000000000000000000000000000000000000000000010000000030000000000100000000000000020000003800000" + "0010c400000c00000020000000000000000000000000000000000000000000000000000300000000001000000000000000200000038000000" + "01084000008000000100000000000000000000000000000000000000000000000000003000000000010000000000000002000000380000000" + "21440000000140000000000000000000000000000000000000000000000000000000030000000000100000000000000000000003000000001" + "0000000000000000000000000000000000000000000100000000c00000000001000000000000000200000038000000010c400000c00000020" + "000000000000000000000000000000000000000000000000000c0000000000100000000000000020000003800000001084000008000000100" + "00000000000000000000000000000000000000000000000000c00000000001000000000000000200000038000000021440000000140000000" + "0000000000000000000000000000000000000000000000000c000000000010000000000000000000000300000000100000000000000000000" + "00000000000000000000000100000000000300000001000000000000000200000038000000010c400000c0000002000000000000000000000" + "00000000000000000000000000000000003000000010000000000000002000000380000000108400000800000010000000000000000000000" + "00000000000000000000000000000000030000000100000000000000020000003800000002144000000014000000000000000000000000000" + "00000000000000000000000000000000300000001000000000000000000000030000000010000000000000000000000000000000000000000" + "000100000000000c00000001000000000000000200000038000000010c400000c000000200000000000000000000000000000000000000000" + "00000000000000c00000001000000000000000200000038000000010840000080000001000000000000000000000000000000000000000000" + "0000000000000c000000010000000000000002000000380000000214400000001400000000000000000000000000000000000000000000000" + "000000000000c00000001000000000000000100000030000000000000000000000000000000000000000000000000000000ffffffff0f0000" + "0000000000000000000100000030000000010000000000000000000000000000000000000000000000ffffffff0f000000010000000000000" + "00400000080000000020002000000000000000000000000000000000000000000402400000000000000000000000000000000000000000000" + "00000000000000000000000000000000ffffffff0f00000040240000000000000000000000000000000000000000000000000000000000000" + "000000000000000ffffffff0f000000"}, +}; + WinCpuMapTestCase _2sockets_48cores = { 48, 2, @@ -2361,6 +2554,7 @@ INSTANTIATE_TEST_SUITE_P(CPUMap, WinCpuMapParserTests, testing::Values(_2sockets_104cores_hyperthreading, _2sockets_48cores_hyperthreading, + _2sockets_36cores_hyperthreading, _2sockets_48cores, _1sockets_24cores_hyperthreading_set1, _1sockets_24cores_hyperthreading_set2, diff --git a/src/plugins/auto/CMakeLists.txt b/src/plugins/auto/CMakeLists.txt index ed24e998a5f421..25e6b8b358d578 100644 --- a/src/plugins/auto/CMakeLists.txt +++ b/src/plugins/auto/CMakeLists.txt @@ -9,14 +9,14 @@ endif() set (TARGET_NAME "openvino_auto_plugin") -file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/utils/*.cpp) +file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/*.cpp) if(ENABLE_AUTO AND ENABLE_MULTI) ie_add_plugin(NAME ${TARGET_NAME} DEVICE_NAME "MULTI" PSEUDO_DEVICE SOURCES ${SOURCES} - VERSION_DEFINES_FOR plugin.cpp) + VERSION_DEFINES_FOR src/plugin.cpp) ie_add_plugin(NAME ${TARGET_NAME} DEVICE_NAME "AUTO" @@ -27,13 +27,17 @@ elseif(ENABLE_AUTO) DEVICE_NAME "AUTO" PSEUDO_DEVICE SOURCES ${SOURCES} - VERSION_DEFINES_FOR plugin.cpp) + VERSION_DEFINES_FOR src/plugin.cpp) elseif(ENABLE_MULTI) ie_add_plugin(NAME ${TARGET_NAME} DEVICE_NAME "MULTI" PSEUDO_DEVICE SOURCES ${SOURCES} - VERSION_DEFINES_FOR plugin.cpp) + VERSION_DEFINES_FOR src/plugin.cpp) +endif() + +if(ENABLE_TESTS) + add_subdirectory(tests) endif() set_ie_threading_interface_for(${TARGET_NAME}) @@ -41,4 +45,4 @@ set_ie_threading_interface_for(${TARGET_NAME}) # must be called after all target_link_libraries ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) -set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) +set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) \ No newline at end of file diff --git a/src/plugins/auto/README.md b/src/plugins/auto/README.md index eae0fa2de31303..2ac7a8883cd065 100644 --- a/src/plugins/auto/README.md +++ b/src/plugins/auto/README.md @@ -17,7 +17,8 @@ In case of any questions, review and merge requests, contact the [AUTO Plugin ma The AUTO plugin follows the OpenVINO™ plugin architecture and consists of several main components: * [docs](./docs) contains developer documentation for the AUTO plugin. - * [current](./) folder contains sources of the AUTO plugin. + * [src](./src/) - folder contains sources of the AUTO plugin. + * [tests](./tests/) - tests for Auto Plugin components. Learn more in the [OpenVINO™ Plugin Developer Guide](https://docs.openvino.ai/latest/openvino_docs_ie_plugin_dg_overview.html). diff --git a/src/plugins/auto/bind_multi_schedule.cpp b/src/plugins/auto/bind_multi_schedule.cpp deleted file mode 100644 index 3ab8f30547d9dd..00000000000000 --- a/src/plugins/auto/bind_multi_schedule.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/////////////////////////////////////////////////////////////////////////////////////////////////// -#include "async_infer_request.hpp" -#include "plugin.hpp" -#include "bind_multi_schedule.hpp" -#include "multi_executable_network.hpp" -// ------------------------------MultiSchedule---------------------------- -namespace MultiDevicePlugin { - -thread_local IE::IInferRequestInternal* BinderMultiSchedule::_sharedRequest = nullptr; - -void BinderMultiSchedule::init(const ScheduleContext::Ptr& sContext) { - MultiSchedule::init(sContext); -} - -Pipeline BinderMultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) { - Pipeline pipeline = { - // if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only: - Stage { - /*TaskExecutor*/ std::make_shared(), /*task*/ [this, &syncInferRequest, workerInferRequest]() { - // by default, no preferred device: - _thisPreferredDeviceName = ""; - auto execNetwork = _multiSContext->_executableNetwork.lock(); - // if any input is remote (e.g. was set with SetBlob), let' use the corresponding device - for (const auto& it : execNetwork->GetInputsInfo()) { - auto b = syncInferRequest->GetBlob(it.first); - auto r = b->as(); - if (r) { - const auto name = r->getDeviceName(); - const auto res = std::find_if( - _multiSContext->_devicePrioritiesInitial.cbegin(), - _multiSContext->_devicePrioritiesInitial.cend(), - [&name](const MultiDevicePlugin::DeviceInformation & d) { - return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." + - d.defaultDeviceID)) == name; - }); - if (_multiSContext->_devicePrioritiesInitial.cend() == res) { - IE_THROW() << - "None of the devices (for which current MULTI-device configuration was " - "initialized) supports a remote blob created on the device named " << name; - } else { - // it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*) - // as the original strings are from the "persistent" vector (with the right lifetime) - _thisPreferredDeviceName = res->deviceName.c_str(); - break; - } - } - } - _thisWorkerInferRequest = *workerInferRequest; - _sharedRequest = std::dynamic_pointer_cast(syncInferRequest)->GetSharedRequest()._ptr.get(); - }}, - // as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest) - // then sets the device-agnostic blobs to the actual (device-specific) request - Stage { - /*TaskExecutor*/std::dynamic_pointer_cast(shared_from_this()), /*task*/ [&syncInferRequest, workerInferRequest]() { - *workerInferRequest = _thisWorkerInferRequest; - auto multiSyncInferRequest = std::dynamic_pointer_cast(syncInferRequest); - multiSyncInferRequest->SetBlobsToAnotherRequest(_thisWorkerInferRequest->_inferRequest); - INFO_RUN([workerInferRequest]() { - (*workerInferRequest)->_startTimes.push_back(std::chrono::steady_clock::now()); - }); - }}, - // final task in the pipeline: - Stage { - /*TaskExecutor*/std::make_shared(workerInferRequest), /*task*/ [this, &syncInferRequest, workerInferRequest]() { - if (nullptr != (*workerInferRequest)->_exceptionPtr) { - std::rethrow_exception((*workerInferRequest)->_exceptionPtr); - } - if (_multiSContext->_needPerfCounters) { - auto multiSyncInferRequest = std::dynamic_pointer_cast - (syncInferRequest); - multiSyncInferRequest->_scheduledRequest = - (*workerInferRequest)->_inferRequest; - } - INFO_RUN([workerInferRequest]() { - (*workerInferRequest)->_endTimes.push_back(std::chrono::steady_clock::now()); - }); - }} - }; - return pipeline; -} - -bool BinderMultiSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) { - std::vector devices; - devices = [&] { - std::lock_guard lock(_multiSContext->_mutex); - return _multiSContext->_devicePriorities; - }(); - for (auto&& device : devices) { - if (!preferred_device.empty() && (device.deviceName != preferred_device)) { - continue; - } - if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) { - return true; - } - } - // no vacant requests this time, storing the task to the respective queue - if (!preferred_device.empty()) { - _inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(inferPipelineTask)); - } else { - _inferPipelineTasks.push(std::move(inferPipelineTask)); - } - return false; -} - -bool BinderMultiSchedule::RunPipelineTask(IE::Task& inferPipelineTask, - NotBusyWorkerRequests& idleWorkerRequests, - const DeviceName& preferred_device) { - WorkerInferRequest* workerRequestPtr = nullptr; - WorkerInferRequest* headWorker = nullptr; - bool flag = false; - while (idleWorkerRequests.try_pop(workerRequestPtr)) { - if (flag && workerRequestPtr == headWorker) - break; - if (!flag) { - headWorker = workerRequestPtr; - flag = true; - } - IdleGuard idleGuard{workerRequestPtr, idleWorkerRequests}; - if (_sharedRequest == workerRequestPtr->_inferRequest._ptr.get()) { - _thisWorkerInferRequest = workerRequestPtr; - { - auto capturedTask = std::move(inferPipelineTask); - capturedTask(); - } - idleGuard.Release(); - return true; - } - } - return false; -} - -void BinderMultiSchedule::run(IE::Task inferPipelineTask) { - if (_thisWorkerInferRequest) { - auto capturedTask = std::move(inferPipelineTask); - capturedTask(); - } else { - ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName); - } -} - -BinderMultiSchedule::~BinderMultiSchedule() { -} - -IInferPtr BinderMultiSchedule::CreateInferRequestImpl( - const std::vector>& inputs, - const std::vector>& outputs) { - auto num = _numRequestsCreated++; - size_t sum = 0; - SoInfer request_to_share_blobs_with; - // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests - // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) - for (const auto& device : _multiSContext->_devicePrioritiesInitial) { - auto& dev_requests = _workerRequests[device.deviceName]; - if ((num - sum) < dev_requests.size()) { - request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; - break; - } - sum += dev_requests.size(); - } - if (!request_to_share_blobs_with) { - IE_THROW() << - "binder mode does not allow oversubsciption of infer requests" - " please use optimal infer request"; - } - auto syncImpl = std::make_shared(inputs, outputs, request_to_share_blobs_with); - return syncImpl; -} - -IInferPtr BinderMultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkInputs, - IE::OutputsDataMap networkOutputs) { - auto num = _numRequestsCreated++; - SoInfer request_to_share_blobs_with; - size_t sum = 0; - // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests - // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) - for (const auto& device : _multiSContext->_devicePrioritiesInitial) { - auto& dev_requests = _workerRequests[device.deviceName]; - if ((num - sum) < dev_requests.size()) { - request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; - break; - } - sum += dev_requests.size(); - } - if (!request_to_share_blobs_with) { - IE_THROW() << - "binder mode does not allow oversubsciption of infer requests" - " please use optimal infer request"; - } - auto syncImpl = std::make_shared(networkInputs, networkOutputs, request_to_share_blobs_with); - return syncImpl; -} - -} // namespace MultiDevicePlugin - diff --git a/src/plugins/auto/async_infer_request.cpp b/src/plugins/auto/src/async_infer_request.cpp similarity index 100% rename from src/plugins/auto/async_infer_request.cpp rename to src/plugins/auto/src/async_infer_request.cpp diff --git a/src/plugins/auto/async_infer_request.hpp b/src/plugins/auto/src/async_infer_request.hpp similarity index 100% rename from src/plugins/auto/async_infer_request.hpp rename to src/plugins/auto/src/async_infer_request.hpp diff --git a/src/plugins/auto/auto_executable_network.cpp b/src/plugins/auto/src/auto_executable_network.cpp similarity index 71% rename from src/plugins/auto/auto_executable_network.cpp rename to src/plugins/auto/src/auto_executable_network.cpp index 48de040c3985c7..fa7b814e13eaae 100644 --- a/src/plugins/auto/auto_executable_network.cpp +++ b/src/plugins/auto/src/auto_executable_network.cpp @@ -17,12 +17,21 @@ AutoExecutableNetwork::AutoExecutableNetwork(AutoScheduleContext::Ptr& context, } std::shared_ptr AutoExecutableNetwork::GetContext() const { - std::lock_guard lock(_autoSContext->_fallbackMutex); - if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) { - return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext(); + if (_autoSchedule->_pCTPUTLoadContext) { + for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) { + if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) { + return _autoSchedule->_pCTPUTLoadContext[i].executableNetwork->GetContext(); + } + } + return nullptr; } else { - _autoSchedule->WaitActualNetworkReady(); - return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext(); + std::lock_guard lock(_autoSContext->_fallbackMutex); + if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) { + return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext(); + } else { + _autoSchedule->WaitActualNetworkReady(); + return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext(); + } } } @@ -51,22 +60,26 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { auto value = _autoSContext->_performanceHint; if (!_autoSContext->_core->isNewAPI()) return value; - if (value == InferenceEngine::PluginConfigParams::THROUGHPUT) + if (value == InferenceEngine::PluginConfigParams::THROUGHPUT) { return ov::hint::PerformanceMode::THROUGHPUT; - else if (value == InferenceEngine::PluginConfigParams::LATENCY) + } else if (value == InferenceEngine::PluginConfigParams::LATENCY) { return ov::hint::PerformanceMode::LATENCY; - else if (value == InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT) + } else if (value == InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT) { return ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT; - else + } else if (value == "UNDEFINED") { + OPENVINO_SUPPRESS_DEPRECATED_START return ov::hint::PerformanceMode::UNDEFINED; + OPENVINO_SUPPRESS_DEPRECATED_END + } else { + OPENVINO_THROW("Unsupported value of ov::hint::PerformanceMode"); + } } else if (name == ov::device::priorities) { auto value = _autoSContext->_config.find(ov::device::priorities.name()); return decltype(ov::device::priorities)::value_type {value->second.as()}; } else if (name == ov::device::properties) { ov::AnyMap all_devices = {}; - if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) { - ov::AnyMap device_properties = {}; - auto& context = _autoSchedule->_loadContext[ACTUALDEVICE]; + auto get_device_supported_metrics = [&all_devices] (const AutoLoadContext& context) { + ov::AnyMap device_properties = {}; auto device_supported_metrics = context.executableNetwork->GetMetric(METRIC_KEY(SUPPORTED_METRICS)); for (auto&& property_name : device_supported_metrics.as>()) { device_properties[property_name] = context.executableNetwork->GetMetric(property_name); @@ -76,6 +89,26 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { device_properties[property_name] = context.executableNetwork->GetConfig(property_name); } all_devices[context.deviceInfo.deviceName] = device_properties; + }; + if (_autoSchedule->_pCTPUTLoadContext) { + for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) { + if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) { + get_device_supported_metrics(_autoSchedule->_pCTPUTLoadContext[i]); + } + } + } else { + { + std::lock_guard lock(_autoSContext->_fallbackMutex); + if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) { + get_device_supported_metrics(_autoSchedule->_loadContext[FALLBACKDEVICE]); + } + } + std::lock_guard lock(_autoSContext->_confMutex); + if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) { + get_device_supported_metrics(_autoSchedule->_loadContext[ACTUALDEVICE]); + } else { + get_device_supported_metrics(_autoSchedule->_loadContext[CPU]); + } } return all_devices; } else if (name == ov::hint::model_priority) { @@ -91,6 +124,26 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { const unsigned int defaultNumForTPUT = 4u; const unsigned int defaultNumForLatency = 1u; unsigned int real = 0; + if (_autoSchedule->_pCTPUTLoadContext) { + std::lock_guard lock(_autoSContext->_fallbackMutex); + unsigned int res = 0u; + for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) { + try { + if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) { + res += (_autoSchedule->_pCTPUTLoadContext[i]) + .executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)) + .as(); + } + } catch (const IE::Exception& iie) { + IE_THROW() + << "Every device used in cumulative mode should " + << "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. " + << "Failed to query the metric for with error:" << + iie.what(); + } + } + return decltype(ov::optimal_number_of_infer_requests)::value_type {res}; + } if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) { real = _autoSchedule->_loadContext[ACTUALDEVICE]. executableNetwork->GetMetric(name).as(); @@ -181,12 +234,13 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { exeDevices.push_back(ExeDevicesString); execution_devices = decltype(ov::execution_devices)::value_type {exeDevices}; }; - if (_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) { - try { - execution_devices = _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetMetric(name); - } catch(const IE::Exception&) { - GetExecutionDevices(_autoSchedule->_loadContext[ACTUALDEVICE].workName); + if (_autoSchedule->_pCTPUTLoadContext) { + std::vector exeDevices = {}; + std::lock_guard lock(_autoSContext->_confMutex); + for (auto n : _autoSContext->_devicePriorities) { + exeDevices.push_back(n.deviceName); } + execution_devices = decltype(ov::execution_devices)::value_type {exeDevices}; } else { std::lock_guard lock(_autoSContext->_confMutex); for (int i = 0; i < CONTEXTNUM; i++) { @@ -203,9 +257,13 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { return execution_devices; } else if (name == ov::model_name) { std::lock_guard lock(_autoSContext->_confMutex); - if (_autoSchedule->_loadContext[CPU].isEnabled && _autoSchedule->_loadContext[CPU].isAlready) - return _autoSchedule->_loadContext[CPU].executableNetwork->GetMetric(name); - return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetMetric(name); + if (_autoSchedule->_pCTPUTLoadContext) { + return _autoSchedule->_pCTPUTLoadContext[0].executableNetwork->GetMetric(name); + } else { + if (_autoSchedule->_loadContext[CPU].isEnabled && _autoSchedule->_loadContext[CPU].isAlready) + return _autoSchedule->_loadContext[CPU].executableNetwork->GetMetric(name); + return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetMetric(name); + } } else if (name == METRIC_KEY(SUPPORTED_METRICS)) { IE_SET_METRIC_RETURN(SUPPORTED_METRICS, {METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS), diff --git a/src/plugins/auto/auto_executable_network.hpp b/src/plugins/auto/src/auto_executable_network.hpp similarity index 100% rename from src/plugins/auto/auto_executable_network.hpp rename to src/plugins/auto/src/auto_executable_network.hpp diff --git a/src/plugins/auto/auto_schedule.cpp b/src/plugins/auto/src/auto_schedule.cpp similarity index 65% rename from src/plugins/auto/auto_schedule.cpp rename to src/plugins/auto/src/auto_schedule.cpp index c645334c2d328b..0e699d6d7ee83e 100644 --- a/src/plugins/auto/auto_schedule.cpp +++ b/src/plugins/auto/src/auto_schedule.cpp @@ -3,7 +3,6 @@ // /////////////////////////////////////////////////////////////////////////////////////////////////// - #include "auto_schedule.hpp" #include "async_infer_request.hpp" #include "auto_executable_network.hpp" @@ -46,10 +45,7 @@ void AutoSchedule::GenerateWorkers(const std::string& device, } else { realDeviceName = device; } - auto itNumRequests = std::find_if(_autoSContext->_devicePriorities.cbegin(), _autoSContext->_devicePriorities.cend(), - [&realDeviceName](const DeviceInformation & d) { - return d.deviceName == realDeviceName; - }); + auto itNumRequests = deviceChecker().checkAndReturnIfDeviceInList(realDeviceName, _autoSContext->_devicePriorities, true); unsigned int optimalNum = 0; try { optimalNum = executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as(); @@ -140,9 +136,7 @@ bool AutoSchedule::selectOtherDevice(const std::string& currentDeviceName) { } else { realDeviceName = deviceName; } - const auto CurrentDeviceIter = std::find_if(_autoSContext->_devicePriorities.begin(), _autoSContext->_devicePriorities.end(), - [=](const DeviceInformation& d) -> bool { - return d.deviceName.find(realDeviceName) != std::string::npos;}); + const auto CurrentDeviceIter = deviceChecker().checkAndReturnIfDeviceInList(realDeviceName, _autoSContext->_devicePriorities); if (CurrentDeviceIter != _autoSContext->_devicePriorities.end()) { if (_autoSContext->_devicePriorities.size() == 1) { LOG_INFO_TAG("No other devices in _devicePriorities"); @@ -183,6 +177,23 @@ bool AutoSchedule::selectOtherDevice(const std::string& currentDeviceName) { return getExecutionDevices(_loadContext[FALLBACKDEVICE].deviceInfo.deviceName.c_str()); } }; + + auto removeInferFailDevice = [&](const std::string& deviceName) { + if (_autoSContext->_devicePriorities.size() > 1) { + const auto CurrentDeviceIter = + deviceChecker().checkAndReturnIfDeviceInList(deviceName, _autoSContext->_devicePriorities); + if (CurrentDeviceIter != _autoSContext->_devicePriorities.end()) { + _autoSContext->_devicePriorities.erase(CurrentDeviceIter); + return true; + } + } + return false; + }; + + if (_pCTPUTLoadContext) { + return removeInferFailDevice(currentDeviceName); + } + return getExecutionDevices(currentDeviceName); } } @@ -214,33 +225,41 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { bool isCumulative = (_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false; if (isCumulative) { - std::list validDevices = - _autoSContext->_plugin->GetValidDevice(_autoSContext->_devicePriorities, - _loadContext[ACTUALDEVICE].networkPrecision); + const auto& validDevices = _autoSContext->_devicePriorities; + // When the hint is ctput and there is only one device, the single-device logic is used if (validDevices.size() == 1) { - // When the hint is ctput and there is only one device, the single-device logic is used instead of - // the MULTI logic - // can not change _autoSContext->_performanceHint to THROUGHPUT, because GetMetric needs to return CTPUT _loadContext[ACTUALDEVICE].deviceInfo = validDevices.front(); _loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = IE::PluginConfigParams::THROUGHPUT; - isCumulative = false; - } else { - // When the hint is ctput and there are more than one device, the MULTI logic is used - std::string deviceName = "MULTI:"; + } else if (validDevices.size() > 1) { + _loadContext[ACTUALDEVICE].isEnabled = false; + // Total number of devices in CTPUT + _nCTputDeviceNums = validDevices.size(); + // Generate contexts for loading each device + _pCTPUTLoadContext.reset(new AutoLoadContext[_nCTputDeviceNums]); + int idx = 0; + DeviceInformation cpuDeviceInformation; for (auto& device : validDevices) { - deviceName += device.deviceName; - deviceName += ((device.deviceName == validDevices.back().deviceName) ? "" : ","); + if (device.deviceName.find("CPU") == std::string::npos) { + _pCTPUTLoadContext[idx].deviceInfo = device; + _pCTPUTLoadContext[idx].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = + IE::PluginConfigParams::THROUGHPUT; + idx++; + } else { + cpuDeviceInformation = device; + cpuDeviceInformation.config.insert( + {ov::affinity.name(), ov::Any(ov::Affinity::CORE).as()}); + } } - _loadContext[ACTUALDEVICE].deviceInfo.deviceName = deviceName; - _loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = - InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT; - _loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERF_COUNT)] = - _autoSContext->_needPerfCounters ? InferenceEngine::PluginConfigParams::YES - : InferenceEngine::PluginConfigParams::NO; - if (_autoSContext->_bindBuffer) - _loadContext[ACTUALDEVICE].deviceInfo.config[ov::intel_auto::device_bind_buffer.name()] = - InferenceEngine::PluginConfigParams::YES; + if (!cpuDeviceInformation.deviceName.empty()) { + _pCTPUTLoadContext[idx].deviceInfo = cpuDeviceInformation; + _pCTPUTLoadContext[idx].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = + IE::PluginConfigParams::THROUGHPUT; + } + } + if (_autoSContext->_LogTag == "MULTI") { + // MULTI's performance hint always is tput + _autoSContext->_performanceHint = IE::PluginConfigParams::THROUGHPUT; } } else { _loadContext[ACTUALDEVICE].deviceInfo = @@ -248,76 +267,118 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { _loadContext[ACTUALDEVICE].networkPrecision, _autoSContext->_modelPriority); } - LOG_INFO_TAG("select device:%s", _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str()); - bool isActualDevCPU = - _loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos && !isCumulative; - // if Actual device is CPU or perf_hint is cumulative, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE] - if (isActualDevCPU || isCumulative || !_autoSContext->_startupfallback) { - _loadContext[CPU].isEnabled = false; - } else { - const auto CPUIter = std::find_if(_autoSContext->_devicePriorities.begin(), _autoSContext->_devicePriorities.end(), - [=](const DeviceInformation& d) -> bool { return d.deviceName.find("CPU") != std::string::npos; }); - // if have CPU Device, enable _loadContext[CPU] - if (CPUIter != _autoSContext->_devicePriorities.end()) { - _loadContext[CPU].isEnabled = true; - _loadContext[CPU].deviceInfo = *CPUIter; - _loadContext[CPU].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = IE::PluginConfigParams::LATENCY; - _loadContext[CPU].workName = "CPU_HELP"; - LOG_INFO_TAG("will load CPU for accelerator"); - } else { + + auto loadDeviceTask = [&](AutoLoadContext* contextPtr, + const std::string& modelPath, + const IE::CNNNetwork& network, + bool isCumulative) { + TryToLoadNetWork(*contextPtr, modelPath, network, isCumulative); + if (contextPtr->isLoadSuccess) { + if (contextPtr->workName.empty()) { + contextPtr->workName = contextPtr->deviceInfo.deviceName; + } + GenerateWorkers(contextPtr->workName, contextPtr->executableNetwork); + // need lock + { + std::lock_guard lock(_autoSContext->_confMutex); + _autoSContext->_config.insert(contextPtr->deviceInfo.config.begin(), + contextPtr->deviceInfo.config.end()); + } + contextPtr->isAlready = true; + // reloadsuccess flag only for _loadContext[FALLBACKDEVICE] + contextPtr->isReloadSuccess = true; + auto& deviceName = contextPtr->deviceInfo.deviceName; + LOG_INFO_TAG("device:%s loading Network finished", deviceName.c_str()); + auto supported_config_keys = _autoSContext->_core->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS)) + .as>(); + DEBUG_RUN([this, &contextPtr, &deviceName, &supported_config_keys] { + std::lock_guard lock(_autoSContext->_confMutex); + for (const auto& cfg : supported_config_keys) { + try { + LOG_DEBUG_TAG("device:%s, GetConfig:%s=%s", + deviceName.c_str(), + cfg.c_str(), + contextPtr->executableNetwork->GetConfig(cfg).as().c_str()); + } catch (const IE::Exception&) { + } + } + }); + } + // Handle device load failure in case of ctput + if (isCumulative && !contextPtr->isLoadSuccess) { + std::string failedDeviceName = contextPtr->deviceInfo.deviceName; + std::lock_guard lock(_autoSContext->_confMutex); + const auto DeviceIter = deviceChecker().checkAndReturnIfDeviceInList(failedDeviceName, _autoSContext->_devicePriorities); + // Remove failed device from _devicePriorities + if (DeviceIter != _autoSContext->_devicePriorities.end()) { + _autoSContext->_devicePriorities.erase(DeviceIter); + } + // Remove failed device from ov::device::priorities in config + auto it_prior = _autoSContext->_config.find(ov::device::priorities.name()); + if (it_prior != _autoSContext->_config.end()) { + auto priorities = it_prior->second.as(); + size_t nPos = priorities.find(failedDeviceName); + if (nPos != std::string::npos) { + // If need to delete failed device and "," then length plus 1 + size_t nNameLen = (nPos + failedDeviceName.length()) == priorities.length() + ? failedDeviceName.length() + : failedDeviceName.length() + 1; + priorities.erase(nPos, nNameLen); + it_prior->second = priorities; + } + } + } + contextPtr->promise.set_value(); + // the first load network process finished + std::call_once(_firstLoadOC, [this]() { + _firstLoadPromise.set_value(); + }); + }; + if (_loadContext[ACTUALDEVICE].isEnabled) { + LOG_INFO_TAG("select device:%s", _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str()); + bool isActualDevCPU = _loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") != std::string::npos; + // if Actual device is CPU or perf_hint is cumulative, disabled _loadContext[CPU], only use + // _loadContext[ACTUALDEVICE] + if (isActualDevCPU || !_autoSContext->_startupfallback) { _loadContext[CPU].isEnabled = false; + } else { + const auto CPUIter = deviceChecker().checkAndReturnIfDeviceInList("CPU", _autoSContext->_devicePriorities); + // if have CPU Device, enable _loadContext[CPU] + if (CPUIter != _autoSContext->_devicePriorities.end()) { + _loadContext[CPU].isEnabled = true; + _loadContext[CPU].deviceInfo = *CPUIter; + _loadContext[CPU].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = IE::PluginConfigParams::LATENCY; + _loadContext[CPU].workName = "CPU_HELP"; + LOG_INFO_TAG("will load CPU for accelerator"); + } else { + _loadContext[CPU].isEnabled = false; + } + } + // initialize the rest members of load context + for (int i = 0; i < CONTEXTNUM; i++) { + if (_loadContext[i].isEnabled) { + _loadContext[i].future = _loadContext[i].promise.get_future(); + auto* contextPtr = &_loadContext[i]; + auto modelPath = _autoSContext->_modelPath; + auto network = _autoSContext->_network; + _loadContext[i].task = std::bind(loadDeviceTask, contextPtr, modelPath, network, isCumulative); + } } } - // initialize the rest members of load context - for (int i = 0; i < CONTEXTNUM; i++) { - if (_loadContext[i].isEnabled) { - _loadContext[i].future = _loadContext[i].promise.get_future(); - auto* contextPtr = &_loadContext[i]; + std::vector otherDevicesloads; + std::vector cpuLoads; + if (_pCTPUTLoadContext) { + for (size_t i = 0; i < _nCTputDeviceNums; i++) { + auto* contextPtr = &_pCTPUTLoadContext[i]; auto modelPath = _autoSContext->_modelPath; auto network = _autoSContext->_network; - _loadContext[i].task = [this, contextPtr, modelPath, network, isCumulative]() mutable { - TryToLoadNetWork(*contextPtr, modelPath, network); - if (contextPtr->isLoadSuccess) { - if (contextPtr->workName.empty()) { - contextPtr->workName = contextPtr->deviceInfo.deviceName; - } - if (!isCumulative) - GenerateWorkers(contextPtr->workName, contextPtr->executableNetwork); - //need lock - { - std::lock_guard lock(_autoSContext->_confMutex); - _autoSContext->_config.insert(contextPtr->deviceInfo.config.begin(), contextPtr->deviceInfo.config.end()); - } - contextPtr->isAlready = true; - // reloadsuccess flag only for _loadContext[FALLBACKDEVICE] - contextPtr->isReloadSuccess = true; - auto& deviceName = contextPtr->deviceInfo.deviceName; - LOG_INFO_TAG("device:%s loading Network finished", deviceName.c_str()); - if (!isCumulative) { - auto supported_config_keys = - _autoSContext->_core->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS)) - .as>(); - DEBUG_RUN([this, &contextPtr, &deviceName, &supported_config_keys] { - std::lock_guard lock(_autoSContext->_confMutex); - for (const auto& cfg : supported_config_keys) { - try { - LOG_DEBUG_TAG( - "device:%s, GetConfig:%s=%s", - deviceName.c_str(), - cfg.c_str(), - contextPtr->executableNetwork->GetConfig(cfg).as().c_str()); - } catch (const IE::Exception&) { - } - } - }); - } - } - contextPtr->promise.set_value(); - // the first load network process finished - std::call_once(_firstLoadOC, [this]() { - _firstLoadPromise.set_value(); - }); - }; + _pCTPUTLoadContext[i].task = std::bind(loadDeviceTask, contextPtr, modelPath, network, isCumulative); + if (i == _nCTputDeviceNums - 1 && + _pCTPUTLoadContext[i].deviceInfo.deviceName.find("CPU") != std::string::npos) { + cpuLoads.push_back(_pCTPUTLoadContext[i].task); + } else { + otherDevicesloads.push_back(_pCTPUTLoadContext[i].task); + } } } OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, @@ -350,13 +411,11 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { _loadContext[CPU].future.wait(); // clean up helper infer requests // first, wait for all the remaining requests to finish - if (!_autoSContext->_runtimeFallback) { - for (auto& iter : _workerRequests["CPU_HELP"]) { - try { - iter._inferRequest._ptr->Wait(IE::InferRequest::WaitMode::RESULT_READY); - } catch (const IE::Exception& iie) { - LOG_DEBUG_TAG("No infer results expected, infer in CPU_HELP throw some errors: %s", iie.what()); - } + for (auto& iter : _workerRequests["CPU_HELP"]) { + try { + iter._inferRequest._ptr->Wait(IE::InferRequest::WaitMode::RESULT_READY); + } catch (const IE::Exception& iie) { + LOG_DEBUG_TAG("No infer results expected, infer in CPU_HELP throw some errors: %s", iie.what()); } } // late enough to check the idle queue now @@ -411,14 +470,38 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { } _loadContext[ACTUALDEVICE].task(); } else { - // only one device need to load network, do not need to load it async - _loadContext[ACTUALDEVICE].task(); - _passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork; + if (_pCTPUTLoadContext) { + for (auto&& device : _autoSContext->_devicePriorities) { + // initialize containers before run async task, if not initialized, it will hang during infer + _idleWorkerRequests[device.deviceName]; + _workerRequests[device.deviceName]; + _inferPipelineTasksDeviceSpecific[device.deviceName] = nullptr; + } + _executor = _autoSContext->_plugin->executorManager()->getIdleCPUStreamsExecutor(IStreamsExecutor::Config{ + "CTPUTDeviceAsyncLoad", + static_cast(std::thread::hardware_concurrency()) /* max possible #streams*/, + 0 /*default threads per stream, workaround for ticket 62376*/, + IStreamsExecutor::ThreadBindingType::NONE}); + // load devices other than CPU first + if (otherDevicesloads.size() > 0) { + // Wait for the devices other than CPU to load the network + _executor->runAndWait(otherDevicesloads); + } + // Finally load the CPU + if (cpuLoads.size() > 0) { + // Wait for CPU to load the network + _executor->runAndWait(cpuLoads); + } + } else { + // only one device need to load network, do not need to load it async + _loadContext[ACTUALDEVICE].task(); + _passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork; + } } - WaitFirstNetworkReady(); + _autoSContext->_hwExecutableNetwork = WaitFirstNetworkReady(); } -void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network) { +void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative) { auto& device = context.deviceInfo.deviceName; auto& deviceConfig = context.deviceInfo.config; auto& deviceList = context.metaDevices; @@ -458,7 +541,7 @@ void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& context.errMessage += device + ":" + e.what(); context.isLoadSuccess = false; } - if (context.isLoadSuccess || curDevIsCPU) { + if (context.isLoadSuccess || curDevIsCPU || isCumulative) { return; } // need to reload network, unregister it's priority @@ -469,11 +552,9 @@ void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& // maybe 0 is loaded to VPUX, 1 is loaded to iGPU _autoSContext->_plugin->UnregisterPriority(_autoSContext->_modelPriority, context.deviceInfo.uniqueName); // remove the current device from deviceList - auto eraseDevice = std::find_if(deviceList.begin(), deviceList.end(), - [device](DeviceInformation & d) { - return d.deviceName == device; - }); - deviceList.erase(eraseDevice); + auto eraseDevice = deviceChecker().checkAndReturnIfDeviceInList(device, deviceList, true); + if (eraseDevice != deviceList.end()) + deviceList.erase(eraseDevice); if (deviceList.empty()) { return; } @@ -512,10 +593,10 @@ void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& } LOG_DEBUG_TAG("try to load %s", context.deviceInfo.deviceName.c_str()); // try to load this candidate device - TryToLoadNetWork(context, modelPath, network); + TryToLoadNetWork(context, modelPath, network, isCumulative); } -void AutoSchedule::WaitFirstNetworkReady() { +SoExecNetwork AutoSchedule::WaitFirstNetworkReady() { if (_firstLoadFuture.valid()) { // wait for the first loading finished _firstLoadFuture.wait(); @@ -523,7 +604,7 @@ void AutoSchedule::WaitFirstNetworkReady() { // check if there is any device that have loaded network successfully for (int i = CONTEXTNUM - 2; i >= 0; i--) { if (_loadContext[i].isEnabled && _loadContext[i].isAlready) { - return; + return _loadContext[i].executableNetwork; } } // the first loading is failed, wait for another loading @@ -532,17 +613,42 @@ void AutoSchedule::WaitFirstNetworkReady() { _loadContext[i].future.wait(); // check if loading is successful if (_loadContext[i].isAlready) { - return; + return _loadContext[i].executableNetwork; } } } + std::ostringstream result; //print errMessage + result << "Load network failed, "; for (int i = CONTEXTNUM - 2; i >= 0; i--) { if (_loadContext[i].isEnabled) { + result << _loadContext[i].errMessage.c_str(); + result << "; "; LOG_ERROR_TAG("load failed, %s", _loadContext[i].errMessage.c_str()); } } - IE_THROW() << GetLogTag() << "load all devices failed"; + // devices loaded successfully in CTPUT + SoExecNetwork execNetwork; + if (_pCTPUTLoadContext) { + int nLoadSucNums = 0; + for (size_t i = 0; i < _nCTputDeviceNums; i++) { + // check if device loaded successfully + if (_pCTPUTLoadContext[i].isAlready) { + if (!execNetwork) { + execNetwork = _pCTPUTLoadContext[i].executableNetwork; + } + nLoadSucNums++; + } else { + result << _pCTPUTLoadContext[i].errMessage.c_str(); + result << "; "; + } + } + // one or more devices loaded successfully + if (nLoadSucNums > 0) { + return execNetwork; + } + } + IE_THROW() << "[" << GetLogTag() << "] " << result.str(); } void AutoSchedule::WaitActualNetworkReady() const { @@ -559,30 +665,43 @@ void AutoSchedule::WaitActualNetworkReady() const { bool AutoSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) { std::vector devices; // AUTO work mode + // Devices that fail infer will be removed from the priority list in the callback, need lock here + std::unique_lock lock(_autoSContext->_fallbackMutex); if (!preferred_device.empty()) { - // if the device needed by customer is not ready, need to wait for it - WaitActualNetworkReady(); - // the preferred_device should be the selected device in AUTO work mode - if (preferred_device != _loadContext[ACTUALDEVICE].deviceInfo.deviceName) { + if (_pCTPUTLoadContext) { + devices = _autoSContext->_devicePriorities; + } else { + // if the device needed by customer is not ready, need to wait for it + WaitActualNetworkReady(); + devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo); + } + if (!deviceChecker().checkIfDeviceInList(preferred_device, devices)) { + lock.unlock(); IE_THROW(NotFound) << "The preferred device should be the selected device"; } - devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo); } else { - // _acceleratorDevice could be the same as _cpuDevice, such as AUTO:CPU - if (_loadContext[FALLBACKDEVICE].isAlready) { - devices.push_back(_loadContext[FALLBACKDEVICE].deviceInfo); + if (_pCTPUTLoadContext) { + for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) { + devices.push_back(_autoSContext->_devicePriorities[i]); + } } else { - if (_loadContext[ACTUALDEVICE].isAlready) { - devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo); + // _acceleratorDevice could be the same as _cpuDevice, such as AUTO:CPU + if (_loadContext[FALLBACKDEVICE].isAlready) { + devices.push_back(_loadContext[FALLBACKDEVICE].deviceInfo); } else { - // replace deviceName with workName, so schedule can select correct - // idleWorkerQueue - auto deviceInfo = _loadContext[CPU].deviceInfo; - deviceInfo.deviceName = _loadContext[CPU].workName; - devices.push_back(std::move(deviceInfo)); + if (_loadContext[ACTUALDEVICE].isAlready) { + devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo); + } else { + // replace deviceName with workName, so schedule can select correct + // idleWorkerQueue + auto deviceInfo = _loadContext[CPU].deviceInfo; + deviceInfo.deviceName = _loadContext[CPU].workName; + devices.push_back(std::move(deviceInfo)); + } } } } + lock.unlock(); for (auto&& device : devices) { if (!preferred_device.empty() && (device.deviceName != preferred_device)) { continue; @@ -644,29 +763,13 @@ IInferPtr AutoSchedule::CreateInferRequest() { if (!syncRequestImpl) syncRequestImpl = CreateInferRequestImpl(execNetwork->_networkInputs, execNetwork->_networkOutputs); syncRequestImpl->setPointerToExecutableNetworkInternal(execNetwork); - bool isCumulative = (_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false; - bool isCTPUTSingleDevice = - isCumulative && _loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("MULTI:") == std::string::npos ? true - : false; - if ((_passthroughExeNet && !isCumulative) || isCTPUTSingleDevice) { - std::string perfmode; - try { - perfmode = _passthroughExeNet->GetConfig( - CONFIG_KEY(PERFORMANCE_HINT)).as(); - } catch (const IE::Exception&) { - LOG_INFO("query perf hint from passthrough network failed"); - } - if (_autoSContext->_batchingDisabled || perfmode != CONFIG_VALUE(THROUGHPUT)) { - syncRequestImpl->setPointerToSo(_passthroughExeNet._so); - } else { - auto so = _passthroughExeNet._ptr->GetPointerToSo(); - // Get the _so from passthrough executable network when batch plugin is disable. - if (!so) - so = _passthroughExeNet._so; - syncRequestImpl->setPointerToSo(so); - } + if (_passthroughExeNet) { + auto so = _passthroughExeNet._ptr->GetPointerToSo(); + // Get the _so from passthrough executable network when batch plugin is disable. + if (!so) + so = _passthroughExeNet._so; + syncRequestImpl->setPointerToSo(so); } else if (std::static_pointer_cast(syncRequestImpl)->GetSharedRequest()) { - // cumulative case, load to MULTI:* auto sharedMultiRequest = std::static_pointer_cast(syncRequestImpl)->GetSharedRequest(); if (sharedMultiRequest._ptr->getPointerToSo()) syncRequestImpl->setPointerToSo(sharedMultiRequest._ptr->getPointerToSo()); diff --git a/src/plugins/auto/auto_schedule.hpp b/src/plugins/auto/src/auto_schedule.hpp similarity index 83% rename from src/plugins/auto/auto_schedule.hpp rename to src/plugins/auto/src/auto_schedule.hpp index 6e08a94005a1e4..2d24dc66244794 100644 --- a/src/plugins/auto/auto_schedule.hpp +++ b/src/plugins/auto/src/auto_schedule.hpp @@ -50,6 +50,8 @@ class AutoSchedule : public MultiSchedule { public: AutoLoadContext _loadContext[CONTEXTNUM]; + std::unique_ptr _pCTPUTLoadContext = nullptr; + size_t _nCTputDeviceNums; protected: void GenerateWorkers(const std::string& device, const SoExecNetwork& executableNetwork) override; @@ -57,10 +59,16 @@ class AutoSchedule : public MultiSchedule { static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyPriorityWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device); DeviceMap _idleWorkerRequests; + AutoScheduleContext::Ptr _autoSContext; private: - void WaitFirstNetworkReady(); - void TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network); + /** + * @brief wait for one of the executable network to finish loading. + * @return An SoPtr object hold an available executable network loaded to HW device. + * @note An exception will be thrown if all loading of network to hw device fails. + */ + SoExecNetwork WaitFirstNetworkReady(); + void TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative); bool selectOtherDevice(const std::string& currentDeviceName); IE::Task releaseActualdeviceTask; @@ -71,8 +79,6 @@ class AutoSchedule : public MultiSchedule { std::future _firstLoadFuture; std::promise _firstLoadPromise; bool _exitFlag = {false}; - size_t _cpuHelpInferCount = 0; - AutoScheduleContext::Ptr _autoSContext; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/auto/src/bind_multi_schedule.cpp b/src/plugins/auto/src/bind_multi_schedule.cpp new file mode 100644 index 00000000000000..3e623bd984c2ab --- /dev/null +++ b/src/plugins/auto/src/bind_multi_schedule.cpp @@ -0,0 +1,98 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/////////////////////////////////////////////////////////////////////////////////////////////////// +#include "async_infer_request.hpp" +#include "plugin.hpp" +#include "bind_multi_schedule.hpp" +// ------------------------------MultiSchedule---------------------------- +namespace MultiDevicePlugin { + +void BinderMultiSchedule::init(const ScheduleContext::Ptr& sContext) { + AutoSchedule::init(sContext); + LOG_INFO_TAG("enable bind buffer for AUTO"); +} + +Pipeline BinderMultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) { + Pipeline pipeline; + struct RequestExecutor : ITaskExecutor { + explicit RequestExecutor(InferenceEngine::SoIInferRequestInternal& inferRequest) : _inferRequest(inferRequest) { + _inferRequest->SetCallback([this](std::exception_ptr exceptionPtr) mutable { + _exceptionPtr = exceptionPtr; + auto capturedTask = std::move(_task); + capturedTask(); + }); + } + void run(InferenceEngine::Task task) override { + _task = std::move(task); + _inferRequest->StartAsync(); + }; + InferenceEngine::SoIInferRequestInternal& _inferRequest; + std::exception_ptr _exceptionPtr; + InferenceEngine::Task _task; + }; + auto requestExecutor = std::make_shared( + std::static_pointer_cast(syncInferRequest)->GetSharedRequest()); + pipeline.emplace_back(requestExecutor, [requestExecutor] { + if (nullptr != requestExecutor->_exceptionPtr) { + std::rethrow_exception(requestExecutor->_exceptionPtr); + } + }); + return pipeline; +} + +BinderMultiSchedule::~BinderMultiSchedule() { +} + +IInferPtr BinderMultiSchedule::CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs) { + auto num = _numRequestsCreated++; + size_t sum = 0; + SoInfer request_to_share_blobs_with; + // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests + // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) + for (const auto& device : _autoSContext->_devicePrioritiesInitial) { + auto& dev_requests = _workerRequests[device.deviceName]; + if ((num - sum) < dev_requests.size()) { + request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; + break; + } + sum += dev_requests.size(); + } + if (!request_to_share_blobs_with) { + IE_THROW() << + "binder mode does not allow oversubsciption of infer requests" + " please use optimal infer request"; + } + auto syncImpl = std::make_shared(inputs, outputs, request_to_share_blobs_with); + return syncImpl; +} + +IInferPtr BinderMultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkInputs, + IE::OutputsDataMap networkOutputs) { + auto num = _numRequestsCreated++; + SoInfer request_to_share_blobs_with; + size_t sum = 0; + // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests + // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) + for (const auto& device : _autoSContext->_devicePrioritiesInitial) { + auto& dev_requests = _workerRequests[device.deviceName]; + if ((num - sum) < dev_requests.size()) { + request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; + break; + } + sum += dev_requests.size(); + } + if (!request_to_share_blobs_with) { + IE_THROW() << + "binder mode does not allow oversubsciption of infer requests" + " please use optimal infer request"; + } + auto syncImpl = std::make_shared(networkInputs, networkOutputs, request_to_share_blobs_with); + return syncImpl; +} + +} // namespace MultiDevicePlugin + diff --git a/src/plugins/auto/bind_multi_schedule.hpp b/src/plugins/auto/src/bind_multi_schedule.hpp similarity index 68% rename from src/plugins/auto/bind_multi_schedule.hpp rename to src/plugins/auto/src/bind_multi_schedule.hpp index 2217f4fb6d7a9d..7f24af39bac108 100644 --- a/src/plugins/auto/bind_multi_schedule.hpp +++ b/src/plugins/auto/src/bind_multi_schedule.hpp @@ -5,7 +5,7 @@ /////////////////////////////////////////////////////////////////////////////////////////////////// #pragma once -#include "multi_schedule.hpp" +#include "auto_schedule.hpp" #ifdef MULTIUNITTEST #define MOCKTESTMACRO virtual @@ -15,22 +15,14 @@ #endif namespace MultiDevicePlugin { -class BinderMultiSchedule : public MultiSchedule { +class BinderMultiSchedule : public AutoSchedule { public: using Ptr = std::shared_ptr; IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override; IE::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector>& inputs, const std::vector>& outputs) override; - void run(IE::Task inferTask) override; void init(const ScheduleContext::Ptr& sContext) override; Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) override; virtual ~BinderMultiSchedule(); - -protected: - static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device); - bool ScheduleToWorkerInferRequest(IE::Task, DeviceName preferred_device = "") override; - -protected: - thread_local static IE::IInferRequestInternal* _sharedRequest; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/auto/common.hpp b/src/plugins/auto/src/common.hpp similarity index 65% rename from src/plugins/auto/common.hpp rename to src/plugins/auto/src/common.hpp index 1b2fb9a85ca1eb..46a31268bf9cde 100644 --- a/src/plugins/auto/common.hpp +++ b/src/plugins/auto/src/common.hpp @@ -81,6 +81,59 @@ struct WorkerInferRequest { MultiImmediateExecutor::Ptr _fallbackExec; }; +struct deviceChecker { + template ::type, std::string>::value, bool>::type = true, + typename U = typename std::vector::const_iterator> + U checkAndReturnIfDeviceInList(const std::string& target, const std::vector& deviceList, bool exactMatch = false) { + if (exactMatch) { + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { return d == target; }); + } + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T & d) { + return d.find(target) != std::string::npos; + }); + } + template ::type, std::string>::value, bool>::type = true> + bool checkIfDeviceInList(const std::string& target, const std::vector& deviceList, bool exactMatch = false) { + if (exactMatch) { + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { return d == target; }) != deviceList.cend(); + } + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { + return d.find(target) != std::string::npos; + }) != deviceList.end(); + } + template ::type, DeviceInformation>::value, bool>::type = true, + typename U = typename std::vector::const_iterator> + U checkAndReturnIfDeviceInList(const std::string& target, const std::vector& deviceList, bool exactMatch = false) { + if (exactMatch) { + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { return d.deviceName == target; }); + } + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { + return d.deviceName.find(target) != std::string::npos; + }); + } + template ::type, DeviceInformation>::value, bool>::type = true> + bool checkIfDeviceInList(const std::string& target, const std::vector& deviceList, bool exactMatch = false) { + if (exactMatch) { + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { return d.deviceName == target; }) != deviceList.end(); + } + return std::find_if(deviceList.begin(), deviceList.end(), + [&target](const T& d) { + return d.deviceName.find(target) != std::string::npos; + }) != deviceList.end(); + } +}; + using NotBusyPriorityWorkerRequests = IE::ThreadSafeBoundedPriorityQueue>; using NotBusyWorkerRequests = IE::ThreadSafeBoundedQueue; template @@ -143,7 +196,6 @@ class MultiScheduleContext : public ScheduleContext { std::mutex _mutex; bool _needPerfCounters; bool _batchingDisabled = {false}; - bool _bindBuffer = false; bool _startupfallback = true; bool _runtimeFallback = true; virtual ~MultiScheduleContext() = default; @@ -161,6 +213,7 @@ class AutoScheduleContext : public MultiScheduleContext { std::mutex _confMutex; std::mutex _fallbackMutex; MultiDeviceInferencePlugin* _plugin; + SoExecNetwork _hwExecutableNetwork; virtual ~AutoScheduleContext() = default; }; diff --git a/src/plugins/auto/executable_network.cpp b/src/plugins/auto/src/executable_network.cpp similarity index 100% rename from src/plugins/auto/executable_network.cpp rename to src/plugins/auto/src/executable_network.cpp diff --git a/src/plugins/auto/executable_network.hpp b/src/plugins/auto/src/executable_network.hpp similarity index 100% rename from src/plugins/auto/executable_network.hpp rename to src/plugins/auto/src/executable_network.hpp diff --git a/src/plugins/auto/infer_request.cpp b/src/plugins/auto/src/infer_request.cpp similarity index 84% rename from src/plugins/auto/infer_request.cpp rename to src/plugins/auto/src/infer_request.cpp index 2453ce60c52881..be047f02ba190f 100644 --- a/src/plugins/auto/infer_request.cpp +++ b/src/plugins/auto/src/infer_request.cpp @@ -5,9 +5,12 @@ /////////////////////////////////////////////////////////////////////////////////////////////////// #include "infer_request.hpp" +#include +#include #include #include #include +#include namespace MultiDevicePlugin { @@ -20,6 +23,12 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const std::vector& in : inputs) { + modelInputsMap[ov::op::util::get_ie_output_name(ngraph::Output(in))] = in; + } + for (const std::shared_ptr& out : outputs) { + modelOutputsMap[ov::op::util::get_ie_output_name(out->input_value(0))] = out; + } CreateInferRequest(request_to_share_blobs_with, ctx); } @@ -57,6 +66,16 @@ void MultiDeviceInferRequest::CreateInferRequest(const InferenceEngine::SoIInfer auto l = it.second->getLayout(); auto p = it.second->getPrecision(); auto dims = it.second->getTensorDesc().getDims(); + // for 1.0 API, dims is not dynamic anyway + if (InferenceEngine::details::product(dims) == 0 && !modelOutputsMap.empty()) { + // replace the dims with one from dynamic shape + const auto outputNodeItr = modelOutputsMap.find(it.first); + if (outputNodeItr != modelOutputsMap.end()) { + const auto shape = outputNodeItr->second->get_input_partial_shape(0); + // update dims + dims = shape.get_max_shape(); + } + } TensorDesc desc = TensorDesc(p, dims, l); if (ctx) { diff --git a/src/plugins/auto/infer_request.hpp b/src/plugins/auto/src/infer_request.hpp similarity index 94% rename from src/plugins/auto/infer_request.hpp rename to src/plugins/auto/src/infer_request.hpp index d540fd686ba8c6..181aedcc97a0a4 100644 --- a/src/plugins/auto/infer_request.hpp +++ b/src/plugins/auto/src/infer_request.hpp @@ -57,6 +57,8 @@ class MultiDeviceInferRequest : public InferenceEngine::IInferRequestInternal { void CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with, InferenceEngine::RemoteContext::Ptr ctx); InferenceEngine::SoIInferRequestInternal _sharedRequest; + std::unordered_map> modelInputsMap; + std::unordered_map> modelOutputsMap; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/auto/itt.hpp b/src/plugins/auto/src/itt.hpp similarity index 100% rename from src/plugins/auto/itt.hpp rename to src/plugins/auto/src/itt.hpp diff --git a/src/plugins/auto/multi_executable_network.cpp b/src/plugins/auto/src/multi_executable_network.cpp similarity index 100% rename from src/plugins/auto/multi_executable_network.cpp rename to src/plugins/auto/src/multi_executable_network.cpp diff --git a/src/plugins/auto/multi_executable_network.hpp b/src/plugins/auto/src/multi_executable_network.hpp similarity index 100% rename from src/plugins/auto/multi_executable_network.hpp rename to src/plugins/auto/src/multi_executable_network.hpp diff --git a/src/plugins/auto/multi_schedule.cpp b/src/plugins/auto/src/multi_schedule.cpp similarity index 94% rename from src/plugins/auto/multi_schedule.cpp rename to src/plugins/auto/src/multi_schedule.cpp index 888c25095bd56e..7448d3603e5d1c 100644 --- a/src/plugins/auto/multi_schedule.cpp +++ b/src/plugins/auto/src/multi_schedule.cpp @@ -245,12 +245,13 @@ MultiSchedule::~MultiSchedule() { reqAllStartTimes.sort(std::less