diff --git a/.ci/azure/linux_debian.yml b/.ci/azure/linux_debian.yml index 96db6129c72ee1..e94a0819705bd7 100644 --- a/.ci/azure/linux_debian.yml +++ b/.ci/azure/linux_debian.yml @@ -228,7 +228,7 @@ jobs: wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2023.list - sudo apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/intel-openvino-2023.list + sudo apt-get update sudo apt-get install openvino -y # install our local one and make sure the conflicts are resolved sudo apt-get install --no-install-recommends dpkg-dev -y diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 74661a77af5794..80e95ea5a9fe73 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -154,6 +154,7 @@ updates: time: "09:00" timezone: "Asia/Dubai" assignees: - - "ilyachur" + - "akashchi" + - "mryzhov" - "ilya-lavrenov" open-pull-requests-limit: 3 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 486cc879ec02d0..d7cba0bc390e59 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -585,7 +585,7 @@ jobs: --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py \ --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_zoo_models.py \ --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_backend.py - + - name: Python API snippets run: | source ${{ env.INSTALL_DIR }}/setupvars.sh @@ -774,12 +774,13 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/functional_test_utils/requirements.txt - - name: Cache Tests Execution Time - id: tests-functional-cpu-cache - uses: actions/cache@v3 + - name: Restore tests execution time + uses: actions/cache/restore@v3 with: path: ${{ env.PARALLEL_TEST_CACHE }} - key: ${{ runner.os }}-tests-functional-cpu-cache + key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-tests-functional-cpu-stamp - name: Intel CPU plugin func tests (parallel) run: | @@ -787,6 +788,13 @@ jobs: python3 ${{ env.PARALLEL_TEST_SCRIPT }} -e ${{ env.INSTALL_TEST_DIR }}/ov_cpu_func_tests -c ${{ env.PARALLEL_TEST_CACHE }} -w ${{ env.INSTALL_TEST_DIR }} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* timeout-minutes: 25 + - name: Save tests execution time + uses: actions/cache/save@v3 + if: github.ref_name == 'master' + with: + path: ${{ env.PARALLEL_TEST_CACHE }} + key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} + - name: Upload Test Results uses: actions/upload-artifact@v3 if: ${{ always() }} diff --git a/.github/workflows/linux_debian.yml b/.github/workflows/linux_debian.yml index b517a368c46ced..bff429be48c90a 100644 --- a/.github/workflows/linux_debian.yml +++ b/.github/workflows/linux_debian.yml @@ -219,7 +219,7 @@ jobs: wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2023.list - sudo apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/intel-openvino-2023.list + sudo apt-get update sudo apt-get install openvino -y # install our local one and make sure the conflicts are resolved sudo apt-get install --no-install-recommends dpkg-dev -y diff --git a/.github/workflows/linux_onnxruntime.yml b/.github/workflows/linux_onnxruntime.yml new file mode 100644 index 00000000000000..178aae4fefac6b --- /dev/null +++ b/.github/workflows/linux_onnxruntime.yml @@ -0,0 +1,182 @@ +name: Linux ONNX Runtime (Ubuntu 20.04, Python 3.11) +on: + workflow_dispatch: + schedule: + # run daily at 00:00 + - cron: '0 0 * * *' +# pull_request: +# paths-ignore: +# - '**/docs/**' +# - 'docs/**' +# - '**/**.md' +# - '**.md' +# - '**/layer_tests_summary/**' +# - '**/conformance/**' +# push: +# paths-ignore: +# - '**/docs/**' +# - 'docs/**' +# - '**/**.md' +# - '**.md' +# - '**/layer_tests_summary/**' +# - '**/conformance/**' +# branches: +# - master + +concurrency: + group: ${{ github.head_ref || github.run_id }}-linux-onnx-runtime + cancel-in-progress: true + +jobs: + Build: + # TODO: remove. Temporary measure to prevent the workflow from scheduling on forks. + if: ${{ github.repository_owner == 'openvinotoolkit' }} + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-8-cores + env: + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + CMAKE_CXX_LINKER_LAUNCHER: ccache + CMAKE_C_LINKER_LAUNCHER: ccache + BUILD_TYPE: Release + OPENVINO_REPO: ${{ github.workspace }}/openvino + ONNX_RUNTIME_REPO: ${{ github.workspace }}/onnxruntime + ONNX_RUNTIME_UTILS: ${{ github.workspace }}/openvino/.ci/azure/ci_utils/onnxruntime + ONNX_RUNTIME_BUILD_DIR: ${{ github.workspace }}/onnxruntime/build + BUILD_DIR: ${{ github.workspace }}/build + INSTALL_DIR: ${{ github.workspace }}/install/openvino + steps: + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + path: 'openvino' + submodules: 'true' + + - name: Clone ONNX Runtime + run: | + branch=`tr -s '\n ' < ${{ env.ONNX_RUNTIME_UTILS }}/version` + git clone --branch $branch --single-branch --recursive https://github.com/microsoft/onnxruntime.git ${{ env.ONNX_RUNTIME_REPO }} + + - name: Create Directories + run: | + mkdir -p ${{ env.BUILD_DIR }} + mkdir -p ${{ env.INSTALL_DIR }} + + - name: Setup Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + # + # Dependencies + # + + - name: Install build dependencies + run: | + sudo -E ${{ env.OPENVINO_REPO }}/install_build_dependencies.sh + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + max-size: "2000M" + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save: ${{ github.ref_name == 'master' && 'true' || 'false' }} + verbose: 2 + key: ${{ github.job }}-linux-onnx-runtime + restore-keys: | + ${{ github.job }}-linux-onnx-runtime + + # + # Build + # + + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + + - name: CMake configure + run: | + cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ + -DENABLE_INTEL_GNA=OFF \ + -DENABLE_INTEL_GPU=OFF \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_PROFILING_ITT=OFF \ + -DENABLE_SAMPLES=OFF \ + -DENABLE_OV_TF_FRONTEND=OFF \ + -DENABLE_OV_TF_LITE=OFF \ + -DENABLE_OV_PADDLE_FRONTEND=OFF \ + -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -S ${{ env.OPENVINO_REPO }} \ + -B ${{ env.BUILD_DIR }} + + - name: Clean ccache stats + run: ccache --zero-stats --show-config + + - name: Build + run: cmake --build ${{ env.BUILD_DIR }} --parallel ${{ steps.cpu-cores.outputs.count }} --config ${{ env.BUILD_TYPE }} + + - name: Show ccache stats + run: ccache --show-stats + + - name: Install OpenVINO + run: cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake + + - name: Build Lin ONNX Runtime + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + + ${{ env.ONNX_RUNTIME_REPO }}/build.sh \ + --config RelWithDebInfo \ + --use_openvino CPU_FP32 \ + --build_shared_lib \ + --parallel \ + --skip_tests \ + --compile_no_warning_as_error \ + --build_dir ${{ env.ONNX_RUNTIME_BUILD_DIR }} + env: + CXXFLAGS: "-Wno-error=deprecated-declarations" + + - name: Run onnxruntime_test_all + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + skip_tests=$(tr -s '\n ' ':' < ${{ env.ONNX_RUNTIME_UTILS }}/skip_tests) + ./onnxruntime_test_all --gtest_filter=-$skip_tests + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo + + - name: Run onnxruntime_shared_lib_test + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ./onnxruntime_shared_lib_test --gtest_filter=-CApiTest.test_custom_op_openvino_wrapper_library + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo + + - name: Run onnxruntime_global_thread_pools_test + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ./onnxruntime_global_thread_pools_test + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo + + - name: Run onnxruntime_api_tests_without_env + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ./onnxruntime_api_tests_without_env + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo + + - name: Run pytorch-converted tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ./onnx_test_runner "${{ env.ONNX_RUNTIME_REPO }}/cmake/external/onnx/onnx/backend/test/data/pytorch-converted" + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo + + - name: Run pytorch-operator tests + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ./onnx_test_runner "${{ env.ONNX_RUNTIME_REPO }}/cmake/external/onnx/onnx/backend/test/data/pytorch-operator" + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b552b3da3406c..e9d4760eb86916 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,8 +106,6 @@ function(openvino_developer_export_targets) if(TARGET "${target_name}") get_target_property(original_name ${target_name} ALIASED_TARGET) if(TARGET "${original_name}") - message(STATUS "The name ${target_name} is an ALIAS for ${original_name}. " - "It will be exported to the OpenVINODeveloperPackage with the original name.") list(REMOVE_ITEM ${EXPORT_COMPONENT} ${target_name}) list(APPEND ${EXPORT_COMPONENT} ${original_name}) endif() diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 48adc7acf3980a..d359a40aa6df51 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -91,6 +91,7 @@ macro(ov_cpack_settings) # - 2022.3 is the first release where Debian updated packages are introduced, others 2022.3.X are LTS 2022.3.0 2022.3.1 2022.3.2 2022.3.3 2022.3.4 2022.3.5 2023.0.0 2023.0.1 2023.0.2 2023.0.3 + 2023.1.0 ) # diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index ad1172b6c833ae..077e970d138bc4 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -77,6 +77,7 @@ macro(ov_cpack_settings) # - 2022.3 is the first release where RPM updated packages are introduced, others 2022.3.X are LTS 2022.3.0 2022.3.1 2022.3.2 2022.3.3 2022.3.4 2022.3.5 2023.0.0 2023.0.1 2023.0.2 2023.0.3 + 2023.1.0 ) find_host_program(rpmlint_PROGRAM NAMES rpmlint DOC "Path to rpmlint") diff --git a/cspell.json b/cspell.json index cfbab3df891ec3..f59d00a6a052f6 100644 --- a/cspell.json +++ b/cspell.json @@ -4,80 +4,407 @@ "dictionaryDefinitions": [], "dictionaries": [], "words": [ + "aarch64", + "acdadcfa", + "acea", + "abmrd", + "acfb", + "acosh", + "Acosh", + "adfcd", + "addcmul", + "addif", + "addmm", + "aeaa", + "agem", + "agew", + "armeabi", "armhf", + "artefacts", + "ARTEFACTS", + "Asinh", + "asynch", + "Atanh", + "autodoc", + "Autograd", + "autoplugin", + "AUTOPLUGIN", "autoremove", + "autosummary", + "bace", + "Backprop", "bblayers", + "Beautif", + "Bilat", + "bindir", "bitbake", + "BFYX", + "BFXY", + "bkgr", + "brctl", + "Bucketize", + "BUILDDIR", "buildtools", + "buildsystems", + "BYXF", + "bvalue", + "bvlc", "caffe", + "caffemodel", + "camvid", + "cbba", + "cbcd", + "cdad", + "cdrom", "chrpath", + "classov", + "cldnn", + "clumber", + "codepath", + "codepaths", + "coeffs", + "concat", + "Concat", + "Conts", + "constexpr", + "consts", + "Consts", + "conv", + "Convolutional", + "CPPLINT", + "cpplint", + "crbegin", + "crend", + "ctest", + "ctput", + "CVAT", + "cython", + "dadb", + "DANDROID", + "DARM", + "Datumaro", + "datumaro", + "DBUILD", + "DCMAKE", + "ddepth", + "Depthwise", + "dearmor", + "devicesupport", + "dequantization", + "Dequantization", + "deeplabv", + "deeced", + "DENABLE", + "delif", + "denormal", + "DENORMAL", + "denormalized", + "Detectron", + "Dequantize", "devel", "devtoolset", "dgpu", "diffstat", + "dldt", + "dlstreamer", + "dkms", "Dockerfiles", + "DOPENVINO", + "downscript", + "doxid", + "doxygen", + "Doxygen", + "doxygensnippet", + "DTHREADING", "dpkg", + "DPYTHON", + "DSELECTIVE", + "dylib", "DWORD", + "efficientdet", + "Efficientdet", + "Einsum", + "Elems", + "Elementwise", + "elementwise", + "Eltwise", "endsphinxdirective", + "enumov", + "emcmake", + "emmake", + "emod", + "emom", + "emow", + "Emscripten", + "emscripten", + "emsdk", "epel", "ERRORLEVEL", + "evolutionally", "executionpolicy", "fafe", + "fdupes", + "flatbuffers", + "FLATBUFFERS", + "frontends", + "Frontends", + "FYXB", + "gaddb", + "GAPI", + "gapi", + "Gaussed", + "gcompoundkernel", + "gcomputation", + "GCPU", + "gcpukernel", + "Gelu", + "GELU", + "Geti", + "getitem", + "gimg", + "gitee", + "gflags", "globbing", "gmmlib", "GNAs", + "gmock", + "gnueabihf", "googlenet", "gpgcheck", "gpgkey", + "graphviz", + "Graphviz", + "groupov", + "gtest", + "hardtanh", "hashfile", + "HDDL", "HKLM", "HOSTTOOLS", + "Hotspots", + "hotspots", + "hostnet", + "hwloc", + "hwquote", + "idbf", + "IDFT", "iigd", + "ifdef", + "ifdown", + "ifup", + "imgproc", + "imshow", + "inet", + "INTEGRITYCHECK", + "ILSVRC", + "inferenced", + "Informations", "insmod", "intelocl", + "INTERPROCEDURAL", + "INSTALLDIR", + "IRDFT", + "jemalloc", "kaldi", + "Keras", + "keypress", + "keyrings", "Khronos", + "KROIs", + "Landm", + "landm", + "Latency", + "Lcov", "ldconfig", + "libc", "libopencl", + "libopencv", "libpython", + "libtbb", + "libtbbbind", + "libtpm", + "libvirtd", "linmac", + "Liskov", + "lowlatency", "LTSC", + "LSTM", + "makefiles", + "malloc", + "memleaks", + "manylinux", "maxdepth", + "miktext", + "Mish", "mklink", + "mmap", + "mobilenet", + "Mobilenet", "monodepth", + "mozallowfullscreen", + "msallowfullscreen", + "MSVC", + "msvc", + "Multiclass", + "muxed", "mxnet", + "namespaceov", + "NCHW", + "ncpu", + "netdev", + "netplan", + "ngraph", + "nireq", + "NNCF", + "nncf", "nocache", "noglob", "nohup", + "nlohmann", "norestart", + "noqueue", + "nproc", + "NUMA", + "numpy", + "Numpy", + "oallowfullscreen", "ocloc", + "OCSP", + "oneapi", + "onetbb", "onnx", "opencl", "openembedded", "openvino", + "Opset", + "opset", + "opsets", + "OVMS", + "ovms", + "ovsa", + "OVSA", + "ovsatool", + "OVTF", "PACKAGECONFIG", + "paddlepaddle", + "parameterizable", + "partitioner", "patchelf", + "passpattern", + "Pexels", + "pdmodel", + "PDPD", "pkgdata", "pkgs", + "pkill", + "polylines", + "postproc", + "postprocess", + "preprocess", + "Preprocess", + "protobuf", + "Protobuf", + "PROTOBUF", + "prototxt", + "PSROI", + "Pugi", + "pugixml", + "PUGIXML", "pypi", + "PYTHONPATH", "pzstd", + "qcow", + "qlen", + "QSPECTRE", + "Qspectre", "quantizer", + "Rects", + "Relu", + "relu", + "rcnn", + "RCNN", + "RDFT", "Redistributable", "remotesigned", "repolist", + "reproject", + "reshapable", + "Requantize", + "retval", + "RHODS", "rmmod", + "runtool", + "scons", + "SCONS", + "segm", + "Selu", "servercore", + "setuptools", "setupvars", "SETX", + "SIMD", + "Softmax", "skylake", "sphinxdirective", + "Strided", + "squeezenet", + "SWTPM", + "swtpm", + "TBBBIND", + "TBBROOT", + "Tensro", + "texlive", + "textrm", + "tflite", + "thirdparty", + "Thresholded", "toctree", + "toolset", + "Torchvision", + "tpmrm", + "tpmstate", + "tput", + "Tunables", + "unet", "Uninstallation", + "unixio", + "unsharp", + "Unsharp", + "Unsh", + "Unsqueeze", + "Usecase", + "usecases", + "USERPROFILE", "userspace", + "VAAPI", + "valgrind", + "vcpkg", + "vcvars", "venv", + "virbr", + "virsh", + "virt", + "virtio", + "VMHWM", + "VMRSS", + "VNNI", + "vtune", + "vtunesummary", + "vtunebottonup", + "WHOLEARCHIVE", "WDDM", "WORKDIR", + "WORKSIZE", + "xbyak", + "Xbyak", + "xdot", + "xvfz", "yocto", + "yolo", + "YOLO", + "yolov", + "Yolov", + "YXFB", "zstd" ], "ignoreWords": [], diff --git a/docs/Documentation/model_introduction.md b/docs/Documentation/model_introduction.md index ad38d118a01922..cadd407ba0b6a6 100644 --- a/docs/Documentation/model_introduction.md +++ b/docs/Documentation/model_introduction.md @@ -15,18 +15,37 @@ openvino_docs_OV_Converter_UG_prepare_model_convert_model_Converting_Model -Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as `TensorFlow Hub `__, `Hugging Face `__, or `Torchvision models `__. - -OpenVINO™ :doc:`supports several model formats ` and can convert them into its own representation, `openvino.Model `__ (`ov.Model `__), providing a conversion API. Converted models can be used for inference with one or multiple OpenVINO Hardware plugins. There are two ways to use the conversion API: using a Python script or calling the ``ovc`` command line tool. +Every deep learning workflow begins with obtaining a model. You can choose to prepare +a custom one, use a ready-made solution and adjust it to your needs, or even download +and run a pre-trained network from an online database, such as +`TensorFlow Hub `__, `Hugging Face `__, +or `Torchvision models `__. + +If your selected model is in one of the :doc:`OpenVINO™ supported model formats `, +you can use it directly, without the need to save as the OpenVINO IR. +(`openvino.Model `__ - +`ov.Model `__). +For this purpose, you can use ``openvino.Core.read_model`` and ``openvino.Core.compile_model`` +methods, so that conversion is performed automatically before inference, for +maximum convenience (note that working with PyTorch differs slightly, the Python API +being the only option, while TensorFlow may present additional considerations +:doc:`TensorFlow Frontend Capabilities and Limitations `). + + +For better performance and more optimization options, OpenVINO offers a conversion +API with two possible approaches: the Python API functions (``openvino.convert_model`` +and ``openvino.save_model``) and the ``ovc`` command line tool, which are described in detail in this article. .. note:: - Prior to OpenVINO 2023.1, model conversion API was exposed as the ``openvino.tools.mo.convert_model`` - function and the ``mo`` command line tool. Now, a new and simplified API is used: the - ``openvino.convert_model`` function and the ``ovc`` command line tool. + Model conversion API prior to OpenVINO 2023.1 is considered deprecated. + Both existing and new projects are recommended to transition to the new + solutions, keeping in mind that they are not fully backwards compatible + with ``openvino.tools.mo.convert_model`` or the ``mo`` CLI tool. + For more details, see the :doc:`Model Conversion API Transition Guide `. + + - All new projects are recommended to use the new tools, keeping in mind that they are not fully - backwards compatible. For more details, consult the :doc:`Model Conversion API Transition Guide `. Convert a Model in Python: ``convert_model`` ############################################## @@ -202,19 +221,15 @@ The figure below illustrates the typical workflow for deploying a trained deep-l Convert a Model in CLI: ``ovc`` ############################### -Another option for model conversion is to use ``ovc`` command-line tool, which stands for OpenVINO Model Converter. The tool combines both ``openvino.convert_model`` and ``openvino.save_model`` functionalities. It is convenient to use when the original model is ready for inference and is in one of the supported file formats: ONNX, TensorFlow, TensorFlow Lite, or PaddlePaddle. As a result, ``ovc`` produces an OpenVINO IR, consisting of ``.xml`` and ``.bin`` files, which needs to be read with the ``ov.read_model()`` method. You can compile and infer the ``ov.Model`` later with :doc:`OpenVINO™ Runtime ` +Another option for model conversion is to use ``ovc`` command-line tool, which stands for OpenVINO Model Converter. The tool combines both ``openvino.convert_model`` and ``openvino.save_model`` functionalities. It is convenient to use when the original model is ready for inference and is in one of the supported file formats: ONNX, TensorFlow, TensorFlow Lite, or PaddlePaddle. As a result, ``ovc`` produces an OpenVINO IR, consisting of ``.xml`` and ``.bin`` files, which needs to be read with the ``openvino.Core.read_model`` method. You can compile and infer the ``ov.Model`` later with :doc:`OpenVINO™ Runtime ` .. note:: PyTorch models cannot be converted with ``ovc``, use ``openvino.convert_model`` instead. The results of both ``ovc`` and ``openvino.convert_model``/``openvino.save_model`` conversion methods are the same. You can choose either of them based on your convenience. Note that there should not be any differences in the results of model conversion if the same set of parameters is used and the model is saved into OpenVINO IR. -Cases when Model Preparation is not Required -############################################ -If a model is represented as a single file from ONNX, PaddlePaddle, TensorFlow and TensorFlow Lite (check :doc:`TensorFlow Frontend Capabilities and Limitations `), it does not require a separate conversion and IR-saving step, that is ``openvino.convert_model`` and ``openvino.save_model``, or ``ovc``. -OpenVINO provides C++ and Python APIs for reading such models by just calling the ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods. These methods perform conversion of the model from the original representation. While this conversion may take extra time compared to using prepared OpenVINO IR, it is convenient when you need to read a model in the original format in C++, since ``openvino.convert_model`` is only available in Python. However, for efficient model deployment with the OpenVINO Runtime, it is still recommended to prepare OpenVINO IR and then use it in your inference application. Additional Resources #################### diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md index 055e94049a78ed..0cafd3066535ab 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md @@ -58,7 +58,7 @@ parameter to be set, for example: Sometimes ``convert_model`` will produce inputs of the model with dynamic rank or dynamic type. Such model may not be supported by the hardware chosen for inference. To avoid this issue, -use the ``input`` argument of ``convert_model``. For more information, refer to `Convert Models Represented as Python Objects `. +use the ``input`` argument of ``convert_model``. For more information, refer to :doc:`Convert Models Represented as Python Objects `. .. important:: diff --git a/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_Paddle.md b/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_Paddle.md index dd3f821229bf99..8b4c549547ebbd 100644 --- a/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_Paddle.md +++ b/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_Paddle.md @@ -89,7 +89,7 @@ Some PaddlePaddle models may require setting ``example_input`` or ``output`` for * Example of converting ``paddle.fluid.dygraph.layers.Layer`` format model: - ``example_input`` is required while ``output`` is optional, which accept the following formats: + ``example_input`` is required while ``output`` is optional. ``example_input`` accepts the following formats: ``list`` with tensor (``paddle.Tensor``) or InputSpec (``paddle.static.input.InputSpec``) diff --git a/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_PyTorch.md b/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_PyTorch.md index 83005b7e978e8c..6fcd6d7c03aaa8 100644 --- a/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_PyTorch.md +++ b/docs/OV_Converter_UG/prepare_model/convert_model/Convert_Model_From_PyTorch.md @@ -40,8 +40,8 @@ The value for the ``example_input`` parameter can be easily derived from knowing import torch import openvino as ov - model = torchvision.models.resnet50(pretrained=True) - ov_model = ov.convert_model(model, example_input=example_input=torch.rand(1, 3, 224, 224)) + model = torchvision.models.resnet50(weights='DEFAULT') + ov_model = ov.convert_model(model, example_input=torch.rand(1, 3, 224, 224)) In practice, the code to evaluate or test the PyTorch model is usually provided with the model itself and can be used to generate a proper ``example_input`` value. A modified example of using ``resnet50`` model from ``torchvision`` is presented below. It demonstrates how to switch inference in the existing PyTorch application to OpenVINO and how to get value for ``example_input``: diff --git a/docs/dev/build_windows.md b/docs/dev/build_windows.md index e63d4830904086..e598fdd33f04e7 100644 --- a/docs/dev/build_windows.md +++ b/docs/dev/build_windows.md @@ -25,29 +25,17 @@ Supported configurations: ```sh git clone https://github.com/openvinotoolkit/openvino.git cd openvino - git submodule update --init --recursive - ``` - (Extra for WoA) To build on Windows on ARM with ARM plugin: - ```sh - git clone https://github.com/openvinotoolkit/openvino_contrib.git - cd openvino_contrib - git submodule update --init --recursive + git submodule update --init ``` 2. Create build directory: ```sh mkdir build && cd build ``` -3. In the `build` directory, run `cmake` to fetch project dependencies and generate a Visual Studio solution. +3. In the `build` directory, run `cmake` to fetch project dependencies and generate a Visual Studio solution: - On Windows x86 64-bits: - ```sh - cmake -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Release - ``` - - On Windows on ARM for ARM64 architecture: ```sh - cmake -G "Visual Studio 16 2019" -DOPENVINO_EXTRA_MODULES=/modules/arm_plugin -DCMAKE_BUILD_TYPE=Release + cmake -G "Visual Studio 17 2022" ``` > **HINT**: **Generating PDB Files and Debugging Your Build**
@@ -62,16 +50,8 @@ Supported configurations: ### Additional Build Options -- Internal JIT GEMM implementation is used by default. - -- Threading Building Blocks (TBB) is used by default. To build Inference Engine with OpenMP threading, set the `-DTHREADING=OMP` option. - -- Required versions of TBB and OpenCV packages are downloaded automatically by the CMake-based script. If you want to use the automatically-downloaded packages but you have already installed TBB or OpenCV packages configured in your environment, you may need to clean the `TBBROOT` and `OpenCV_DIR` environment variables before running the `cmake` command; otherwise they won'tnbe downloaded and the build may fail if incompatible versions were installed. - -- If the CMake-based build script can not find and download the OpenCV package that is supported on your platform, or if you want to use a custom build of the OpenCV library, refer to the [Use Custom OpenCV Builds](./cmake_options_for_custom_compilation.md#Building-with-custom-OpenCV) section for details. - - To build the OpenVINO Runtime Python API: - 1. First, install all additional packages (e.g., cython and opencv) listed in the file: + 1. First, install all additional packages (e.g., cython) listed in the file: ```sh pip install -r \src\bindings\python\src\compatibility\openvino\requirements-dev.txt ``` @@ -95,15 +75,12 @@ Supported configurations: pip install build/wheel/openvino-2023.0.0-9612-cp11-cp11-win_arm64.whl ``` -- OpenVINO runtime compilation options: - `-DENABLE_OV_ONNX_FRONTEND=ON` enables the building of the ONNX importer. - ### Building OpenVINO with Ninja* Build System ```sh call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Auxiliary\Build\vcvars64.bat" cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Release .. -cmake --build . --config Release +ninja . ``` ## See also diff --git a/docs/dev/cmake_options_for_custom_compilation.md b/docs/dev/cmake_options_for_custom_compilation.md index 1b4f3b7eb5752b..847997e067d5c3 100644 --- a/docs/dev/cmake_options_for_custom_compilation.md +++ b/docs/dev/cmake_options_for_custom_compilation.md @@ -114,6 +114,9 @@ This document provides description and default values for CMake options that can * `OFF` is default, because it increases binary size. * `SELECTIVE_BUILD` enables [[Conditional compilation|ConditionalCompilation]] feature. * `OFF` is default. +* `ENABLE_MLAS_FOR_CPU` enables MLAS library for CPU plugin + * `ON` is default for x86_64 and AARCH64 platforms + * Affects only OpenVINO CPU plugin ## Building with OpenCV diff --git a/docs/dev/static_libaries.md b/docs/dev/static_libaries.md index e001b6f24c5fc1..e538a37555f558 100644 --- a/docs/dev/static_libaries.md +++ b/docs/dev/static_libaries.md @@ -135,7 +135,7 @@ cmake -DCMAKE_TOOLCHAIN_FILE=/cmake/toolchains/mt.runtime.w * The enabled and tested capabilities of OpenVINO Runtime in a static build: * OpenVINO common runtime - work with `ov::Model`, perform model loading on particular device - * CPU and GNA inference plugins (**GPU and MYRIAD are not enabled**) + * CPU and GNA inference plugins (**GPU is not enabled**) * MULTI, HETERO, AUTO, and BATCH inference modes * IR, ONNX, PDPD, and TF frontends to read `ov::Model` * Static build support for building static libraries only for OpenVINO Runtime libraries. All other third-party prebuilt dependencies remain in the same format: diff --git a/docs/documentation.md b/docs/documentation.md index a25e784165b78f..276e4e6e0930a6 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -12,9 +12,9 @@ :hidden: API Reference - OpenVINO IR format and Operation Sets + OpenVINO IR format and Operation Sets + Legacy Features Tool Ecosystem - Legacy Features OpenVINO Extensibility Media Processing and CV Libraries OpenVINO™ Security diff --git a/docs/home.rst b/docs/home.rst index d8f359e65aaa5a..4ed32d3aea261b 100644 --- a/docs/home.rst +++ b/docs/home.rst @@ -24,10 +24,10 @@ OpenVINO 2023.0
  • An open-source toolkit for optimizing and deploying deep learning models.
    Boost your AI deep-learning inference performance!
    • -
  • Use PyTorch models directly, without converting them first. +
  • Use PyTorch models directly, without converting them first.
    Learn more...
    • -
  • OpenVINO via PyTorch 2.0 torch.compile()
    Use OpenVINO directly in PyTorch-native applications! +
  • OpenVINO via PyTorch 2.0 torch.compile()
    Use OpenVINO directly in PyTorch-native applications!
    Learn more...
  • Do you like Generative AI? You will love how it performs with OpenVINO!
    diff --git a/docs/install_guides/installing-openvino-linux-header.md b/docs/install_guides/installing-openvino-linux-header.md index f0bb87d87f0ade..a45b11d20e2f5e 100644 --- a/docs/install_guides/installing-openvino-linux-header.md +++ b/docs/install_guides/installing-openvino-linux-header.md @@ -22,14 +22,15 @@ Use Docker -If you want to install OpenVINO™ Runtime on your Linux machine, these are your options: +If you want to install OpenVINO™ Runtime on Linux, you have the following options: -* :doc:`Install OpenVINO Runtime using an Archive File ` +* :doc:`Install OpenVINO using an Archive File ` * :doc:`Install OpenVINO using PyPI ` -* :doc:`Install OpenVINO Runtime using APT ` -* :doc:`Install OpenVINO Runtime using YUM ` -* :doc:`Install OpenVINO Runtime using Conda Forge ` -* :doc:`Install OpenVINO Runtime using Homebrew ` +* :doc:`Install OpenVINO using APT ` +* :doc:`Install OpenVINO using YUM ` +* :doc:`Install OpenVINO using Conda Forge ` +* :doc:`Install OpenVINO using vcpkg ` +* :doc:`Install OpenVINO using Homebrew ` * :doc:`Install OpenVINO using Docker ` diff --git a/docs/install_guides/installing-openvino-macos-header.md b/docs/install_guides/installing-openvino-macos-header.md index dff827ce9a89e8..2e0d70b61d04be 100644 --- a/docs/install_guides/installing-openvino-macos-header.md +++ b/docs/install_guides/installing-openvino-macos-header.md @@ -12,19 +12,21 @@ :maxdepth: 3 :hidden: - From Archive - Using Homebrew - From PyPI - Using Conda Forge + Use Archive + Use Homebrew + Use PyPI + Use Conda Forge Use vcpkg -If you want to install OpenVINO™ Runtime on macOS, there are a few ways to accomplish this. We prepared following options for you: +If you want to install OpenVINO™ Runtime on macOS, you have the following options: -* :doc:`Install OpenVINO Runtime from an Archive File ` -* :doc:`Install OpenVINO from PyPI ` -* :doc:`Install OpenVINO Runtime using Conda Forge ` -* :doc:`Install OpenVINO Runtime via Homebrew ` + +* :doc:`Install OpenVINO using an Archive File ` +* :doc:`Install OpenVINO using PyPI ` +* :doc:`Install OpenVINO using Conda Forge ` +* :doc:`Install OpenVINO using Homebrew ` +* :doc:`Install OpenVINO using vcpkg ` diff --git a/docs/install_guides/installing-openvino-windows-header.md b/docs/install_guides/installing-openvino-windows-header.md index 3044c2accef729..65b1803ec711ff 100644 --- a/docs/install_guides/installing-openvino-windows-header.md +++ b/docs/install_guides/installing-openvino-windows-header.md @@ -22,9 +22,10 @@ If you want to install OpenVINO™ Runtime on Windows, you have the following options: -* :doc:`Install OpenVINO Runtime from an Archive File ` -* :doc:`Install OpenVINO Runtime using PyPI ` -* :doc:`Install OpenVINO Runtime using Conda Forge ` +* :doc:`Install OpenVINO using an Archive File ` +* :doc:`Install OpenVINO using PyPI ` +* :doc:`Install OpenVINO using Conda Forge ` +* :doc:`Install OpenVINO using vcpkg ` * :doc:`Install OpenVINO using Docker ` diff --git a/src/bindings/c/src/CMakeLists.txt b/src/bindings/c/src/CMakeLists.txt index 737dcc3d272a1e..e491424cb27afb 100644 --- a/src/bindings/c/src/CMakeLists.txt +++ b/src/bindings/c/src/CMakeLists.txt @@ -8,11 +8,12 @@ set(TARGET_NAME openvino_c) ov_deprecated_no_errors() add_definitions(-DIN_OV_COMPONENT) -file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) -file(GLOB_RECURSE HEADERS ${OpenVINO_C_API_SOURCE_DIR}/include/*.h) +file(GLOB SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +file(GLOB_RECURSE LEGACY_HEADERS ${OpenVINO_C_API_SOURCE_DIR}/include/c_api/*.h) +file(GLOB_RECURSE HEADERS ${OpenVINO_C_API_SOURCE_DIR}/include/openvino/*.h) # create library -add_library(${TARGET_NAME} ${HEADERS} ${SOURCES}) +add_library(${TARGET_NAME} ${LEGACY_HEADERS} ${HEADERS} ${SOURCES}) add_library(openvino::runtime::c ALIAS ${TARGET_NAME}) target_link_libraries(${TARGET_NAME} PRIVATE openvino openvino::util) @@ -24,7 +25,7 @@ if(NOT BUILD_SHARED_LIBS) target_compile_definitions(${TARGET_NAME} PUBLIC OPENVINO_STATIC_LIBRARY) endif() -ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) +ov_add_clang_format_target(${TARGET_NAME}_clang FOR_SOURCES ${HEADERS} ${SOURCES}) set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) diff --git a/src/bindings/c/src/common.h b/src/bindings/c/src/common.h index 92e024a7123d65..ff2e0bcac07312 100644 --- a/src/bindings/c/src/common.h +++ b/src/bindings/c/src/common.h @@ -17,7 +17,7 @@ #define CATCH_IE_EXCEPTION(StatusCode, ExceptionType) \ catch (const InferenceEngine::ExceptionType&) { \ return ov_status_e::StatusCode; \ - } \ + } #define CATCH_OV_EXCEPTION(StatusCode, ExceptionType) \ catch (const ov::ExceptionType&) { \ @@ -42,7 +42,7 @@ CATCH_IE_EXCEPTION(INFER_CANCELLED, InferCancelled) \ catch (...) { \ return ov_status_e::UNKNOW_EXCEPTION; \ - } \ + } #define GET_PROPERTY_FROM_ARGS_LIST \ std::string property_key = va_arg(args_ptr, char*); \ diff --git a/src/bindings/python/CMakeLists.txt b/src/bindings/python/CMakeLists.txt index 4c4c9bce4d7804..b46eaaf9883e16 100644 --- a/src/bindings/python/CMakeLists.txt +++ b/src/bindings/python/CMakeLists.txt @@ -20,8 +20,7 @@ endif() # Check python requirements # -set(ov_python_req "${OpenVINOPython_SOURCE_DIR}/requirements.txt") -set(ie_python_req "${OpenVINOPython_SOURCE_DIR}/src/compatibility/openvino/requirements-dev.txt") +set(ie_build_python_req "${OpenVINOPython_SOURCE_DIR}/src/compatibility/openvino/requirements-dev.txt") function(ov_check_python_build_conditions) # user explicitly specified ENABLE_PYTHON=ON @@ -34,7 +33,7 @@ function(ov_check_python_build_conditions) endif() # Try to find python3 and its libs - find_host_package(PythonInterp 3 ${find_package_mode}) + find_package(PythonInterp 3 ${find_package_mode}) if(PYTHONINTERP_FOUND) if(PYTHON_VERSION_MINOR GREATER_EQUAL 11) set(pybind11_min_version 2.9.2) @@ -57,14 +56,14 @@ function(ov_check_python_build_conditions) if(EXISTS ${pybind11_tools_dir}) list(APPEND CMAKE_MODULE_PATH ${pybind11_tools_dir}) else() - find_host_package(pybind11 ${pybind11_min_version} QUIET) + find_package(pybind11 ${pybind11_min_version} QUIET) if(pybind11_FOUND) list(APPEND CMAKE_MODULE_PATH "${pybind11_DIR}") endif() endif() # use libraries with the same version as python itself set(PYBIND11_PYTHON_VERSION ${PYTHON_VERSION_STRING}) - find_host_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} EXACT ${find_package_mode}) + find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} EXACT ${find_package_mode}) set(PYTHONLIBSNEW_FOUND ${PYTHONLIBS_FOUND} PARENT_SCOPE) endfunction() # try to find python libraries @@ -72,7 +71,7 @@ function(ov_check_python_build_conditions) if(PYTHONLIBSNEW_FOUND) # clear Python_ADDITIONAL_VERSIONS to find only python library matching PYTHON_EXECUTABLE unset(Python_ADDITIONAL_VERSIONS CACHE) - find_host_package(PythonLibs ${PYTHON_VERSION_STRING} EXACT ${find_package_mode}) + find_package(PythonLibs ${PYTHON_VERSION_STRING} EXACT ${find_package_mode}) endif() if(NOT PYTHONLIBS_FOUND) message(${message_mode} "Python development libraries are not found. OpenVINO Python API will be turned off (ENABLE_PYTHON is OFF)") @@ -81,31 +80,23 @@ function(ov_check_python_build_conditions) message(${message_mode} "Python 3.x interpreter is not found. OpenVINO Python API will be turned off (ENABLE_PYTHON is OFF)") endif() - # check pyopenvino requirements to OV 2.0 API - ov_check_pip_packages(REQUIREMENTS_FILE ${ov_python_req} - RESULT_VAR ov_python_req_FOUND - WARNING_MESSAGE "install python3 -m pip install -r ${ov_python_req} for OV API 2.0 requirements" - MESSAGE_MODE TRACE) - # ov_python_req are not mandatory for build - set(ov_python_req_FOUND ON) - # check for Cython requirement for build IE API 1.0 - ov_check_pip_packages(REQUIREMENTS_FILE ${ie_python_req} - RESULT_VAR ie_python_req_FOUND - WARNING_MESSAGE "install python3 -m pip install -r ${ie_python_req} for IE API 1.0 requirements" + ov_check_pip_packages(REQUIREMENTS_FILE ${ie_build_python_req} + RESULT_VAR ie_build_python_req_FOUND + WARNING_MESSAGE "install python3 -m pip install -r ${ie_build_python_req} for IE API 1.0 requirements" MESSAGE_MODE TRACE) # cython can be installed as a debian package, so pip requirements can be unsatisfied # so, let's check to find cython anyway - if(NOT ie_python_req_FOUND) + if(NOT ie_build_python_req_FOUND) find_package(Cython QUIET PATHS "${OpenVINOPython_SOURCE_DIR}/src/compatibility/openvino/cmake" NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH) if(CYTHON_VERSION VERSION_GREATER_EQUAL 0.29) - set(ie_python_req_FOUND ON) + set(ie_build_python_req_FOUND ON) else() - message(${message_mode} "Python requirements '${ie_python_req}' are missed, IE Python API 1.0 will not be built (ENABLE_PYTHON is OFF)") + message(${message_mode} "Python requirements '${ie_build_python_req}' are missed, IE Python API 1.0 will not be built (ENABLE_PYTHON is OFF)") endif() endif() @@ -116,20 +107,51 @@ function(ov_check_python_build_conditions) set(python_debug OFF) endif() - if(PYTHONLIBS_FOUND AND ov_python_req_FOUND AND ie_python_req_FOUND AND NOT python_debug) + if(PYTHONLIBS_FOUND AND ie_build_python_req_FOUND AND NOT python_debug) set(ENABLE_PYTHON_DEFAULT ON PARENT_SCOPE) else() set(ENABLE_PYTHON_DEFAULT OFF PARENT_SCOPE) endif() # to disable API 1.0 - set(ie_python_req_FOUND ${ie_python_req_FOUND} PARENT_SCOPE) + set(ie_build_python_req_FOUND ${ie_build_python_req_FOUND} PARENT_SCOPE) # set pybind11 minimal version set(pybind11_min_version ${pybind11_min_version} PARENT_SCOPE) endfunction() ov_check_python_build_conditions() +# check __init__.py files alignment + +function(ov_check_init_files_alignment) + # check the files in pairs + list(APPEND init_files + "${OpenVINOPython_SOURCE_DIR}/src/openvino/__init__.py" + "${OpenVINOPython_SOURCE_DIR}/src/compatibility/openvino/__init__.py" + "${OpenVINO_SOURCE_DIR}/tools/mo/openvino/__init__.py" + "${OpenVINO_SOURCE_DIR}/tools/pot/openvino/__init__.py" + "${OpenVINO_SOURCE_DIR}/tools/ovc/openvino/__init__.py" + "${OpenVINO_SOURCE_DIR}/tools/benchmark_tool/openvino/__init__.py" + "${OpenVINO_SOURCE_DIR}/tools/openvino_dev/src/openvino/__init__.py") + + list(LENGTH init_files init_files_count) + math(EXPR file_loop_range "${init_files_count}-2") + foreach(init_file_idx RANGE 0 ${file_loop_range}) + math(EXPR init_file_idx_next "${init_file_idx}+1") + list(GET init_files ${init_file_idx} file1) + list(GET init_files ${init_file_idx_next} file2) + + execute_process(COMMAND ${CMAKE_COMMAND} -E compare_files ${file1} ${file2} + RESULT_VARIABLE compare_result + ) + if(compare_result EQUAL 1) + message(FATAL_ERROR "The __init__.py files are misaligned: ${file1} and ${file2}") + endif() + endforeach() +endfunction() + +ov_check_init_files_alignment() + ie_option(ENABLE_PYTHON "Enables OpenVINO Python API build" ${ENABLE_PYTHON_DEFAULT}) # @@ -178,7 +200,7 @@ ie_dependent_option(ENABLE_WHEEL "Build wheel packages for PyPI" ${ENABLE_WHEEL_ if(NOT ENABLE_PYTHON) if(CMAKE_SOURCE_DIR STREQUAL OpenVINOPython_SOURCE_DIR) - message(FATAL_ERROR "Python OpenVINO API requirements are not satisfied. Please, install ${ie_python_req} and ${ov_python_req}") + message(FATAL_ERROR "Python OpenVINO API requirements are not satisfied. Please, install ${ie_build_python_req}") else() return() endif() @@ -201,7 +223,7 @@ endif() add_subdirectory(src/compatibility/pyngraph) add_subdirectory(src/pyopenvino) -if(ie_python_req_FOUND) +if(ie_build_python_req_FOUND) add_subdirectory(src/compatibility/openvino) else() message(WARNING "NOTE: Python API for OpenVINO 1.0 is disabled") diff --git a/src/bindings/python/constraints.txt b/src/bindings/python/constraints.txt index 30be494ea327bc..1762fa681551d3 100644 --- a/src/bindings/python/constraints.txt +++ b/src/bindings/python/constraints.txt @@ -1,5 +1,5 @@ # used in multiple components -numpy>=1.16.6,<1.26 # Python bindings, frontends +numpy>=1.16.6,<1.27 # Python bindings, frontends # pytest pytest>=5.0,<7.5 diff --git a/src/bindings/python/src/compatibility/openvino/__init__.py b/src/bindings/python/src/compatibility/openvino/__init__.py index 8f0113d5bcaf6c..90552e0befed68 100644 --- a/src/bindings/python/src/compatibility/openvino/__init__.py +++ b/src/bindings/python/src/compatibility/openvino/__init__.py @@ -57,6 +57,6 @@ # Tools try: # Model Conversion API - ovc should reside in the main namespace - from openvino.tools.ovc import convert_model, InputCutInfo + from openvino.tools.ovc import convert_model except ImportError: pass diff --git a/src/bindings/python/src/openvino/frontend/tensorflow/graph_iterator.py b/src/bindings/python/src/openvino/frontend/tensorflow/graph_iterator.py index ac7bb04958a3e4..29dc6b1ad58973 100644 --- a/src/bindings/python/src/openvino/frontend/tensorflow/graph_iterator.py +++ b/src/bindings/python/src/openvino/frontend/tensorflow/graph_iterator.py @@ -40,6 +40,10 @@ def get_input_names(self) -> list: return [] inp_ops = filter(lambda op: op.type == "Placeholder", self.m_graph.get_operations()) inp_names = [] + if hasattr(self.m_graph, 'inputs') and self.m_graph.inputs: + for inp in self.m_graph.inputs: + inp_names.append(inp.op.name) + return inp_names for inp in inp_ops: assert isinstance(inp, tf.Operation), "Unknown node type. Expected tf.Operation, got {}".format(type(inp)) assert hasattr(inp, "node_def") and isinstance(inp.node_def, tf.compat.v1.NodeDef), \ @@ -58,11 +62,13 @@ def get_output_names(self) -> list: # Note: used only for the library functions if not self.m_inner_graph: return [] - # tf.Graph has ordered outputs which are stored in 'outputs' field, - # but using this field results in mismatch of outputs in inner graph and outputs in outer graph - # during the injection of subgraph. - # For this reason only nodes without outputs are considered graph outputs here - # as this approach does not lead to conflicts. + + if hasattr(self.m_graph, 'outputs') and self.m_graph.outputs: + outputs = [] + for out in self.m_graph.outputs: + outputs.append(out.name) + return outputs + # If graph has no 'outputs' field, find nodes without outputs and consider them graph outputs. # The order of outputs is important and wrong order may lead to conversion error. non_outputs = set() for op in self.m_graph.get_operations(): diff --git a/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py b/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py index 0ecffc4040abe6..4fab05c17de380 100644 --- a/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py +++ b/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py @@ -54,6 +54,7 @@ def __init__(self, operation: tf.Operation, share_weights: bool, inner_graph: bo self.m_operation = operation self.m_inner_graph = inner_graph self.m_data_type = None + self.m_parsed_content = None # Copies value from inner buffer of TF_Operation to NodeDef class. self.m_node_def = self.m_operation.node_def @@ -87,11 +88,11 @@ def __init__(self, operation: tf.Operation, share_weights: bool, inner_graph: bo if self.m_operation.type == "Placeholder": self.m_data_type = tf.dtypes.DType(self.m_node_def.attr["dtype"].type).name - if self.m_data_type == "resource" and not self.m_inner_graph: + if not self.m_inner_graph: variable_value = TFGraphNodeDecoder.get_variable(self.m_operation) if variable_value is not None: # does not copy data - self.m_parsed_content = variable_value.value().__array__() + self.m_parsed_content = variable_value.__array__() if isinstance(self.m_parsed_content, bytes): self.m_data_type = "string" @@ -103,7 +104,7 @@ def get_op_name(self) -> str: def get_op_type(self) -> str: if self.m_operation.type == "Placeholder": type_attr = tf.dtypes.DType(self.m_node_def.attr["dtype"].type) - if type_attr.name == "resource" and not self.m_inner_graph: + if not self.m_inner_graph and self.m_parsed_content is not None: if TFGraphNodeDecoder.get_variable(self.m_operation) is not None: return "Const" raise Exception("Could not get variable for resource Placeholder {0}".format(self.m_operation.name)) @@ -116,10 +117,11 @@ def get_variable(operation): return None for var_tensor, op_tensor in tf_graph.captures: if operation.outputs[0].name == op_tensor.name: - resource_name = var_tensor._name + if var_tensor.dtype.name != 'resource': + return var_tensor for variable_value in operation.graph.variables: - if variable_value.name == resource_name: - return variable_value + if id(variable_value.handle) == id(var_tensor): + return variable_value.value() return None return None diff --git a/src/bindings/python/src/openvino/frontend/tensorflow/utils.py b/src/bindings/python/src/openvino/frontend/tensorflow/utils.py index 056e0904b4c22f..f4ac5b0a121565 100644 --- a/src/bindings/python/src/openvino/frontend/tensorflow/utils.py +++ b/src/bindings/python/src/openvino/frontend/tensorflow/utils.py @@ -339,7 +339,7 @@ def create_tf_graph_iterator(input_model, placeholder_shapes, placeholder_data_t if hasattr(input_model, 'outputs') and hasattr(input_model, 'structured_outputs') and \ isinstance(input_model.structured_outputs, dict): external_names = sorted(list(input_model.structured_outputs.keys())) - internal_names = sorted([tensor.name for tensor in input_model.outputs]) + internal_names = [tensor.name for tensor in input_model.outputs] if len(external_names) == len(internal_names): for external_name, internal_name in zip(external_names, internal_names): output_names_map = output_names_map or {} diff --git a/src/bindings/python/src/pyopenvino/graph/ops/if.cpp b/src/bindings/python/src/pyopenvino/graph/ops/if.cpp index 13de82c7acd3aa..d1eb84eb014d46 100644 --- a/src/bindings/python/src/pyopenvino/graph/ops/if.cpp +++ b/src/bindings/python/src/pyopenvino/graph/ops/if.cpp @@ -19,7 +19,17 @@ void regclass_graph_op_If(py::module m) { py::class_, ov::Node> cls(m, "if_op"); cls.doc() = "openvino.impl.op.If wraps ov::op::v0::If"; cls.def(py::init<>()); - cls.def(py::init&>(), py::arg("execution_condition")); + cls.def(py::init&>(), + py::arg("execution_condition"), + R"( + Constructs If with condition. + + :param execution_condition: condition node. + :type execution_condition: openvino.runtime.Output + + :rtype: openvino.impl.op.If + )"); + cls.def(py::init([](const std::shared_ptr& execution_condition) { if (MultiSubgraphHelpers::is_constant_or_parameter(execution_condition)) { return std::make_shared(execution_condition->output(0)); @@ -29,18 +39,114 @@ void regclass_graph_op_If(py::module m) { return std::make_shared(); } }), - py::arg("execution_condition")); - cls.def("get_else_body", &ov::op::v8::If::get_else_body); - cls.def("set_then_body", &ov::op::v8::If::set_then_body, py::arg("body")); - cls.def("set_else_body", &ov::op::v8::If::set_else_body, py::arg("body")); + py::arg("execution_condition"), + R"( + Constructs If with condition. + + :param execution_condition: condition node. + :type execution_condition: openvino.runtime.Node + + :rtype: openvino.impl.op.If + )"); + + cls.def("get_else_body", + &ov::op::v8::If::get_else_body, + R"( + Gets else_body as Model object. + + :return: else_body as Model object. + :rtype: openvino.Model + )"); + + cls.def("set_then_body", + &ov::op::v8::If::set_then_body, + py::arg("body"), + R"( + Sets new Model object as new then_body. + + :param body: new body for 'then' branch. + :type body: openvino.Model + + :rtype: None + )"); + + cls.def("set_else_body", + &ov::op::v8::If::set_else_body, + py::arg("body"), + R"( + Sets new Model object as new else_body. + + :param body: new body for 'else' branch. + :type body: openvino.Model + + :rtype: None + )"); + cls.def("set_input", &ov::op::v8::If::set_input, py::arg("value"), py::arg("then_parameter"), - py::arg("else_parameter")); - cls.def("set_output", &ov::op::v8::If::set_output, py::arg("then_result"), py::arg("else_result")); - cls.def("get_function", &ov::op::util::MultiSubGraphOp::get_function, py::arg("index")); - cls.def("set_function", &ov::op::util::MultiSubGraphOp::set_function, py::arg("index"), py::arg("func")); + py::arg("else_parameter"), + R"( + Sets new input to the operation associated with parameters of each sub-graphs. + + :param value: input to operation. + :type value: openvino.runtime.Output + + :param then_result: parameter for then_body or nullptr. + :type then_result: openvino.runtime.Node + + :param else_result: parameter for else_body or nullptr. + :type else_result: openvino.runtime.Node + + :rtype: None + )"); + + cls.def("set_output", + &ov::op::v8::If::set_output, + py::arg("then_result"), + py::arg("else_result"), + R"( + Sets new output from the operation associated with results of each sub-graphs. + + :param then_result: result from then_body. + :type then_result: openvino.runtime.Node + + :param else_result: result from else_body. + :type else_result: openvino.runtime.Node + + :return: output from operation. + :rtype: openvino.runtime.Output + )"); + + cls.def("get_function", + &ov::op::util::MultiSubGraphOp::get_function, + py::arg("index"), + R"( + Gets internal sub-graph by index in MultiSubGraphOp. + + :param index: sub-graph's index in op. + :type index: int + + :return: Model with sub-graph. + :rtype: openvino.Model + )"); + + cls.def("set_function", + &ov::op::util::MultiSubGraphOp::set_function, + py::arg("index"), + py::arg("func"), + R"( + Adds sub-graph to MultiSubGraphOp. + + :param index: index of new sub-graph. + :type index: int + + :param func: func new sub_graph as a Model. + :type func: openvino.Model + + :rtype: None + )"); cls.def( "set_input_descriptions", @@ -48,7 +154,20 @@ void regclass_graph_op_If(py::module m) { self->set_input_descriptions(index, MultiSubgraphHelpers::list_to_input_descriptor(inputs)); }, py::arg("index"), - py::arg("inputs")); + py::arg("inputs"), + R"( + Sets list with connections between operation inputs and internal sub-graph parameters. + + :param index: index of internal sub-graph. + :type index: int + + :param inputs: list of input descriptions. + :type inputs: list[Union[openvino.runtime.op.util.MergedInputDescription, + openvino.runtime.op.util.InvariantInputDescription, + openvino.runtime.op.util.SliceInputDescription]] + + :rtype: None + )"); cls.def( "set_output_descriptions", @@ -56,7 +175,19 @@ void regclass_graph_op_If(py::module m) { self->set_output_descriptions(index, MultiSubgraphHelpers::list_to_output_descriptor(outputs)); }, py::arg("index"), - py::arg("outputs")); + py::arg("outputs"), + R"( + Sets list with connections between operation outputs and internal sub-graph parameters. + + :param index: index of internal sub-graph. + :type index: int + + :param outputs: list of output descriptions. + :type outputs: list[Union[openvino.runtime.op.util.BodyOutputDescription, + openvino.runtime.op.util.ConcatOutputDescription]] + + :rtype: None + )"); cls.def( "get_output_descriptions", @@ -69,7 +200,17 @@ void regclass_graph_op_If(py::module m) { return result; }, - py::arg("index")); + py::arg("index"), + R"( + Gets list with connections between operation outputs and internal sub-graph parameters. + + :param index: index of internal sub-graph. + :type index: int + + :return: list of output descriptions. + :rtype: list[Union[openvino.runtime.op.util.BodyOutputDescription, + openvino.runtime.op.util.ConcatOutputDescription]] + )"); cls.def( "get_input_descriptions", @@ -82,7 +223,18 @@ void regclass_graph_op_If(py::module m) { return result; }, - py::arg("index")); + py::arg("index"), + R"( + Gets list with connections between operation inputs and internal sub-graph parameters. + + :param index: index of internal sub-graph. + :type index: int + + :return: list of input descriptions. + :rtype: list[Union[openvino.runtime.op.util.MergedInputDescription, + openvino.runtime.op.util.InvariantInputDescription, + openvino.runtime.op.util.SliceInputDescription]] + )"); cls.def("__repr__", [](const ov::op::v8::If& self) { std::stringstream shapes_ss; diff --git a/src/common/conditional_compilation/docs/develop_cc_for_new_component.md b/src/common/conditional_compilation/docs/develop_cc_for_new_component.md index 4166a907434b13..32c10478dc6310 100644 --- a/src/common/conditional_compilation/docs/develop_cc_for_new_component.md +++ b/src/common/conditional_compilation/docs/develop_cc_for_new_component.md @@ -69,7 +69,6 @@ It checks whether the code region in this module is active or inactive by the ma There is an example of `conditional_compilation_gen.h`: ``` -#define ov_pass_FixRtInfo_run_on_function 1 #define ov_pass_GraphRewrite_run_on_model 1 #define ov_pass_InitNodeInfo_run_on_function 1 #define ov_pass_ConstantFolding_run_on_model 1 diff --git a/src/common/snippets/src/pass/hash.cpp b/src/common/snippets/src/pass/hash.cpp index 8d509b568867fc..48dd9586ae4337 100644 --- a/src/common/snippets/src/pass/hash.cpp +++ b/src/common/snippets/src/pass/hash.cpp @@ -62,7 +62,7 @@ static uint64_t hash_combine(uint64_t seed, const T &v) { namespace rt_info { // some node attr is not type of ov::RuntimeAttribute, need dedicate visitor. -const std::vector list_of_names{ +static const std::vector list_of_names{ "PrimitivesPriority", "alt_width", }; diff --git a/src/common/transformations/include/transformations/common_optimizations/simplify_shape_of_sub_graph.hpp b/src/common/transformations/include/transformations/common_optimizations/simplify_shape_of_sub_graph.hpp index d4ff490bd2c388..7f2e533c5b79ed 100644 --- a/src/common/transformations/include/transformations/common_optimizations/simplify_shape_of_sub_graph.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/simplify_shape_of_sub_graph.hpp @@ -41,7 +41,11 @@ class ov::pass::GroupedGatherElimination : public ov::pass::MatcherPass { class ov::pass::SimplifyShapeOfSubGraph : public ov::pass::ModelPass { public: OPENVINO_RTTI("SimplifyShapeOfSubGraph", "0"); + explicit SimplifyShapeOfSubGraph(bool use_shapes = true) : m_use_shapes(use_shapes){}; bool run_on_model(const std::shared_ptr& m) override; + +private: + bool m_use_shapes; }; /** diff --git a/src/common/transformations/include/transformations/fix_rt_info.hpp b/src/common/transformations/include/transformations/fix_rt_info.hpp deleted file mode 100644 index a9c33645633074..00000000000000 --- a/src/common/transformations/include/transformations/fix_rt_info.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -/** - * @brief Defines initialize node runtime information pass - * @file init_node_info.hpp - */ - -#include -#include - -#include "openvino/pass/graph_rewrite.hpp" -#include "transformations_visibility.hpp" - -namespace ov { -namespace pass { - -class TRANSFORMATIONS_API FixRtInfo; - -} // namespace pass -} // namespace ov - -/** - * @ingroup ie_transformation_common_api - * @brief FixRtInfo transformation helps to fix info attributes in a single place. - * User can pass runtime attribute using various types. - * This Pass should generalize them runtime info representation. - * - * Used to extract runtime attributes from shared pointer to `ov::RuntimeAttributeWrapper` to standard or trivial types - */ -class ov::pass::FixRtInfo : public ov::pass::ModelPass { -public: - OPENVINO_RTTI("FixRtInfo", "0"); - bool run_on_model(const std::shared_ptr& m) override; -}; diff --git a/src/common/transformations/include/transformations/symbolic_transformations/chained_maximum.hpp b/src/common/transformations/include/transformations/symbolic_transformations/chained_maximum.hpp new file mode 100644 index 00000000000000..caa2586ebeeea4 --- /dev/null +++ b/src/common/transformations/include/transformations/symbolic_transformations/chained_maximum.hpp @@ -0,0 +1,26 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ov { +namespace pass { +class TRANSFORMATIONS_API ChainedMaximumOptimization; +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief Optimizes graphs based on value labels / symbols + * Maximum(Maximum(A, B), B) -> Maximum(A, B) + * Maximum(Maximum(A, B), A) -> Maximum(A, B) + */ +class ov::pass::ChainedMaximumOptimization : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("ChainedMaximumOptimization", "0"); + ChainedMaximumOptimization(); +}; \ No newline at end of file diff --git a/src/common/transformations/include/transformations/symbolic_transformations/label_optimization.hpp b/src/common/transformations/include/transformations/symbolic_transformations/label_optimization.hpp new file mode 100644 index 00000000000000..9260be86d2744a --- /dev/null +++ b/src/common/transformations/include/transformations/symbolic_transformations/label_optimization.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include +#include +#include + +namespace ov { +namespace pass { +class TRANSFORMATIONS_API ApplyTableOfEquivalence; +class TRANSFORMATIONS_API OptimizeLabelsUsedAsValues; +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief Resets symbols / labels on output shapes and values according to table of symbol / label equivalence. It + * allows to reduce number of labels used in the model and to disambiguate label values. + */ +class ov::pass::ApplyTableOfEquivalence : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("ApplyTableOfEquivalence", "0"); + bool run_on_model(const std::shared_ptr& m) override; +}; + +/** + * @ingroup ie_transformation_common_api + * @brief Collects sources where each symbol / label initially appeared (on shape or shape sub-graph) and attaches all + * value usages of this label to this initial source + */ +class ov::pass::OptimizeLabelsUsedAsValues : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("OptimizeLabelsUsedAsValues", "0"); + bool run_on_model(const std::shared_ptr& m) override; +}; \ No newline at end of file diff --git a/src/common/transformations/include/transformations/symbolic_transformations/nop_broadcast.hpp b/src/common/transformations/include/transformations/symbolic_transformations/nop_broadcast.hpp new file mode 100644 index 00000000000000..c6356277a42e81 --- /dev/null +++ b/src/common/transformations/include/transformations/symbolic_transformations/nop_broadcast.hpp @@ -0,0 +1,25 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ov { +namespace pass { +class TRANSFORMATIONS_API NopBroadcast; +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief Optimizes out Broadcast(data, Maximum(shape, ones)) if labels on data and shape are equal + * Use case with data being empty should not be considered here since original graph has Maximum with ones + */ +class ov::pass::NopBroadcast : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("NopBroadcast", "0"); + NopBroadcast(); +}; \ No newline at end of file diff --git a/src/common/transformations/include/transformations/symbolic_transformations/symbolic_optimizations.hpp b/src/common/transformations/include/transformations/symbolic_transformations/symbolic_optimizations.hpp new file mode 100644 index 00000000000000..1cf3cf9577dc78 --- /dev/null +++ b/src/common/transformations/include/transformations/symbolic_transformations/symbolic_optimizations.hpp @@ -0,0 +1,50 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include + +namespace ov { +namespace pass { +class TRANSFORMATIONS_API SymbolicOptimizations; +class TRANSFORMATIONS_API SymbolicPropagation; +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief Runs optimizations which are based on symbolic shape inference + */ +class ov::pass::SymbolicOptimizations : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("SymbolicOptimizations", "0"); + explicit SymbolicOptimizations(bool full_run = true); + bool run_on_model(const std::shared_ptr& m) override; + std::shared_ptr get_manager() { + return m_manager; + }; + +private: + std::shared_ptr m_manager; +}; + +/** + * @ingroup ie_transformation_common_api + * @brief Assigns labels / symbols to all tensors on shapes and values. Uses shape inference and other special rules to + * propagate labels / symbols + */ +class ov::pass::SymbolicPropagation : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("SymbolicPropagation"); + SymbolicPropagation(); + bool run_on_model(const std::shared_ptr& m) override; + +private: + std::shared_ptr m_te; +}; diff --git a/src/common/transformations/include/transformations/symbolic_transformations/utils.hpp b/src/common/transformations/include/transformations/symbolic_transformations/utils.hpp new file mode 100644 index 00000000000000..2f3d84dfe825ff --- /dev/null +++ b/src/common/transformations/include/transformations/symbolic_transformations/utils.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/core/descriptor/tensor.hpp" +#include "openvino/core/dimension.hpp" +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/type/element_type.hpp" + +namespace ov { +namespace symbol { +namespace util { + +/// \brief Collects labels from shape. Labels of static dimensions are guaranteed to be ov::no_labels +/// +/// \param shape Shape object to collect labels from +/// \param labels TensorLabel object to collect labels to +/// +/// \return Status of collecting the labels (false if rank is static else true) +TRANSFORMATIONS_API bool get_labels(const ov::PartialShape& shape, ov::TensorLabel& labels); + +/// \brief Collects labels from tensor of Output object +/// +/// \param output Output object to collect labels from +/// \param labels TensorLabel object to collect labels to +/// +/// \return Status of collecting the labels (false if tensor has no labels else true) +TRANSFORMATIONS_API bool get_labels(const ov::Output& output, ov::TensorLabel& labels); + +/// \brief Compares +/// +/// \param lhs TensorLabel object to compare +/// \param rhs TensorLabel object to compare +/// +/// \return true if labels are unique and equal between lhs and rhs else false +TRANSFORMATIONS_API bool are_unique_and_equal_labels(const ov::TensorLabel& lhs, const ov::TensorLabel& rhs); +} // namespace util +} // namespace symbol +} // namespace ov diff --git a/src/common/transformations/include/transformations/utils/utils.hpp b/src/common/transformations/include/transformations/utils/utils.hpp index 1af4dfbf1551e8..9a2036fff1b20d 100644 --- a/src/common/transformations/include/transformations/utils/utils.hpp +++ b/src/common/transformations/include/transformations/utils/utils.hpp @@ -214,11 +214,13 @@ TRANSFORMATIONS_API std::vector> get_node_target_inputs(const std::s TRANSFORMATIONS_API std::shared_ptr node_to_get_shape_value_of_indices_from_shape_node( const std::shared_ptr& shape_node, - const std::vector& indices); + const std::vector& indices, + const std::vector>& copy_rt_info_from = {}); TRANSFORMATIONS_API std::shared_ptr node_to_get_shape_value_of_indices_from_shape_source( const Output& shape_source, - const std::vector& indices); + const std::vector& indices, + const std::vector>& copy_rt_info_from = {}); TRANSFORMATIONS_API bool is_dequantization_subgraph(const Output& node); @@ -230,6 +232,28 @@ TRANSFORMATIONS_API bool is_constant_and_all_values_equal_int(const Output TRANSFORMATIONS_API bool is_on_constant_path(const ov::Output& output); +template +ov::pass::pattern::op::ValuePredicate constant_predicate(std::function&)> predicate) { + return pass::pattern::op::as_value_predicate([=](std::shared_ptr n) -> bool { + if (auto constant = as_type_ptr(n)) { + auto values = constant->cast_vector(); + return predicate(values); + } + return false; + }); +} } // namespace util } // namespace op } // namespace ov + +#define INT_CONSTANT_WITH_PREDICATE(expression) \ + pattern::wrap_type( \ + ov::op::util::constant_predicate([](const std::vector& value) -> bool { \ + return expression; \ + })) + +#define FLOAT_CONSTANT_WITH_PREDICATE(expression) \ + pattern::wrap_type( \ + ov::op::util::constant_predicate([](const std::vector& value) -> bool { \ + return expression; \ + })) diff --git a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp index 1638f2c4301e2b..8ad09f95dede3e 100644 --- a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -111,6 +111,7 @@ #include "transformations/op_conversions/softmax_decomposition.hpp" #include "transformations/op_conversions/softsign_decomposition.hpp" #include "transformations/op_conversions/unique_decomposition.hpp" +#include "transformations/symbolic_transformations/symbolic_optimizations.hpp" bool ov::pass::CommonOptimizations::run_on_model(const std::shared_ptr& f) { RUN_ON_FUNCTION_SCOPE(CommonOptimizations); @@ -230,7 +231,8 @@ bool ov::pass::CommonOptimizations::run_on_model(const std::shared_ptr(); + REGISTER_PASS(manager, StridesOptimization) + REGISTER_PASS(manager, SymbolicOptimizations) REGISTER_PASS(manager, Validate) manager.run_passes(f); diff --git a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp index 0bdae82810317a..068e1f27a291e9 100644 --- a/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/moc_transformations.cpp @@ -147,7 +147,7 @@ bool ov::pass::MOCTransformations::run_on_model(const std::shared_ptr REGISTER_PASS(manager, Validate) } REGISTER_PASS(manager, ConvertQuantizeDequantize) - REGISTER_PASS(manager, SimplifyShapeOfSubGraph) + REGISTER_PASS(manager, SimplifyShapeOfSubGraph, m_use_shapes) if (!m_use_shapes) { manager.register_pass(); diff --git a/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp b/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp index 9cbf24a6dae0e3..9ce5ea86b901ab 100644 --- a/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/nop_elimination.cpp @@ -833,7 +833,8 @@ ov::pass::NopSliceBeforeGatherElements::NopSliceBeforeGatherElements() { ov::pass::PrepareShapeOpsForEliminationAroundBE::PrepareShapeOpsForEliminationAroundBE() { MATCHER_SCOPE(PrepareShapeOpsForEliminationAroundBE); - auto first_label = pattern::wrap_type(pattern::rank_equals(0)); + auto first_label = pattern::wrap_type( + pattern::rank_equals(0)); auto other_input_label = pattern::any_input(pattern::rank_equals(0)); auto binary_op_label = pattern::wrap_type& f Manager manager; manager.set_per_pass_validation(false); + REGISTER_PASS(manager, PrepareShapeOpsForEliminationAroundBE) REGISTER_PASS(manager, SharedOpOptimization) REGISTER_PASS(manager, EliminateGatherUnsqueeze) // should run after SharedOpOptimization + REGISTER_PASS(manager, NopElimination, m_use_shapes) REGISTER_PASS(manager, GroupedGatherElimination) // GatherNopElimination depends on shape, so it requires shape propagation // if previous transformations has resolved some dynamic shapes. diff --git a/src/common/transformations/src/transformations/fix_rt_info.cpp b/src/common/transformations/src/transformations/fix_rt_info.cpp deleted file mode 100644 index b70176ca7aecd1..00000000000000 --- a/src/common/transformations/src/transformations/fix_rt_info.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "transformations/fix_rt_info.hpp" - -#include -#include - -#include "itt.hpp" -#include "openvino/core/rt_info.hpp" -#include "openvino/opsets/opset1.hpp" -#include "transformations/rt_info/primitives_priority_attribute.hpp" - -bool ov::pass::FixRtInfo::run_on_model(const std::shared_ptr& f) { - RUN_ON_FUNCTION_SCOPE(FixRtInfo); - - for (auto& node : f->get_ops()) { - // Recursively apply transformation for sub-graph based operations - if (auto sub_graph_node = std::dynamic_pointer_cast(node)) { - if (auto sub_graph = sub_graph_node->get_function()) { - run_on_model(sub_graph); - } - } - auto& rt_info = node->get_rt_info(); - { - auto it_info = rt_info.find("PrimitivesPriority"); - if (it_info != rt_info.end()) { - if (it_info->second.is()) { - rt_info.emplace(ov::PrimitivesPriority::get_type_info_static(), - it_info->second.as()); - } - if (it_info->second.is()) { - rt_info.emplace(ov::PrimitivesPriority::get_type_info_static(), - ov::PrimitivesPriority{it_info->second.as()}); - } - rt_info.erase(it_info); - } - } - } - return false; -} diff --git a/src/common/transformations/src/transformations/init_node_info.cpp b/src/common/transformations/src/transformations/init_node_info.cpp index 93c958d14f661b..efc2627199c723 100644 --- a/src/common/transformations/src/transformations/init_node_info.cpp +++ b/src/common/transformations/src/transformations/init_node_info.cpp @@ -8,9 +8,7 @@ #include #include "itt.hpp" -#include "openvino/core/rt_info.hpp" -#include "openvino/opsets/opset1.hpp" -#include "transformations/fix_rt_info.hpp" +#include "openvino/op/util/sub_graph_base.hpp" #include "transformations/rt_info/fused_names_attribute.hpp" #include "transformations/rt_info/primitives_priority_attribute.hpp" @@ -27,6 +25,5 @@ bool ov::pass::InitNodeInfo::run_on_model(const std::shared_ptr& f) { auto& rtInfo = node->get_rt_info(); rtInfo.emplace(FusedNames::get_type_info_static(), FusedNames{node->get_friendly_name()}); } - FixRtInfo{}.run_on_model(f); return false; } diff --git a/src/common/transformations/src/transformations/symbolic_transformations/chained_maximum.cpp b/src/common/transformations/src/transformations/symbolic_transformations/chained_maximum.cpp new file mode 100644 index 00000000000000..02510be34dcb4a --- /dev/null +++ b/src/common/transformations/src/transformations/symbolic_transformations/chained_maximum.cpp @@ -0,0 +1,50 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/chained_maximum.hpp" + +#include +#include + +#include "itt.hpp" +#include "openvino/core/dimension_tracker.hpp" +#include "transformations/symbolic_transformations/utils.hpp" + +using namespace ov::symbol::util; + +ov::pass::ChainedMaximumOptimization::ChainedMaximumOptimization() { + MATCHER_SCOPE(ChainedMaximumOptimization); + auto A_input = pattern::any_input(); + auto B_input = pattern::any_input(); + auto C_input = pattern::any_input(); + auto first_maximum = pattern::wrap_type({A_input, B_input}); + auto maximum = pattern::wrap_type({first_maximum, C_input}); + + ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { + const auto& vm = m.get_pattern_value_map(); + + auto A = vm.at(A_input), B = vm.at(B_input), C = vm.at(C_input); + auto output_to_replace = vm.at(first_maximum); + + ov::TensorLabel A_labels, B_labels, C_labels; + bool A_read = get_labels(A, A_labels); + bool B_read = get_labels(B, B_labels); + bool C_read = get_labels(C, C_labels); + + if (!A_read && !B_read && !C_read) + return false; + + if (are_unique_and_equal_labels(A_labels, C_labels)) { + // Matched Maximum(Maximum(A, B), C) with A == C -> Maximum(B, C) + return ov::replace_output_update_name(output_to_replace, B); + } else if (are_unique_and_equal_labels(B_labels, C_labels)) { + // Matched Maximum(Maximum(A, B), C) with B == C -> Maximum(A, C) + return ov::replace_output_update_name(output_to_replace, A); + } + return false; + }; + + auto m = std::make_shared(maximum, matcher_name); + register_matcher(m, matcher_pass_callback); +} diff --git a/src/common/transformations/src/transformations/symbolic_transformations/label_optimization.cpp b/src/common/transformations/src/transformations/symbolic_transformations/label_optimization.cpp new file mode 100644 index 00000000000000..049fd0e72c06c3 --- /dev/null +++ b/src/common/transformations/src/transformations/symbolic_transformations/label_optimization.cpp @@ -0,0 +1,265 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/label_optimization.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itt.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/op/util/multi_subgraph_base.hpp" + +namespace { +void update_label(const ov::EqTable& table, ov::label_t& label) { + if (label != ov::no_label && table.count(label)) { + const auto& alternative_label = *table.at(label)->begin(); + if (alternative_label != ov::no_label) + label = std::min(label, alternative_label); + } +} + +void apply_table_of_equivalence_on_model(const std::shared_ptr& m, const ov::EqTable& table) { + for (const auto& op : m->get_ordered_ops()) { + // handle inner sub-graphs + if (auto multi_subgraph_op = std::dynamic_pointer_cast(op)) + for (const auto& sub_graph : multi_subgraph_op->get_functions()) + if (sub_graph) + apply_table_of_equivalence_on_model(sub_graph, table); + + for (auto& output : op->outputs()) { + // shape relabeling + auto shape = output.get_partial_shape(); + for (auto& d : shape) { + if (d.is_static()) + continue; + auto label = ov::DimensionTracker::get_label(d); + update_label(table, label); + if (label != ov::no_label) + ov::DimensionTracker::set_label(d, label); + } + op->set_output_type(output.get_index(), output.get_element_type(), shape); + // value relabeling + auto value_labels = output.get_tensor().get_value_label(); + for (auto& label : value_labels) + update_label(table, label); + output.get_tensor().set_value_label(value_labels); + } + } +} +} // namespace + +bool ov::pass::ApplyTableOfEquivalence::run_on_model(const std::shared_ptr& m) { + RUN_ON_FUNCTION_SCOPE(ApplyTableOfEquivalence); + if (auto te = ov::table_of_equivalence(m)) + apply_table_of_equivalence_on_model(m, te->get_equivalence_table()); + return false; +} + +// label to source map +using LTS_map = std::unordered_map>; + +namespace { +int64_t get_idx_of_label_in_source(const ov::Output& source, const ov::label_t& label) { + int64_t idx = -1; + if (label == ov::no_label) + return idx; + auto pshape = source.get_partial_shape(); + auto rank = pshape.rank(); + if (rank.is_dynamic()) + return idx; + for (int64_t i = 0; i < rank.get_length(); ++i) { + auto l = ov::DimensionTracker::get_label(pshape[i]); + if (l == label) { + idx = i; + break; + } + } + return idx; +} + +ov::Output alternative_source_from_existing_value(const ov::label_t& label, + const ov::Output& original_output, + LTS_map& label_value_source) { + auto alternative_source = ov::Output(); + if (label_value_source.count(label)) { + alternative_source = label_value_source[label]; + const auto &original_shape = original_output.get_shape(), &alternative_shape = alternative_source.get_shape(); + const auto &original_et = original_output.get_element_type(), + &alternative_et = alternative_source.get_element_type(); + if (alternative_shape != original_shape && (original_shape.empty() || original_shape == ov::Shape{0})) { + auto squeeze = std::make_shared(alternative_source); + ov::copy_runtime_info(original_output.get_node_shared_ptr(), squeeze); + alternative_source = squeeze->output(0); + } else if (alternative_shape != original_shape) { + auto shape = ov::op::v0::Constant::create(ov::element::i64, {original_shape.size()}, original_shape); + auto reshape = std::make_shared(alternative_source, shape, false); + ov::copy_runtime_info(original_output.get_node_shared_ptr(), reshape); + alternative_source = reshape->output(0); + } + if (alternative_et != original_et) { + auto convert = std::make_shared(alternative_source, original_et); + ov::copy_runtime_info(original_output.get_node_shared_ptr(), convert); + alternative_source = convert->output(0); + } + } + return alternative_source; +} + +ov::Output alternative_source_from_shape_source(const LTS_map& label_shape_source, + const ov::label_t& label, + const ov::Output& original_output, + LTS_map& label_value_source) { + auto alternative_source = ov::Output(); + if (label_shape_source.count(label)) { + // replacing via constructing the label source and saving it for the future + const auto& source = label_shape_source.at(label); + const int64_t& idx = get_idx_of_label_in_source(source, label); + if (idx == -1) + return alternative_source; + const auto& original_et = original_output.get_element_type(); + std::shared_ptr shape; + if (original_et == ov::element::i32 || original_et == ov::element::i64) { + shape = std::make_shared(source, original_et); + } else { + shape = std::make_shared(source); + ov::copy_runtime_info(original_output.get_node_shared_ptr(), shape); + shape = std::make_shared(shape, original_et); + } + auto indices = ov::op::v0::Constant::create(ov::element::i64, original_output.get_shape(), {idx}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); + auto gather = std::make_shared(shape, indices, axis); + ov::copy_runtime_info(original_output.get_node_shared_ptr(), {shape, indices, axis, gather}); + alternative_source = gather; + label_value_source[label] = alternative_source; + } + return alternative_source; +} + +ov::Output get_alternative_source_from_value_or_shape_source(const LTS_map& label_shape_source, + const ov::label_t& label, + const ov::Output& original_output, + LTS_map& label_value_source) { + auto alternative_source = ov::Output(); + if (label == ov::no_label) + return alternative_source; + alternative_source = alternative_source_from_existing_value(label, original_output, label_value_source); + if (!alternative_source.get_node_shared_ptr()) + alternative_source = + alternative_source_from_shape_source(label_shape_source, label, original_output, label_value_source); + return alternative_source; +} + +ov::Output alternative_source_from_concat_input_sources(const LTS_map& label_shape_source, + const ov::label_t& label, + const ov::Output& original_output, + LTS_map& label_value_source) { + auto alternative_source = ov::Output(); + if (label_shape_source.count(label)) { + const auto& source = label_shape_source.at(label); + auto concat = ov::as_type_ptr(source.get_node_shared_ptr()); + if (!concat || concat->get_input_size() != 2) + return alternative_source; + int64_t idx = get_idx_of_label_in_source(source, label); + if (idx == -1 || idx != concat->get_concatenation_axis()) + return alternative_source; + // optimize using the knowledge of the Concat SI and what happens on the axis + const auto& lhs_pshape = concat->get_input_partial_shape(0); + const auto& rhs_pshape = concat->get_input_partial_shape(1); + if (lhs_pshape.rank().is_static() && rhs_pshape.rank().is_static()) { + auto lhs_label = ov::DimensionTracker::get_label(lhs_pshape[idx]); + auto lhs_alternative = get_alternative_source_from_value_or_shape_source(label_shape_source, + lhs_label, + original_output, + label_value_source); + + auto rhs_label = ov::DimensionTracker::get_label(rhs_pshape[idx]); + auto rhs_alternative = get_alternative_source_from_value_or_shape_source(label_shape_source, + rhs_label, + original_output, + label_value_source); + + if (lhs_alternative.get_node_shared_ptr() && rhs_alternative.get_node_shared_ptr()) { + alternative_source = std::make_shared(lhs_alternative, rhs_alternative); + ov::copy_runtime_info(original_output.get_node_shared_ptr(), alternative_source.get_node_shared_ptr()); + alternative_source.get_tensor().set_value_label({label}); + label_value_source[label] = alternative_source; + } + } + } + return alternative_source; +} + +void optimize_value_usage(ov::Output& output, LTS_map& label_shape_source, LTS_map& label_value_source) { + auto value_labels = output.get_tensor().get_value_label(); + if (value_labels.size() != 1) + return; + auto label = value_labels[0]; + if (label == ov::no_label) + return; + auto pshape = output.get_partial_shape(); + if (pshape.is_dynamic() || ov::shape_size(pshape.to_shape()) != 1) + return; + + ov::Output alternative_source = + alternative_source_from_concat_input_sources(label_shape_source, label, output, label_value_source); + if (!alternative_source.get_node_shared_ptr()) + alternative_source = + get_alternative_source_from_value_or_shape_source(label_shape_source, label, output, label_value_source); + + if (alternative_source.get_node_shared_ptr() != nullptr) { + evaluate_both_bounds(alternative_source); + output.replace(alternative_source); + } else { + // in case we can not optimize it -- it is label which appeared just now on the value path + label_value_source[label] = output; + } +} + +void save_shape_sources(const ov::Output& output, LTS_map& label_shape_source) { + for (const auto& d : output.get_partial_shape()) { + if (d.is_static()) + continue; + auto label = ov::DimensionTracker::get_label(d); + if (label == ov::no_label || label_shape_source.count(label)) + continue; + label_shape_source[label] = output; + } +} +} // namespace + +bool ov::pass::OptimizeLabelsUsedAsValues::run_on_model(const std::shared_ptr& m) { + RUN_ON_FUNCTION_SCOPE(OptimizeLabelsUsedAsValues); + LTS_map label_shape_source; + LTS_map label_value_source; + for (const auto& op : m->get_ordered_ops()) { + // Result has output port which has shared (during validate_and_infer_type) tensor with input port. + // Transformations may replace input of Result. After replacement and before Result::validate_and_infer_type -- + // output tensor of Result may contain inaccurate shape / labels due to the sharing with tensor which may be + // already detached from the model. To avoid creating ShapeOf from Result in these cases we exclude it from this + // optimization entirely + if (auto result = ov::as_type_ptr(op)) + continue; + + // LTS maps aren't shared with sub-graphs because inner graph can not access outer graph for label sources + if (auto multi_subgraph_op = std::dynamic_pointer_cast(op)) + for (const auto& sub_graph : multi_subgraph_op->get_functions()) + if (sub_graph) + run_on_model(sub_graph); + + for (auto& output : op->outputs()) { + optimize_value_usage(output, label_shape_source, label_value_source); + save_shape_sources(output, label_shape_source); + } + } + return true; +} diff --git a/src/common/transformations/src/transformations/symbolic_transformations/nop_broadcast.cpp b/src/common/transformations/src/transformations/symbolic_transformations/nop_broadcast.cpp new file mode 100644 index 00000000000000..889bdd4b209d20 --- /dev/null +++ b/src/common/transformations/src/transformations/symbolic_transformations/nop_broadcast.cpp @@ -0,0 +1,61 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/nop_broadcast.hpp" + +#include +#include +#include +#include +#include + +#include "compare.hpp" +#include "itt.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "transformations/symbolic_transformations/utils.hpp" +#include "transformations/utils/utils.hpp" + +using namespace std; +using namespace ov; +using namespace ov::op; +using namespace ov::symbol::util; + +namespace { +shared_ptr broadcast_label(const OutputVector& inputs) { + return ov::pass::pattern::wrap_type(inputs, [](Output output) { + const auto& op = output.get_node_shared_ptr(); + auto data_rank = op->get_input_partial_shape(0).rank(); + auto new_shape_shape = op->get_input_partial_shape(1); + return data_rank.is_static() && new_shape_shape.is_static() && data_rank == new_shape_shape[0]; + }); +} +} // namespace + +ov::pass::NopBroadcast::NopBroadcast() { + MATCHER_SCOPE(NopBroadcast); + auto data_label = pattern::any_input(pattern::has_static_rank()); + + auto shape_label = pattern::wrap_type(); + auto ones = INT_CONSTANT_WITH_PREDICATE(std::all_of(value.begin(), value.end(), cmp::Equal(1))); + auto maximum = pattern::wrap_type({shape_label, ones}); + + auto broadcast_3_ins = broadcast_label({data_label, maximum, pattern::any_input()}); + auto broadcast_2_ins = broadcast_label({data_label, maximum}); + auto broadcast = make_shared(OutputVector{broadcast_2_ins, broadcast_3_ins}); + + ov::matcher_pass_callback matcher_pass_callback = [=](pattern::Matcher& m) { + const auto& vm = m.get_pattern_value_map(); + auto data = vm.at(data_label); + auto shape = vm.at(shape_label); + + ov::TensorLabel data_labels, shape_labels; + if (!get_labels(data.get_partial_shape(), data_labels) || !get_labels(shape, shape_labels) || + !are_unique_and_equal_labels(data_labels, shape_labels)) + return false; + return ov::replace_output_update_name(m.get_match_root(), data); + }; + + auto m = std::make_shared(broadcast, matcher_name); + register_matcher(m, matcher_pass_callback); +} diff --git a/src/common/transformations/src/transformations/symbolic_transformations/symbolic_optimizations.cpp b/src/common/transformations/src/transformations/symbolic_transformations/symbolic_optimizations.cpp new file mode 100644 index 00000000000000..7451df397ba33c --- /dev/null +++ b/src/common/transformations/src/transformations/symbolic_transformations/symbolic_optimizations.cpp @@ -0,0 +1,146 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/symbolic_optimizations.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itt.hpp" + +namespace { +void symbolic_set_up_for_shape(ov::DimensionTracker& dt, ov::PartialShape& shape) { + if (shape.rank().is_dynamic()) + return; + for (auto& d : shape) { + bool is_static = d.is_static(), has_label = ov::DimensionTracker::has_label(d); + if (is_static && has_label) + dt.reset_tracking_info(d); // remove labels from static dims on shapes to reduce label clutter + if (is_static || has_label) + continue; + dt.set_up_for_tracking(d); + } +} + +void special_case_range_label_propagation(const std::shared_ptr& node) { + /* Label propagation through specific Range operation + start shift + | \ / + | Add step == 1 + \ / / + Range + */ + if (!ov::is_type(node) && !ov::is_type(node)) + return; + + auto output_shape = node->get_output_partial_shape(0); + if (output_shape.rank().is_dynamic() || output_shape.size() != 1) + return; + + OPENVINO_SUPPRESS_DEPRECATED_START + auto step_value = ov::get_constant_from_source(node->input_value(2)); + OPENVINO_SUPPRESS_DEPRECATED_END + if (!step_value || step_value->cast_vector()[0] != 1) + return; + + auto start_labels = node->get_input_tensor(0).get_value_label(); + if (start_labels.size() != 1 || start_labels[0] == ov::no_label) + return; + auto start_label = start_labels[0]; + + auto stop_node = node->input_value(1).get_node_shared_ptr(); + if (!ov::is_type(stop_node)) + return; + auto add_in0_labels = stop_node->get_input_tensor(0).get_value_label(); + if (add_in0_labels.size() != 1 || add_in0_labels[0] == ov::no_label) + return; + auto add_in0_label = add_in0_labels[0]; + + auto add_in1_labels = stop_node->get_input_tensor(1).get_value_label(); + if (add_in1_labels.size() != 1 || add_in1_labels[0] == ov::no_label) + return; + auto add_in1_label = add_in1_labels[0]; + + if (add_in0_label == start_label) + ov::DimensionTracker::set_label(output_shape[0], add_in1_label); + else if (add_in1_label == start_label) + ov::DimensionTracker::set_label(output_shape[0], add_in0_label); + node->set_output_type(0, node->get_output_element_type(0), output_shape); +} +} // namespace + +ov::pass::SymbolicPropagation::SymbolicPropagation() { + m_te = std::make_shared(); +} + +bool ov::pass::SymbolicPropagation::run_on_model(const std::shared_ptr& m) { + RUN_ON_MODEL_SCOPE(SymbolicPropagation); + + auto te = m_te; + ov::set_up_symbolic_info(m, te); + ov::DimensionTracker dt(te); + + for (const auto& op : m->get_ordered_ops()) { + // since we disable invalidation with the following two lines, we have to invalidate manually here + op->invalidate_values(); + for (auto& output : op->outputs()) + ov::set_up_symbolic_info(output, te); + op->revalidate_and_infer_types(); + // Recursively apply transformation for sub-graph based operations + if (auto multi_subgraph_op = std::dynamic_pointer_cast(op)) + for (const auto& sub_graph : multi_subgraph_op->get_functions()) + if (sub_graph) + run_on_model(sub_graph); + + // additional label propagation rules must be triggered here + special_case_range_label_propagation(op); + // additional label propagation rules must be triggered here + + for (auto& output : op->outputs()) { + auto shape = output.get_partial_shape(); + symbolic_set_up_for_shape(dt, shape); + OPENVINO_SUPPRESS_DEPRECATED_START + output.get_tensor().set_tensor_type(output.get_element_type(), shape); + OPENVINO_SUPPRESS_DEPRECATED_END + } + } + return true; +} + +ov::pass::SymbolicOptimizations::SymbolicOptimizations(bool full_run) { + m_manager = std::make_shared(); + m_manager->set_per_pass_validation(false); + +#define REGISTER_SYMBOLIC(region, ...) m_manager->register_pass(__VA_ARGS__); + + REGISTER_SYMBOLIC(SymbolicPropagation) + if (full_run) { + // symbolic based transformations allowing for better static dimension propagation + REGISTER_SYMBOLIC(ChainedMaximumOptimization) + REGISTER_SYMBOLIC(NopBroadcast) + // regular transformations which are needed right now since they clean up unnecessary operations + REGISTER_SYMBOLIC(NopElimination) // Broadcast (Tile) Ones + Remove Slice Before GatherElements + REGISTER_SYMBOLIC(SharedOpOptimization) // Shared GatherElements + } + // transformations which use labels for optimizations + REGISTER_SYMBOLIC(ApplyTableOfEquivalence) + if (full_run) { + REGISTER_SYMBOLIC(OptimizeLabelsUsedAsValues) // reduce shape sub-graphs + } +} + +bool ov::pass::SymbolicOptimizations::run_on_model(const std::shared_ptr& m) { + RUN_ON_FUNCTION_SCOPE(SymbolicOptimizations); + m_manager->run_passes(m); + ov::remove_symbolic_info(m); + return true; +} diff --git a/src/common/transformations/src/transformations/symbolic_transformations/utils.cpp b/src/common/transformations/src/transformations/symbolic_transformations/utils.cpp new file mode 100644 index 00000000000000..3fedc3bd4c85be --- /dev/null +++ b/src/common/transformations/src/transformations/symbolic_transformations/utils.cpp @@ -0,0 +1,34 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/utils.hpp" + +#include +#include +#include + +bool ov::symbol::util::get_labels(const ov::PartialShape& shape, ov::TensorLabel& labels) { + if (shape.rank().is_dynamic()) + return false; + labels.clear(); + labels.reserve(shape.size()); + for (const auto& d : shape) + labels.push_back((d.is_dynamic() ? ov::DimensionTracker::get_label(d) : ov::no_label)); + return true; +} + +bool ov::symbol::util::get_labels(const ov::Output& output, ov::TensorLabel& labels) { + const auto& tensor = output.get_tensor(); + labels = tensor.get_value_label(); + return !labels.empty(); +} + +bool ov::symbol::util::are_unique_and_equal_labels(const ov::TensorLabel& lhs, const ov::TensorLabel& rhs) { + if (rhs.size() != lhs.size() || rhs.empty()) + return false; + for (size_t i = 0; i < lhs.size(); ++i) + if (lhs[i] != rhs[i] || lhs[i] == ov::no_label) + return false; + return true; +} diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp index d7609c935f1d61..42fa6f85cc61cf 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_gather.cpp @@ -57,16 +57,17 @@ TSGatherForward::TSGatherForward() { } } - size_t axis; + size_t order_axis; if (axes[0] < 0) { auto data_rank = main_node->get_input_partial_shape(0).rank(); if (data_rank.is_dynamic()) { return false; } - axis = static_cast(axes[0] + data_rank.get_length()); + order_axis = static_cast(axes[0] + data_rank.get_length()); } else { - axis = static_cast(axes[0]); + order_axis = static_cast(axes[0]); } + const size_t axis = order_val[order_axis]; /* https://docs.openvino.ai/2023.0/openvino_docs_ops_movement_Gather_8.html The Gather output shape has the same shape as the input, @@ -136,8 +137,7 @@ TSGatherForward::TSGatherForward() { if (!success) { return false; } - auto new_axis = - ov::op::v0::Constant::create(gather_axis->get_element_type(), gather_axis->get_shape(), {order_val[axis]}); + auto new_axis = ov::op::v0::Constant::create(gather_axis->get_element_type(), gather_axis->get_shape(), {axis}); main_node->input(2).replace_source_output(new_axis); copy_runtime_info(gather_axis, new_axis); diff --git a/src/common/transformations/src/transformations/utils/utils.cpp b/src/common/transformations/src/transformations/utils/utils.cpp index a62cd141b9f83b..62b1765e7ba275 100644 --- a/src/common/transformations/src/transformations/utils/utils.cpp +++ b/src/common/transformations/src/transformations/utils/utils.cpp @@ -162,16 +162,24 @@ std::vector> get_node_target_inputs(const std::shared_ptr& nod std::shared_ptr node_to_get_shape_value_of_indices_from_shape_node( const std::shared_ptr& shape_node, - const std::vector& indices) { - return make_try_fold(shape_node, - v0::Constant::create(ov::element::i64, {indices.size()}, indices), - v0::Constant::create(ov::element::i64, {}, {0})); + const std::vector& indices, + const std::vector>& copy_rt_info_from) { + const auto& indices_op = v0::Constant::create(ov::element::i64, {indices.size()}, indices); + const auto& axis_op = v0::Constant::create(ov::element::i64, {}, {0}); + auto op = make_try_fold(shape_node, indices_op, axis_op); + if (!copy_rt_info_from.empty()) + ov::copy_runtime_info(copy_rt_info_from, {op, indices_op, axis_op}); + return op; } -std::shared_ptr node_to_get_shape_value_of_indices_from_shape_source(const ov::Output& shape_source, - const std::vector& indices) { +std::shared_ptr node_to_get_shape_value_of_indices_from_shape_source( + const ov::Output& shape_source, + const std::vector& indices, + const std::vector>& copy_rt_info_from) { const auto& shape_node = make_try_fold(shape_source); - return node_to_get_shape_value_of_indices_from_shape_node(shape_node, indices); + if (!copy_rt_info_from.empty()) + ov::copy_runtime_info(copy_rt_info_from, shape_node); + return node_to_get_shape_value_of_indices_from_shape_node(shape_node, indices, copy_rt_info_from); } bool shapes_equal_except_dynamic_expected_batch(const ov::PartialShape& expected, const ov::PartialShape& actual) { diff --git a/src/common/transformations/tests/symbolic_transformations/chained_maximum.cpp b/src/common/transformations/tests/symbolic_transformations/chained_maximum.cpp new file mode 100644 index 00000000000000..f0507897fc4a36 --- /dev/null +++ b/src/common/transformations/tests/symbolic_transformations/chained_maximum.cpp @@ -0,0 +1,134 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/chained_maximum.hpp" + +#include + +#include +#include +#include +#include +#include + +#include "common_test_utils/ov_test_utils.hpp" +#include "openvino/core/dimension_tracker.hpp" +#include "transformations/symbolic_transformations/symbolic_optimizations.hpp" + +using namespace ov; +using namespace ov::op; +using namespace std; + +TEST_F(TransformationTestsF, ChainedMaximumAC) { + // A == C + // Maximum(Maximum(A, B), C) -> Maximum(B, C) + { + auto input = make_shared(element::f32, PartialShape::dynamic(4)); + + auto A = make_shared(input); + auto B = v0::Constant::create(element::i64, {}, {1}); + auto C = make_shared(input); + + auto maximum_0 = make_shared(A, B); + auto maximum_1 = make_shared(maximum_0, C); + + auto data = make_shared(element::f32, PartialShape::dynamic()); + auto broadcast = make_shared(data, maximum_1); + + model = make_shared(NodeVector{broadcast}, ParameterVector{input, data}); + manager.set_per_pass_validation(false); + manager.register_pass(); + manager.register_pass(); + } + { + auto input = make_shared(element::f32, PartialShape::dynamic(4)); + + auto B = v0::Constant::create(element::i64, {}, {1}); + auto C = make_shared(input); + + auto maximum = make_shared(B, C); + + auto data = make_shared(element::f32, PartialShape::dynamic()); + auto broadcast = make_shared(data, maximum); + + model_ref = make_shared(NodeVector{broadcast}, ParameterVector{input, data}); + } +} + +TEST_F(TransformationTestsF, ChainedMaximumBC) { + // B == C + // Maximum(Maximum(A, B), C) -> Maximum(A, C) + { + auto input = make_shared(element::f32, PartialShape::dynamic(4)); + + auto A = v0::Constant::create(element::i64, {}, {1}); + auto B = make_shared(input); + auto C = make_shared(input); + + auto maximum_0 = make_shared(A, B); + auto maximum_1 = make_shared(maximum_0, C); + auto data = make_shared(element::f32, PartialShape::dynamic()); + auto broadcast = make_shared(data, maximum_1); + + model = make_shared(NodeVector{broadcast}, ParameterVector{input, data}); + manager.set_per_pass_validation(false); + manager.register_pass(); + manager.register_pass(); + } + { + auto input = make_shared(element::f32, PartialShape::dynamic(4)); + + auto A = v0::Constant::create(element::i64, {}, {1}); + auto C = make_shared(input); + + auto maximum = make_shared(A, C); + + auto data = make_shared(element::f32, PartialShape::dynamic()); + auto broadcast = make_shared(data, maximum); + + model_ref = make_shared(NodeVector{broadcast}, ParameterVector{input, data}); + } +} + +TEST_F(TransformationTestsF, ChainedMaximumNegativeNoLabels) { + { + auto shape = PartialShape::dynamic(4); + auto input = make_shared(element::f32, shape); + + auto A = v0::Constant::create(element::i64, {}, {1}); + auto B = make_shared(input); + auto C = make_shared(input); + + auto maximum_0 = make_shared(A, B); + auto maximum_1 = make_shared(maximum_0, C); + + auto data = make_shared(element::f32, PartialShape::dynamic()); + auto broadcast = make_shared(data, maximum_1); + + model = make_shared(NodeVector{broadcast}, ParameterVector{input, data}); + manager.register_pass(); + } +} + +TEST_F(TransformationTestsF, ChainedMaximumNegativeDifferentLabels) { + { + auto input_0 = make_shared(element::f32, PartialShape::dynamic(4)); + auto input_1 = make_shared(element::f32, PartialShape::dynamic(4)); + + auto A = v0::Constant::create(element::i64, {}, {1}); + auto B = make_shared(input_0); + auto C = make_shared(input_1); + + auto maximum_0 = make_shared(A, B); + auto maximum_1 = make_shared(maximum_0, C); + + auto data = make_shared(element::f32, PartialShape::dynamic()); + auto broadcast = make_shared(data, maximum_1); + + model = make_shared(NodeVector{broadcast}, ParameterVector{input_0, input_1, data}); + manager.set_per_pass_validation(false); + manager.register_pass(); + manager.register_pass(); + } +} diff --git a/src/common/transformations/tests/symbolic_transformations/label_optimization.cpp b/src/common/transformations/tests/symbolic_transformations/label_optimization.cpp new file mode 100644 index 00000000000000..408467db47779f --- /dev/null +++ b/src/common/transformations/tests/symbolic_transformations/label_optimization.cpp @@ -0,0 +1,97 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/label_optimization.hpp" + +#include + +#include "common_test_utils/ov_test_utils.hpp" +#include "openvino/core/dimension_tracker.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/gather.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/shape_of.hpp" +#include "openvino/op/util/symbolic_info.hpp" +#include "openvino/pass/manager.hpp" +#include "transformations/symbolic_transformations/symbolic_optimizations.hpp" +#include "transformations/symbolic_transformations/utils.hpp" + +using namespace ov; +using namespace ov::op; +using namespace std; + +#define TEST_SHAPE(source, ...) \ + { \ + auto expected_labels = ov::TensorLabel({__VA_ARGS__}); \ + ov::TensorLabel labels; \ + EXPECT_TRUE(ov::symbol::util::get_labels(source->get_output_partial_shape(0), labels)); \ + EXPECT_EQ(labels, expected_labels); \ + } + +TEST(TransformationTests, ApplyTableOfEquivalence_Concat) { + auto input_1 = make_shared(element::f32, PartialShape::dynamic(4)); + auto input_2 = make_shared(element::f32, PartialShape::dynamic(4)); + auto concat = make_shared(OutputVector{input_1, input_2}, -1); + // shape inference notes that all the non-axis dimensions are equal to each other + auto model = make_shared(NodeVector{concat}, ParameterVector{input_2, input_1}); + + pass::Manager manager; + manager.set_per_pass_validation(false); + manager.register_pass(); + manager.register_pass(); + manager.run_passes(model); + + TEST_SHAPE(input_1, {1, 2, 3, 4}); + TEST_SHAPE(input_2, {1, 2, 3, 8}); + TEST_SHAPE(concat, {1, 2, 3, 9}); +} + +TEST_F(TransformationTestsF, ApplyTableOfEquivalence_Concat_Values) { + { + auto input_1 = make_shared(element::f32, PartialShape::dynamic(4)); + auto input_2 = make_shared(element::f32, PartialShape::dynamic(4)); + auto concat = make_shared(OutputVector{input_1, input_2}, -1); + + auto shape = make_shared(concat); + auto gather = make_shared(shape, + v0::Constant::create(element::i64, {1}, {-1}), + v0::Constant::create(element::i64, {}, {0})); + + auto reshape = make_shared( + concat, + make_shared(OutputVector{gather, v0::Constant::create(element::i64, {1}, {-1})}, 0), + false); + + model = make_shared(NodeVector{reshape}, ParameterVector{input_2, input_1}); + + manager.set_per_pass_validation(false); + manager.register_pass(); + manager.register_pass(); + manager.register_pass(); + } + { + auto input_1 = make_shared(element::f32, PartialShape::dynamic(4)); + auto input_2 = make_shared(element::f32, PartialShape::dynamic(4)); + auto concat = make_shared(OutputVector{input_1, input_2}, -1); + + auto shape_1 = make_shared(input_1); + auto gather_1 = make_shared(shape_1, + v0::Constant::create(element::i64, {1}, {3}), + v0::Constant::create(element::i64, {}, {0})); + + auto shape_2 = make_shared(input_2); + auto gather_2 = make_shared(shape_2, + v0::Constant::create(element::i64, {1}, {3}), + v0::Constant::create(element::i64, {}, {0})); + + auto sum = make_shared(gather_1, gather_2); + + auto reshape = make_shared( + concat, + make_shared(OutputVector{sum, v0::Constant::create(element::i64, {1}, {-1})}, 0), + false); + model_ref = make_shared(NodeVector{reshape}, ParameterVector{input_2, input_1}); + } +} diff --git a/src/common/transformations/tests/symbolic_transformations/nop_broadcast.cpp b/src/common/transformations/tests/symbolic_transformations/nop_broadcast.cpp new file mode 100644 index 00000000000000..63f97dfc7cb04b --- /dev/null +++ b/src/common/transformations/tests/symbolic_transformations/nop_broadcast.cpp @@ -0,0 +1,111 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/symbolic_transformations/nop_broadcast.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +#include "common_test_utils/ov_test_utils.hpp" +#include "openvino/core/dimension_tracker.hpp" + +using namespace ov; +using namespace ov::op; +using namespace std; + +namespace { +void label_shape(ov::PartialShape& shape) { + auto table = std::make_shared(42); + auto tracker = ov::DimensionTracker(table); + tracker.set_up_for_tracking(shape); +} +} // namespace + +TEST_F(TransformationTestsF, NopBroadcastOpset1) { + { + auto shape = PartialShape::dynamic(4); + label_shape(shape); // we label shape with consecutive labels: 42, 43, 44, 45 + + auto data = make_shared(element::f32, shape); + + auto labeled_input = make_shared(element::f32, shape); + auto shape_of = make_shared(labeled_input); + auto ones = ov::op::v0::Constant::create(element::i64, {}, {1}); + auto maximum = make_shared(shape_of, ones); + + auto broadcast = make_shared(data, maximum); + auto relu = make_shared(broadcast); + + model = make_shared(NodeVector{relu}, ParameterVector{data, labeled_input}); + manager.register_pass(); + } + { + auto shape = PartialShape::dynamic(4); + label_shape(shape); // we label shape with consecutive labels: 42, 43, 44, 45 + + auto data = make_shared(element::f32, shape); + auto relu = make_shared(data); + + auto labeled_input = make_shared(element::f32, shape); + + model_ref = make_shared(NodeVector{relu}, ParameterVector{data, labeled_input}); + } +} + +TEST_F(TransformationTestsF, NopBroadcastOpset3) { + { + auto shape = PartialShape::dynamic(4); + label_shape(shape); // we label shape with consecutive labels: 42, 43, 44, 45 + + auto data = make_shared(element::f32, shape); + + auto labeled_input = make_shared(element::f32, shape); + auto shape_of = make_shared(labeled_input); + auto ones = ov::op::v0::Constant::create(element::i64, {4}, {1, 1, 1, 1}); + auto maximum = make_shared(shape_of, ones); + + auto broadcast = make_shared(data, maximum); + auto relu = make_shared(broadcast); + + model = make_shared(NodeVector{relu}, ParameterVector{data, labeled_input}); + manager.register_pass(); + } + { + auto shape = PartialShape::dynamic(4); + label_shape(shape); // we label shape with consecutive labels: 42, 43, 44, 45 + + auto data = make_shared(element::f32, shape); + auto relu = make_shared(data); + + auto labeled_input = make_shared(element::f32, shape); + + model_ref = make_shared(NodeVector{relu}, ParameterVector{data, labeled_input}); + } +} + +TEST_F(TransformationTestsF, NopBroadcastNegative) { + { + auto shape = PartialShape::dynamic(1); + label_shape(shape); // we label shape with consecutive labels: 42 + + auto data = make_shared(element::f32, shape); + + auto labeled_input = make_shared(element::f32, shape); + auto shape_of = make_shared(labeled_input); + auto ones = ov::op::v0::Constant::create(element::i64, {2}, {1, 1}); + auto maximum = make_shared(shape_of, ones); + + auto broadcast = make_shared(data, maximum); + auto relu = make_shared(broadcast); + + model = make_shared(NodeVector{relu}, ParameterVector{data, labeled_input}); + manager.register_pass(); + } +} diff --git a/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp b/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp index 34586bcdd81178..2a410fb0181333 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_gather_test.cpp @@ -63,6 +63,7 @@ auto wrapper = [](const TestCase& test_case) { struct GatherForwardArguments { OutputVector inputs_to_main; + std::function&, const OutputVector&)> create_input_transpose_to_main; Output new_Gather_first_input; AxisVector new_transpose_order; }; @@ -76,7 +77,7 @@ auto test_forward_gather = [](const GatherForwardArguments& test_arguments) { test_case.inputs_to_main = test_arguments.inputs_to_main; // Test model description: - test_case.model.preprocess_inputs_to_main = {{set_transpose_for}, {{0}}}; + test_case.model.preprocess_inputs_to_main = {{test_arguments.create_input_transpose_to_main}, {{0}}}; test_case.model.main_op = {CREATE_GATHER_FACTORY(Gather)}; test_case.model.model_template = create_model; @@ -104,24 +105,44 @@ auto test_forward_gather = [](const GatherForwardArguments& test_arguments) { return wrapper(test_case); }; +class SetTransposeWithOrder { +public: + SetTransposeWithOrder(const AxisVector& order) : _order(order) {} + OutputVector operator()(const vector& idxs, const OutputVector& out_vec) const { + return set_transpose_with_order(idxs, out_vec, _order); + } + +private: + const AxisVector _order; +}; + vector tests_arguments_fw{ {{{parameter(f32, {3, 4, 5, 6}), constant(i32, {2}, {0, 2}), constant(i32, {1}, {2})}}, + set_transpose_for, constant(i32, {1}, {1}), AxisVector{3, 2, 1, 0}}, {{parameter(f32, {2, 4}), constant(i32, {}, {0}), constant(i32, {1}, {1})}, + set_transpose_for, constant(i32, {1}, {0}), AxisVector{0}}, {{parameter(f32, {2, 4}), constant(i32, {1}, {0}), constant(i32, {1}, {1})}, + set_transpose_for, constant(i32, {1}, {0}), AxisVector{1, 0}}, {{parameter(f32, {2, 3, 4}), constant(i32, {2, 3}, {0, 1, 0, 1, 0, 1}), constant(i32, {1}, {1})}, + set_transpose_for, constant(i32, {1}, {1}), - AxisVector{3, 1, 2, 0}}}; + AxisVector{3, 1, 2, 0}}, + {{parameter(f32, {64, 49, 3, 3, 32}), constant(i32, {}, {1}), constant(i32, {}, {0})}, + SetTransposeWithOrder(AxisVector{2, 0, 3, 1, 4}), + constant(i32, {}, {2}), + AxisVector{0, 2, 1, 3}}}; INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_0, TSTestFixture, test_forward_gather(tests_arguments_fw[0])); INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_1, TSTestFixture, test_forward_gather(tests_arguments_fw[1])); INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_2, TSTestFixture, test_forward_gather(tests_arguments_fw[2])); INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_3, TSTestFixture, test_forward_gather(tests_arguments_fw[3])); +INSTANTIATE_TEST_SUITE_P(TSCommonGatherForward_4, TSTestFixture, test_forward_gather(tests_arguments_fw[4])); struct GatherBackwardArguments { OutputVector inputs_to_main; diff --git a/src/common/transformations/tests/transpose_sinking/ts_test_utils.cpp b/src/common/transformations/tests/transpose_sinking/ts_test_utils.cpp index d0a81c5dd404bf..31be099f03a868 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_test_utils.cpp +++ b/src/common/transformations/tests/transpose_sinking/ts_test_utils.cpp @@ -52,6 +52,19 @@ OutputVector set_transpose_for(const vector& idxs, const OutputVector& o return result; } +OutputVector set_transpose_with_order(const vector& idxs, + const OutputVector& out_vec, + const vector& transpose_order_axes) { + OutputVector result = out_vec; + for (const auto& idx : idxs) { + const auto& out = out_vec[idx]; + auto order = make_shared(element::i32, Shape{transpose_order_axes.size()}, transpose_order_axes); + auto transpose = make_shared(out, order); + result[idx] = transpose; + } + return result; +} + OutputVector set_gather_for(const vector& idxs, const OutputVector& out_vec) { OutputVector result = out_vec; for (const auto& idx : idxs) { diff --git a/src/common/transformations/tests/transpose_sinking/ts_test_utils.hpp b/src/common/transformations/tests/transpose_sinking/ts_test_utils.hpp index 826d75e0b1bcdb..cc8685c1d7e911 100644 --- a/src/common/transformations/tests/transpose_sinking/ts_test_utils.hpp +++ b/src/common/transformations/tests/transpose_sinking/ts_test_utils.hpp @@ -21,6 +21,9 @@ std::string to_string(const ov::Shape& shape); ov::ParameterVector filter_parameters(const ov::OutputVector& out_vec); ov::OutputVector set_transpose_for(const std::vector& idxs, const ov::OutputVector& out_vec); +ov::OutputVector set_transpose_with_order(const std::vector& idxs, + const ov::OutputVector& out_vec, + const std::vector& transpose_order_axes); ov::OutputVector set_gather_for(const std::vector& idxs, const ov::OutputVector& out_vec); std::shared_ptr create_main_node(const ov::OutputVector& inputs, size_t num_ops, const FactoryPtr& creator); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 4d8545a5a21802..869b9a02c49272 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -8,10 +8,12 @@ set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) add_definitions(-DIN_OV_CORE_LIBRARY) set(OV_CORE_INCLUDE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/include) +set(OV_CORE_DEV_API_PATH ${CMAKE_CURRENT_SOURCE_DIR}/dev_api) file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp) -file(GLOB_RECURSE PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp) +file(GLOB_RECURSE PUBLIC_HEADERS ${OV_CORE_INCLUDE_PATH}/*.hpp) +file(GLOB_RECURSE DEV_HEADERS ${OV_CORE_DEV_API_PATH}/*.hpp) add_subdirectory(builder) add_subdirectory(reference) @@ -93,7 +95,7 @@ ov_abi_free_target(ngraph_obj) ov_ncc_naming_style(FOR_TARGET ngraph_obj SOURCE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include") -ov_add_clang_format_target(ngraph_clang FOR_TARGETS ngraph_obj) +ov_add_clang_format_target(ngraph_clang FOR_SOURCES ${LIBRARY_SRC} ${PUBLIC_HEADERS} ${DEV_HEADERS}) if(NOT BUILD_SHARED_LIBS) target_compile_definitions(ngraph_obj PUBLIC OPENVINO_STATIC_LIBRARY) @@ -122,7 +124,6 @@ file(GLOB_RECURSE smart_reshape_srcs ${CMAKE_CURRENT_SOURCE_DIR}/src/pass/smart_ file(GLOB_RECURSE rt_info_srcs ${CMAKE_CURRENT_SOURCE_DIR}/src/pass/rt_info/*.cpp) set_source_files_properties("${CMAKE_CURRENT_SOURCE_DIR}/src/pass/convert_precision.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/pass/convert_fp32_to_fp16.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/pass/fix_rt_info.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/pass/init_node_info.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/pass/serialize.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/op/type_relaxed.cpp" diff --git a/src/core/dev_api/openvino/core/dimension_tracker.hpp b/src/core/dev_api/openvino/core/dimension_tracker.hpp index 51f85727e2380c..612ab4a01f8318 100644 --- a/src/core/dev_api/openvino/core/dimension_tracker.hpp +++ b/src/core/dev_api/openvino/core/dimension_tracker.hpp @@ -9,6 +9,7 @@ #include #include "openvino/core/dimension.hpp" +#include "openvino/core/partial_shape.hpp" #include "openvino/core/type/element_type.hpp" namespace ov { @@ -31,9 +32,11 @@ class OPENVINO_API DimensionTracker { void set_table_of_equivalence(ov::Dimension& d) const; static const std::shared_ptr& get_table_of_equivalence(const ov::Dimension& d); + void set_up_for_tracking(ov::PartialShape& shape); void set_up_for_tracking(ov::Dimension& d); void set_up_for_tracking(ov::Dimension& d, label_t label) const; static void reset_tracking_info(ov::Dimension& d); + private: std::shared_ptr m_table_of_equivalence; }; @@ -44,13 +47,14 @@ using ValTable = std::unordered_map; class OPENVINO_API TableOfEquivalence { public: - explicit TableOfEquivalence(label_t label = 1) : current_label(label) {}; + explicit TableOfEquivalence(label_t label = 1) : current_label(label){}; void set_as_equal(const ov::Dimension& lhs, const ov::Dimension& rhs); bool are_equal(const ov::Dimension& lhs, const ov::Dimension& rhs); const EqTable& get_equivalence_table() const; const ValTable& get_value_equivalence_table() const; label_t get_next_label(); + private: label_t current_label; EqTable dimension_table_of_equivalence; diff --git a/src/core/dev_api/shape_util.hpp b/src/core/dev_api/shape_util.hpp index fb935ed51275d9..10acfcbc80f2fe 100644 --- a/src/core/dev_api/shape_util.hpp +++ b/src/core/dev_api/shape_util.hpp @@ -5,6 +5,7 @@ #pragma once #include "openvino/core/shape.hpp" +#include "openvino/op/util/attr_types.hpp" namespace ov { namespace util { @@ -50,7 +51,7 @@ OPENVINO_API Shape reduce(const Shape& input, const AxisSet& axes); OPENVINO_API Shape reduce(const Shape& input, const AxisSet& axes, const bool keep_dims); /** - * @brief Creates reduced vector from input by removing elements. + * @brief Creates reduced vector from input by removing elements. * * @param input Input vector for reduce calculation. * @param axes Reduction axes. @@ -76,6 +77,6 @@ OPENVINO_API Shape reduce_keep_dims(const Shape& input, const AxisSet& axes); * * @return Result shape from inputs with applied broadcast specification. */ -Shape get_broadcast_shape(const Shape& first, const Shape& second, const op::AutoBroadcastSpec& broadcast_spec); +Shape get_broadcast_shape(const Shape& first, const Shape& second, const ov::op::AutoBroadcastSpec& broadcast_spec); } // namespace util } // namespace ov diff --git a/src/core/include/openvino/op/constant.hpp b/src/core/include/openvino/op/constant.hpp index 00e48a7f392951..3c2d048e0db7bf 100644 --- a/src/core/include/openvino/op/constant.hpp +++ b/src/core/include/openvino/op/constant.hpp @@ -295,10 +295,11 @@ class OPENVINO_API Constant : public Op { /// \brief Return the Constant's value as a vector cast to type T /// - /// \tparam T Type to which data vector's entries will be cast. + /// \tparam T Type to which data vector's entries will be cast. + /// \param num_elements (Optional) Number of elements to cast. In default case returns all elements /// \return Constant's data vector. template - std::vector cast_vector() const { + std::vector cast_vector(int64_t num_elements = -1) const { auto source_type = get_element_type(); std::vector rc; using Type_t = element::Type_t; @@ -306,54 +307,58 @@ class OPENVINO_API Constant : public Op { # pragma warning(push) # pragma warning(disable : 4244) #endif + size_t num_elements_in_constant = shape_size(m_shape); + size_t num_elements_to_cast = + (num_elements < 0 ? num_elements_in_constant + : std::min(static_cast(num_elements), num_elements_in_constant)); switch (source_type) { case Type_t::boolean: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::bf16: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::f16: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::f32: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::f64: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::i4: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::i8: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::i16: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::i32: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::i64: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::u1: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::u4: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::u8: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::u16: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::u32: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; case Type_t::u64: - cast_vector(rc); + cast_vector(rc, num_elements_to_cast); break; default: OPENVINO_THROW("unsupported type"); @@ -438,15 +443,19 @@ class OPENVINO_API Constant : public Op { typename std::enable_if::type = true> - void cast_vector(std::vector& output_vector) const { + void cast_vector(std::vector& output_vector, size_t num_elements) const { // this function is workaround for waring during windows building // build complains for vector creation based on iterators // which point on different type than destination vector::value_type using IN_T = fundamental_type_for; auto source_vector = get_vector(); - output_vector.reserve(source_vector.size()); + auto output_size = std::min(num_elements, source_vector.size()); + output_vector.reserve(output_size); - std::transform(source_vector.begin(), source_vector.end(), std::back_inserter(output_vector), [](IN_T c) { + std::transform(source_vector.begin(), + source_vector.begin() + output_size, + std::back_inserter(output_vector), + [](IN_T c) { #ifdef __clang__ # pragma clang diagnostic push # ifdef __has_warning @@ -465,12 +474,13 @@ class OPENVINO_API Constant : public Op { # pragma warning(disable : 4018) # pragma warning(disable : 4804) #endif - if (!std::is_same::value) { - OPENVINO_ASSERT(!std::numeric_limits::is_signed || std::numeric_limits::lowest() <= c, - "Cannot cast vector from constant. Some values are outside the range."); - OPENVINO_ASSERT(std::numeric_limits::max() >= c, - "Cannot cast vector from constant. Some values are outside the range."); - } + if (!std::is_same::value) { + OPENVINO_ASSERT( + !std::numeric_limits::is_signed || std::numeric_limits::lowest() <= c, + "Cannot cast vector from constant. Some values are outside the range."); + OPENVINO_ASSERT(std::numeric_limits::max() >= c, + "Cannot cast vector from constant. Some values are outside the range."); + } #if defined(__clang__) # pragma clang diagnostic pop #elif defined(__GNUC__) @@ -478,16 +488,16 @@ class OPENVINO_API Constant : public Op { #elif defined(_MSC_VER) # pragma warning(pop) #endif - return static_cast(c); - }); + return static_cast(c); + }); } template ::type = true> - void cast_vector(std::vector& output) const { + void cast_vector(std::vector& output, size_t num_elements) const { using IN_T = fundamental_type_for; - const auto element_number = shape_size(m_shape); + const auto element_number = std::min(num_elements, shape_size(m_shape)); const auto source_begin = get_data_ptr(); const auto source_end = std::next(source_begin, (element_number + 7) / 8); const auto round_element_no = element_number % 8 ? element_number - element_number % 8 + 8 : element_number; @@ -504,9 +514,9 @@ class OPENVINO_API Constant : public Op { template ::type = true> - void cast_vector(std::vector& output) const { + void cast_vector(std::vector& output, size_t num_elements) const { using IN_T = fundamental_type_for; - const auto element_number = shape_size(m_shape); + const auto element_number = std::min(num_elements, shape_size(m_shape)); const auto source_begin = get_data_ptr(); const auto source_end = std::next(source_begin, (element_number + 1) / 2); const auto round_element_no = element_number % 2 ? element_number + 1 : element_number; @@ -522,9 +532,9 @@ class OPENVINO_API Constant : public Op { template ::type = true> - void cast_vector(std::vector& output) const { + void cast_vector(std::vector& output, size_t num_elements) const { using IN_T = fundamental_type_for; - const auto element_number = shape_size(m_shape); + const auto element_number = std::min(num_elements, shape_size(m_shape)); const auto source_begin = get_data_ptr(); const auto source_end = std::next(source_begin, (element_number + 1) / 2); const auto round_element_no = element_number % 2 ? element_number + 1 : element_number; diff --git a/src/core/include/openvino/op/loop.hpp b/src/core/include/openvino/op/loop.hpp index cb174d588b7bc3..c1ad30e5308e4a 100644 --- a/src/core/include/openvino/op/loop.hpp +++ b/src/core/include/openvino/op/loop.hpp @@ -10,6 +10,7 @@ #include "openvino/op/constant.hpp" #include "openvino/op/parameter.hpp" #include "openvino/op/util/sub_graph_base.hpp" +#include "openvino/runtime/tensor.hpp" namespace ov { namespace op { @@ -61,9 +62,7 @@ class OPENVINO_API Loop : public op::util::SubGraphOp { bool visit_attributes(AttributeVisitor& visitor) override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - OPENVINO_SUPPRESS_DEPRECATED_START - bool evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const override; - OPENVINO_SUPPRESS_DEPRECATED_END + bool evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const override; bool has_evaluate() const override; protected: diff --git a/src/core/include/openvino/op/util/multi_subgraph_base.hpp b/src/core/include/openvino/op/util/multi_subgraph_base.hpp index dbdb7b7d724b67..d4ee8eae9ca536 100644 --- a/src/core/include/openvino/op/util/multi_subgraph_base.hpp +++ b/src/core/include/openvino/op/util/multi_subgraph_base.hpp @@ -212,7 +212,7 @@ class OPENVINO_API MultiSubGraphOp : public Op { virtual void set_function(int index, const std::shared_ptr& func) { m_bodies[index] = func; } - /// \brief Gets vector with connections beewtwen operation inputs + /// \brief Gets vector with connections between operation inputs /// and internal sub-graph parameters /// /// \param index index of internal sub-graph @@ -220,7 +220,7 @@ class OPENVINO_API MultiSubGraphOp : public Op { const MultiSubgraphInputDescriptionVector& get_input_descriptions(int index) const { return m_input_descriptions[index]; } - /// \brief Gets vector with connections beewtwen operation inputs + /// \brief Gets vector with connections between operation inputs /// and internal sub-graph parameters /// /// \param index index of internal sub-graph @@ -228,7 +228,7 @@ class OPENVINO_API MultiSubGraphOp : public Op { MultiSubgraphInputDescriptionVector& get_input_descriptions(int index) { return m_input_descriptions[index]; } - /// \brief Gets vector with connections beewtwen operation outputs + /// \brief Gets vector with connections between operation outputs /// and internal sub-graph results /// /// \param index index of internal sub-graph @@ -236,7 +236,7 @@ class OPENVINO_API MultiSubGraphOp : public Op { const MultiSubgraphOutputDescriptionVector& get_output_descriptions(int index) const { return m_output_descriptions[index]; } - /// \brief Gets vector with connections beewtwen operation outputs + /// \brief Gets vector with connections between operation outputs /// and internal sub-graph results /// /// \param index index of internal sub-graph @@ -244,7 +244,7 @@ class OPENVINO_API MultiSubGraphOp : public Op { MultiSubgraphOutputDescriptionVector& get_output_descriptions(int index) { return m_output_descriptions[index]; } - /// \brief Sets vector with connections beewtwen operation inputs + /// \brief Sets vector with connections between operation inputs /// and internal sub-graph parameters /// /// \param index index of internal sub-graph @@ -253,7 +253,7 @@ class OPENVINO_API MultiSubGraphOp : public Op { m_input_descriptions[index] = inputs; } - /// \brief Sets vector with connections beewtwen operation outputs + /// \brief Sets vector with connections between operation outputs /// and internal sub-graph results /// /// \param index index of internal sub-graph diff --git a/src/core/include/openvino/op/util/symbolic_info.hpp b/src/core/include/openvino/op/util/symbolic_info.hpp new file mode 100644 index 00000000000000..fe58c4413d2baf --- /dev/null +++ b/src/core/include/openvino/op/util/symbolic_info.hpp @@ -0,0 +1,54 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/core/node.hpp" +#include "openvino/core/runtime_attribute.hpp" +#include "openvino/core/visibility.hpp" + +namespace ov { + +OPENVINO_API void set_up_symbolic_info(const std::shared_ptr& model, + const std::shared_ptr& table); +OPENVINO_API void set_up_symbolic_info(const ov::Output& output, + const std::shared_ptr& table); + +OPENVINO_API void populate_tensor_with_missing_labels(ov::descriptor::Tensor& tensor); + +OPENVINO_API bool skip_invalidation(const ov::descriptor::Tensor& tensor); +OPENVINO_API std::shared_ptr table_of_equivalence(const std::shared_ptr& model); +OPENVINO_API std::shared_ptr table_of_equivalence(const ov::descriptor::Tensor& tensor); + +OPENVINO_API void remove_symbolic_info(const std::shared_ptr& model, bool outermost_model = true); + +/** + * @ingroup ie_runtime_attr_api + * @brief SymbolicInfo class represents runtime info attribute that instructs ov::Output objects to skip invalidation of + * partial values and labels during partial value propagation and keeps shared_ptr to TableOfEquivalence. + */ +class OPENVINO_API SymbolicInfo : public RuntimeAttribute { +public: + OPENVINO_RTTI("SymbolicInfo", "0"); + explicit SymbolicInfo(bool skip_invalidation, const std::shared_ptr& table) + : m_skip_invalidation{skip_invalidation}, + m_table{table} {}; + bool is_copyable() const override { + return false; + } + bool get_skip_invalidation() const { + return m_skip_invalidation; + } + std::shared_ptr get_table() const { + return m_table; + } + +private: + bool m_skip_invalidation; + std::shared_ptr m_table; +}; + +} // namespace ov diff --git a/src/core/reference/CMakeLists.txt b/src/core/reference/CMakeLists.txt index dbd126d84a4bf0..0a6c16313d7ccb 100644 --- a/src/core/reference/CMakeLists.txt +++ b/src/core/reference/CMakeLists.txt @@ -35,6 +35,7 @@ endif() target_include_directories(${TARGET_NAME} PUBLIC $ + $ $) target_include_directories(${TARGET_NAME} SYSTEM PRIVATE diff --git a/src/core/reference/include/openvino/reference/abs.hpp b/src/core/reference/include/openvino/reference/abs.hpp index c58e70c5b4de02..426634a63a2d34 100644 --- a/src/core/reference/include/openvino/reference/abs.hpp +++ b/src/core/reference/include/openvino/reference/abs.hpp @@ -4,6 +4,7 @@ #pragma once +#include #include #include diff --git a/src/core/reference/include/openvino/reference/adaptive_max_pool.hpp b/src/core/reference/include/openvino/reference/adaptive_max_pool.hpp index 9aa0c3afa2af83..69c8ef2c940834 100644 --- a/src/core/reference/include/openvino/reference/adaptive_max_pool.hpp +++ b/src/core/reference/include/openvino/reference/adaptive_max_pool.hpp @@ -10,6 +10,7 @@ #include "ngraph/axis_vector.hpp" #include "ngraph/shape.hpp" +#include "openvino/reference/adaptive_avg_pool.hpp" namespace ov { namespace reference { diff --git a/src/core/reference/include/openvino/reference/convert_color_nv12.hpp b/src/core/reference/include/openvino/reference/convert_color_nv12.hpp index 4e8db24eb29668..a42aff6184c17f 100644 --- a/src/core/reference/include/openvino/reference/convert_color_nv12.hpp +++ b/src/core/reference/include/openvino/reference/convert_color_nv12.hpp @@ -107,12 +107,12 @@ void color_convert_i420(const T* arg_y, } } -OPENVINO_SUPPRESS_DEPRECATED_START -template +template inline bool color_convert_nv12(const std::shared_ptr& op, - const ov::HostTensorVector& outputs, - const ov::HostTensorVector& inputs, + ov::TensorVector& outputs, + const ov::TensorVector& inputs, ov::op::util::ConvertColorNV12Base::ColorConversion type) { + using ET = typename ov::element_type_traits::value_type; static const size_t N_DIM = 0; static const size_t H_DIM = 1; static const size_t W_DIM = 2; @@ -122,17 +122,17 @@ inline bool color_convert_nv12(const std::shared_ptr& op, auto single_plane = op->get_input_size() == 1; const auto& y_tensor = inputs[0]; - auto batch_size = y_tensor->get_shape()[N_DIM]; - auto image_w = y_tensor->get_shape()[W_DIM]; - auto image_h = y_tensor->get_shape()[H_DIM]; + auto batch_size = y_tensor.get_shape()[N_DIM]; + auto image_w = y_tensor.get_shape()[W_DIM]; + auto image_h = y_tensor.get_shape()[H_DIM]; if (single_plane) { image_h = image_h * 2 / 3; } - outputs[0]->set_shape({batch_size, image_h, image_w, 3}); // 3 is RGB + outputs[0].set_shape({batch_size, image_h, image_w, 3}); // 3 is RGB if (single_plane) { - color_convert_nv12(y_tensor->get_data_ptr(), - y_tensor->get_data_ptr() + image_w * image_h, - outputs[0]->get_data_ptr(), + color_convert_nv12(y_tensor.data(), + y_tensor.data() + image_w * image_h, + outputs[0].data(), batch_size, image_h, image_w, @@ -141,9 +141,9 @@ inline bool color_convert_nv12(const std::shared_ptr& op, type); } else { const auto& uv_tensor = inputs[1]; - color_convert_nv12(y_tensor->get_data_ptr(), - uv_tensor->get_data_ptr(), - outputs[0]->get_data_ptr(), + color_convert_nv12(y_tensor.data(), + uv_tensor.data(), + outputs[0].data(), batch_size, image_h, image_w, @@ -154,11 +154,12 @@ inline bool color_convert_nv12(const std::shared_ptr& op, return true; } -template +template inline bool color_convert_i420(const std::shared_ptr& op, - const ov::HostTensorVector& outputs, - const ov::HostTensorVector& inputs, + ov::TensorVector& outputs, + const ov::TensorVector& inputs, ov::op::util::ConvertColorI420Base::ColorConversion type) { + using ET = typename ov::element_type_traits::value_type; static const size_t N_DIM = 0; static const size_t H_DIM = 1; static const size_t W_DIM = 2; @@ -168,18 +169,18 @@ inline bool color_convert_i420(const std::shared_ptr& op, auto single_plane = op->get_input_size() == 1; const auto& y_tensor = inputs[0]; - auto batch_size = y_tensor->get_shape()[N_DIM]; - auto image_w = y_tensor->get_shape()[W_DIM]; - auto image_h = y_tensor->get_shape()[H_DIM]; + auto batch_size = y_tensor.get_shape()[N_DIM]; + auto image_w = y_tensor.get_shape()[W_DIM]; + auto image_h = y_tensor.get_shape()[H_DIM]; if (single_plane) { image_h = image_h * 2 / 3; } - outputs[0]->set_shape({batch_size, image_h, image_w, 3}); // 3 is RGB + outputs[0].set_shape({batch_size, image_h, image_w, 3}); // 3 is RGB if (single_plane) { - color_convert_i420(y_tensor->get_data_ptr(), - y_tensor->get_data_ptr() + image_w * image_h, - y_tensor->get_data_ptr() + 5 * image_w * image_h / 4, - outputs[0]->get_data_ptr(), + color_convert_i420(y_tensor.data(), + y_tensor.data() + image_w * image_h, + y_tensor.data() + 5 * image_w * image_h / 4, + outputs[0].data(), batch_size, image_h, image_w, @@ -189,10 +190,10 @@ inline bool color_convert_i420(const std::shared_ptr& op, } else { const auto& u_tensor = inputs[1]; const auto& v_tensor = inputs[2]; - color_convert_i420(y_tensor->get_data_ptr(), - u_tensor->get_data_ptr(), - v_tensor->get_data_ptr(), - outputs[0]->get_data_ptr(), + color_convert_i420(y_tensor.data(), + u_tensor.data(), + v_tensor.data(), + outputs[0].data(), batch_size, image_h, image_w, @@ -203,6 +204,5 @@ inline bool color_convert_i420(const std::shared_ptr& op, return true; } -OPENVINO_SUPPRESS_DEPRECATED_END } // namespace reference } // namespace ov diff --git a/src/core/reference/include/openvino/reference/ctc_loss.hpp b/src/core/reference/include/openvino/reference/ctc_loss.hpp index 03fae356ad69a7..1b16b352eb4870 100644 --- a/src/core/reference/include/openvino/reference/ctc_loss.hpp +++ b/src/core/reference/include/openvino/reference/ctc_loss.hpp @@ -6,6 +6,8 @@ #include +#include + #include "ngraph/shape_util.hpp" namespace ov { diff --git a/src/core/reference/include/openvino/reference/einsum.hpp b/src/core/reference/include/openvino/reference/einsum.hpp index 667a923739ce9c..8e477959bcba37 100644 --- a/src/core/reference/include/openvino/reference/einsum.hpp +++ b/src/core/reference/include/openvino/reference/einsum.hpp @@ -11,6 +11,6 @@ namespace ov { namespace reference { -void einsum(const HostTensorVector& outputs, const HostTensorVector& inputs, const std::string& equation); +void einsum(ov::TensorVector& outputs, const ov::TensorVector& inputs, const std::string& equation); } // namespace reference } // namespace ov diff --git a/src/core/reference/include/openvino/reference/fft.hpp b/src/core/reference/include/openvino/reference/fft.hpp index f35ef8ce79b981..474da3734157ed 100644 --- a/src/core/reference/include/openvino/reference/fft.hpp +++ b/src/core/reference/include/openvino/reference/fft.hpp @@ -23,13 +23,9 @@ #include #include #include -#include #include -#include "ngraph/node.hpp" -#include "ngraph/op/util/op_types.hpp" -#include "ngraph/ops.hpp" -#include "ngraph/shape_util.hpp" +#include "openvino/runtime/tensor.hpp" namespace ov { namespace reference { @@ -43,8 +39,8 @@ void fft(const float* input_data, const Shape& output_shape, FFTKind fft_kind); -void fft_postprocessing(const HostTensorVector& outputs, - const ngraph::element::Type output_type, +void fft_postprocessing(ov::TensorVector& outputs, + const ov::element::Type output_type, const std::vector& fft_result); std::vector canonicalize_axes(const int64_t* axes_data, diff --git a/src/core/reference/include/openvino/reference/function.hpp b/src/core/reference/include/openvino/reference/function.hpp index 575821e81b4a0f..24c10da48b40b8 100644 --- a/src/core/reference/include/openvino/reference/function.hpp +++ b/src/core/reference/include/openvino/reference/function.hpp @@ -12,6 +12,6 @@ namespace ov { namespace reference { -void function(const std::shared_ptr& function, const HostTensorVector& inputs, HostTensorVector& outputs); +void function(const std::shared_ptr& function, const ov::TensorVector& inputs, ov::TensorVector& outputs); } } // namespace ov diff --git a/src/core/reference/include/openvino/reference/hard_sigmoid.hpp b/src/core/reference/include/openvino/reference/hard_sigmoid.hpp index 4dd92870791bdb..dd2cb98df1da4c 100644 --- a/src/core/reference/include/openvino/reference/hard_sigmoid.hpp +++ b/src/core/reference/include/openvino/reference/hard_sigmoid.hpp @@ -4,6 +4,7 @@ #pragma once +#include #include #include diff --git a/src/core/reference/include/openvino/reference/if.hpp b/src/core/reference/include/openvino/reference/if.hpp index aaa47d0624ba29..27fc71ed393027 100644 --- a/src/core/reference/include/openvino/reference/if.hpp +++ b/src/core/reference/include/openvino/reference/if.hpp @@ -13,7 +13,7 @@ namespace reference { void if_reference(const std::vector>& body, const std::vector& out_descs, const std::vector& input_descs, - const HostTensorVector& out, - const HostTensorVector& args); + ov::TensorVector& out, + const ov::TensorVector& args); } } // namespace ov diff --git a/src/core/reference/include/openvino/reference/is_finite.hpp b/src/core/reference/include/openvino/reference/is_finite.hpp index 21677d0f9f9066..302da8485d3c83 100644 --- a/src/core/reference/include/openvino/reference/is_finite.hpp +++ b/src/core/reference/include/openvino/reference/is_finite.hpp @@ -4,6 +4,7 @@ #pragma once +#include #include #include diff --git a/src/core/reference/include/openvino/reference/is_inf.hpp b/src/core/reference/include/openvino/reference/is_inf.hpp index 156dafcfe6e9b0..210248ff2fcd2b 100644 --- a/src/core/reference/include/openvino/reference/is_inf.hpp +++ b/src/core/reference/include/openvino/reference/is_inf.hpp @@ -4,9 +4,12 @@ #pragma once +#include #include #include +#include "openvino/op/is_inf.hpp" + namespace ov { namespace reference { template diff --git a/src/core/reference/include/openvino/reference/is_nan.hpp b/src/core/reference/include/openvino/reference/is_nan.hpp index 47eed0e9944800..7b96e3abb08a79 100644 --- a/src/core/reference/include/openvino/reference/is_nan.hpp +++ b/src/core/reference/include/openvino/reference/is_nan.hpp @@ -4,6 +4,7 @@ #pragma once +#include #include #include diff --git a/src/core/reference/include/openvino/reference/loop.hpp b/src/core/reference/include/openvino/reference/loop.hpp index f233aaf0089576..19e16c2067d564 100644 --- a/src/core/reference/include/openvino/reference/loop.hpp +++ b/src/core/reference/include/openvino/reference/loop.hpp @@ -15,7 +15,7 @@ void loop(const std::shared_ptr& body, const op::util::OutputDescriptionVector& out_descs, const op::util::InputDescriptionVector& input_descs, const op::v5::Loop::SpecialBodyPorts& special_ports, - const HostTensorVector& out, - const HostTensorVector& args); + ov::TensorVector& out, + const ov::TensorVector& args); } } // namespace ov diff --git a/src/core/reference/include/openvino/reference/lstm_cell.hpp b/src/core/reference/include/openvino/reference/lstm_cell.hpp index b82eef98ba95d6..1a21f21cb3e497 100644 --- a/src/core/reference/include/openvino/reference/lstm_cell.hpp +++ b/src/core/reference/include/openvino/reference/lstm_cell.hpp @@ -6,6 +6,7 @@ #include +#include "openvino/op/lstm_cell.hpp" #include "openvino/reference/add.hpp" #include "openvino/reference/clamp.hpp" #include "openvino/reference/matmul.hpp" diff --git a/src/core/reference/include/openvino/reference/non_max_suppression.hpp b/src/core/reference/include/openvino/reference/non_max_suppression.hpp index ca4d9ef49c2955..b9e37e28c6a365 100644 --- a/src/core/reference/include/openvino/reference/non_max_suppression.hpp +++ b/src/core/reference/include/openvino/reference/non_max_suppression.hpp @@ -36,7 +36,7 @@ void non_max_suppression5(const float* boxes_data, int64_t* valid_outputs, const bool sort_result_descending); -void nms5_postprocessing(const HostTensorVector& outputs, +void nms5_postprocessing(ov::TensorVector& outputs, const ngraph::element::Type output_type, const std::vector& selected_indices, const std::vector& selected_scores, @@ -58,7 +58,7 @@ void non_max_suppression(const float* boxes_data, int64_t* valid_outputs, const bool sort_result_descending); -void nms_postprocessing(const HostTensorVector& outputs, +void nms_postprocessing(ov::TensorVector& outputs, const ngraph::element::Type output_type, const std::vector& selected_indices, const std::vector& selected_scores, diff --git a/src/core/reference/include/openvino/reference/roi_pooling.hpp b/src/core/reference/include/openvino/reference/roi_pooling.hpp index 7efa4daf38467d..5dbe13d1de51b6 100644 --- a/src/core/reference/include/openvino/reference/roi_pooling.hpp +++ b/src/core/reference/include/openvino/reference/roi_pooling.hpp @@ -4,6 +4,9 @@ #pragma once +#include +#include + #include "ngraph/shape.hpp" namespace ov { diff --git a/src/core/reference/include/openvino/reference/tensor_iterator.hpp b/src/core/reference/include/openvino/reference/tensor_iterator.hpp index f4ff4d8919ee76..05a66f2e640eb3 100644 --- a/src/core/reference/include/openvino/reference/tensor_iterator.hpp +++ b/src/core/reference/include/openvino/reference/tensor_iterator.hpp @@ -5,18 +5,21 @@ #pragma once #include -#include + +#include "openvino/core/model.hpp" +#include "openvino/op/util/sub_graph_base.hpp" +#include "openvino/runtime/tensor.hpp" namespace ov { namespace reference { using custom_evaluate_function = std::function< - void(const std::shared_ptr& function, const HostTensorVector& inputs, HostTensorVector& outputs)>; + void(const std::shared_ptr& function, const ov::TensorVector& inputs, ov::TensorVector& outputs)>; void tensor_iterator(uint64_t num_iterations, const std::shared_ptr& body, const op::util::OutputDescriptionVector& out_descs, const op::util::InputDescriptionVector& input_descs, - const HostTensorVector& out, - const HostTensorVector& args, + ov::TensorVector& out, + const ov::TensorVector& args, const custom_evaluate_function& evaluate = nullptr); } // namespace reference } // namespace ov diff --git a/src/core/reference/src/op/einsum.cpp b/src/core/reference/src/op/einsum.cpp index 1901073f184a8a..125e33e3aa6b49 100644 --- a/src/core/reference/src/op/einsum.cpp +++ b/src/core/reference/src/op/einsum.cpp @@ -5,9 +5,8 @@ #include "openvino/reference/einsum.hpp" #include -#include -#include "ngraph/shape.hpp" +#include "openvino/op/einsum.hpp" #include "openvino/reference/broadcast.hpp" #include "openvino/reference/matmul.hpp" #include "openvino/reference/multiply.hpp" @@ -16,7 +15,6 @@ #include "openvino/reference/transpose.hpp" #include "openvino/reference/utils/span.hpp" -NGRAPH_SUPPRESS_DEPRECATED_START namespace ov { namespace reference { namespace { @@ -27,7 +25,7 @@ namespace { std::vector> compute_einsum_path(size_t num_inputs) { // TODO: implement algorithm for finding (pseudo-)optimal einsum_path std::vector> einsum_path; - NGRAPH_CHECK(num_inputs > 0); + OPENVINO_ASSERT(num_inputs > 0); for (size_t input_ind = num_inputs - 1; input_ind > 0; --input_ind) { einsum_path.push_back(std::make_pair(0, input_ind)); } @@ -96,7 +94,7 @@ std::string generate_grouping_subscript(const std::string& input_subscript, return input_subscript; } - auto labels = ngraph::opset7::Einsum::extract_labels(input_subscript); + auto labels = ov::op::v7::Einsum::extract_labels(input_subscript); std::string required_subscript = ""; for (auto index : labels_inds) { required_subscript += labels[index]; @@ -108,15 +106,15 @@ std::string generate_grouping_subscript(const std::string& input_subscript, std::unordered_map compute_label_dim_map(const Rank& input_rank, const std::string& input_subscript) { constexpr char ellipsis[] = "..."; - auto labels = ngraph::opset7::Einsum::extract_labels(input_subscript); + auto labels = ov::op::v7::Einsum::extract_labels(input_subscript); size_t input_rank_length = labels.size(); - NGRAPH_CHECK(input_rank.is_static() || (std::find(labels.begin(), labels.end(), ellipsis) == labels.end()), - "Input rank cannot be dynamic in case of ellipsis in input subscript"); + OPENVINO_ASSERT(input_rank.is_static() || (std::find(labels.begin(), labels.end(), ellipsis) == labels.end()), + "Input rank cannot be dynamic in case of ellipsis in input subscript"); if (input_rank.is_static()) { input_rank_length = input_rank.get_length(); } std::unordered_map resulted_map; - NGRAPH_CHECK(input_rank_length >= labels.size()); + OPENVINO_ASSERT(input_rank_length >= labels.size()); size_t num_broadcasted_dims = input_rank_length - labels.size() + 1; size_t current_dim = 0; @@ -146,7 +144,7 @@ std::unordered_map compute_label_dim_map(const Ran /// range [s_begin;s_end) /// Shape compute_sub_shape(const Shape& input_shape, size_t begin, size_t end, bool is_product = false) { - NGRAPH_CHECK(end <= input_shape.size()); + OPENVINO_ASSERT(end <= input_shape.size()); if (end <= begin) { // return empty shape return Shape(); @@ -179,17 +177,17 @@ void compute_ranges(const Rank& input_rank, constexpr char ellipsis[] = "..."; size_t common_rank = common_labels.size(); if (std::find(common_labels.begin(), common_labels.end(), ellipsis) != common_labels.end()) { - NGRAPH_CHECK(label_to_dim_map.find(ellipsis) != label_to_dim_map.end()); + OPENVINO_ASSERT(label_to_dim_map.find(ellipsis) != label_to_dim_map.end()); common_rank += label_to_dim_map[ellipsis].size() - 1; } size_t sep_rank = sep_labels.size(); if (std::find(sep_labels.begin(), sep_labels.end(), ellipsis) != sep_labels.end()) { - NGRAPH_CHECK(label_to_dim_map.find(ellipsis) != label_to_dim_map.end()); + OPENVINO_ASSERT(label_to_dim_map.find(ellipsis) != label_to_dim_map.end()); sep_rank += label_to_dim_map[ellipsis].size() - 1; } size_t reduced_rank = reduced_labels.size(); if (std::find(reduced_labels.begin(), reduced_labels.end(), ellipsis) != reduced_labels.end()) { - NGRAPH_CHECK(label_to_dim_map.find(ellipsis) != label_to_dim_map.end()); + OPENVINO_ASSERT(label_to_dim_map.find(ellipsis) != label_to_dim_map.end()); reduced_rank += label_to_dim_map[ellipsis].size() - 1; } @@ -255,15 +253,15 @@ Shape compute_matmul_output_shape(const Shape& common_sub_shape, /// inputs with indices input_ind1 and input_ind2 and inserted new input and /// the corresponsing subscript in the tail /// -void update_operands(HostTensorVector& inputs, +void update_operands(ov::TensorVector& inputs, std::vector& input_subscripts, size_t input_ind1, size_t input_ind2, - const HostTensorPtr& new_input, + const ov::Tensor& new_input, const std::string& new_subscript) { - NGRAPH_CHECK(input_ind1 < input_ind2); - NGRAPH_CHECK(input_ind2 < inputs.size()); - NGRAPH_CHECK(input_ind2 < input_subscripts.size()); + OPENVINO_ASSERT(input_ind1 < input_ind2); + OPENVINO_ASSERT(input_ind2 < inputs.size()); + OPENVINO_ASSERT(input_ind2 < input_subscripts.size()); inputs.erase(inputs.begin() + input_ind2); inputs.erase(inputs.begin() + input_ind1); inputs.push_back(new_input); @@ -275,26 +273,28 @@ void update_operands(HostTensorVector& inputs, /// \brief Unsqueeze input by given dimensions if a vector of unsqueezing /// dimensions is not empty template -HostTensorPtr unsqueeze_input(const HostTensorPtr& input, std::vector& unsqueeze_axes) { +ov::Tensor unsqueeze_input(const ov::Tensor& input, std::vector& unsqueeze_axes) { if (unsqueeze_axes.empty()) { return input; } - Shape input_shape = input->get_shape(); + Shape input_shape = input.get_shape(); Shape output_shape = input_shape; std::sort(unsqueeze_axes.begin(), unsqueeze_axes.end()); for (auto unsqueeze_axis : unsqueeze_axes) { - NGRAPH_CHECK(unsqueeze_axis >= 0); - NGRAPH_CHECK(static_cast(unsqueeze_axis) <= output_shape.size()); + OPENVINO_ASSERT(unsqueeze_axis >= 0); + OPENVINO_ASSERT(static_cast(unsqueeze_axis) <= output_shape.size()); output_shape.insert(output_shape.begin() + unsqueeze_axis, 1); } - HostTensorPtr output = std::shared_ptr(new HostTensor(input->get_element_type(), output_shape)); - const AxisVector order = ngraph::get_default_order(input->get_shape()); - const auto element_type = input->get_element_type(); + auto output = ov::Tensor(input.get_element_type(), output_shape); + OPENVINO_SUPPRESS_DEPRECATED_START + const AxisVector order = ngraph::get_default_order(input.get_shape()); + OPENVINO_SUPPRESS_DEPRECATED_END + const auto element_type = input.get_element_type(); - reference::reshape(reinterpret_cast(input->get_data_ptr()), - reinterpret_cast(output->get_data_ptr()), + reference::reshape(reinterpret_cast(input.data()), + reinterpret_cast(output.data()), input_shape, order, output_shape, @@ -307,31 +307,31 @@ HostTensorPtr unsqueeze_input(const HostTensorPtr& input, std::vector& /// equation and reduce dimensions corresponding to such labels /// template -void reduce_input(HostTensorVector& inputs, +void reduce_input(ov::TensorVector& inputs, std::vector& input_subscripts, const std::string& output_subscript, size_t input_ind) { // perform sanity check for arguments auto num_inputs = inputs.size(); - NGRAPH_CHECK(num_inputs == input_subscripts.size(), "Each input must have own subscript."); - NGRAPH_CHECK(input_ind < num_inputs, "Input index is out of range."); + OPENVINO_ASSERT(num_inputs == input_subscripts.size(), "Each input must have own subscript."); + OPENVINO_ASSERT(input_ind < num_inputs, "Input index is out of range."); const auto& input_ptr = inputs[input_ind]; const auto& input_subscript = input_subscripts[input_ind]; - const auto input_shape = input_ptr->get_shape(); + const auto input_shape = input_ptr.get_shape(); // compute output shape and axes to reduce - ngraph::Shape output_shape; - ngraph::AxisSet reduced_axes; - auto labels = ngraph::opset7::Einsum::extract_labels(input_subscripts[input_ind]); - auto label_dim_map = compute_label_dim_map(input_ptr->get_partial_shape().rank(), input_subscript); + ov::Shape output_shape; + ov::AxisSet reduced_axes; + auto labels = ov::op::v7::Einsum::extract_labels(input_subscripts[input_ind]); + auto label_dim_map = compute_label_dim_map(input_ptr.get_shape().size(), input_subscript); std::string new_input_subscript = ""; for (const auto& label : labels) { // check if the current label is met in the other input subscripts // or the output subscript bool is_dim_reduced = is_dimension_reduced(input_subscripts, output_subscript, label, {input_ind}); - NGRAPH_CHECK(label_dim_map.find(label) != label_dim_map.end()); + OPENVINO_ASSERT(label_dim_map.find(label) != label_dim_map.end()); auto label_dims = label_dim_map[label]; // if label is not met, dimension corresponding to the label is to reduce @@ -350,9 +350,9 @@ void reduce_input(HostTensorVector& inputs, return; } - HostTensorPtr output_ptr = std::shared_ptr(new HostTensor(input_ptr->get_element_type(), output_shape)); + auto output_ptr = ov::Tensor(input_ptr.get_element_type(), output_shape); - reference::sum(input_ptr->get_data_ptr(), output_ptr->get_data_ptr(), input_shape, reduced_axes); + reference::sum(input_ptr.data(), output_ptr.data(), input_shape, reduced_axes); // update a vector of inputs and input subscripts inputs[input_ind] = output_ptr; @@ -362,14 +362,14 @@ void reduce_input(HostTensorVector& inputs, /// \brief Transpose input to layout specified through the required subscript /// template -void transpose_input(HostTensorVector& inputs, +void transpose_input(ov::TensorVector& inputs, std::vector& input_subscripts, const std::string& required_subscript, size_t input_ind) { // perform sanity check for arguments auto num_inputs = inputs.size(); - NGRAPH_CHECK(num_inputs == input_subscripts.size(), "Each input must have own subscript."); - NGRAPH_CHECK(input_ind < num_inputs, "Input index is out of range."); + OPENVINO_ASSERT(num_inputs == input_subscripts.size(), "Each input must have own subscript."); + OPENVINO_ASSERT(input_ind < num_inputs, "Input index is out of range."); // generate permutation vector by searching for bijection between // input_subscripts and required_subscript @@ -384,29 +384,29 @@ void transpose_input(HostTensorVector& inputs, // find permutation that establishes bijection between the input subscript // and the required one - auto label_dim_map = compute_label_dim_map(input_ptr->get_partial_shape().rank(), input_subscript); - auto labels = ngraph::opset7::Einsum::extract_labels(input_subscript); - auto required_labels = ngraph::opset7::Einsum::extract_labels(required_subscript); - NGRAPH_CHECK(labels.size() == required_labels.size()); + auto label_dim_map = compute_label_dim_map(input_ptr.get_shape().size(), input_subscript); + auto labels = ov::op::v7::Einsum::extract_labels(input_subscript); + auto required_labels = ov::op::v7::Einsum::extract_labels(required_subscript); + OPENVINO_ASSERT(labels.size() == required_labels.size()); for (const auto& required_label : required_labels) { - NGRAPH_CHECK(label_dim_map.find(required_label) != label_dim_map.end()); + OPENVINO_ASSERT(label_dim_map.find(required_label) != label_dim_map.end()); auto label_dims = label_dim_map[required_label]; permutation.insert(permutation.end(), label_dims.begin(), label_dims.end()); } - const auto input_shape = input_ptr->get_shape(); - const auto element_type = input_ptr->get_element_type(); + const auto input_shape = input_ptr.get_shape(); + const auto element_type = input_ptr.get_element_type(); Shape output_shape(input_shape.size()); std::transform(permutation.begin(), permutation.end(), output_shape.begin(), [&](const int64_t& v) { - NGRAPH_CHECK(v >= 0, "Negative values for transpose axes order are not supported."); - NGRAPH_CHECK(v < int64_t(input_shape.size()), "Transpose axis ", v, " is out of shape range."); + OPENVINO_ASSERT(v >= 0, "Negative values for transpose axes order are not supported."); + OPENVINO_ASSERT(v < int64_t(input_shape.size()), "Transpose axis ", v, " is out of shape range."); return input_shape[v]; }); - HostTensorPtr output_ptr = std::shared_ptr(new HostTensor(element_type, output_shape)); + auto output_ptr = ov::Tensor(element_type, output_shape); - reference::transpose(reinterpret_cast(input_ptr->get_data_ptr()), - reinterpret_cast(output_ptr->get_data_ptr()), + reference::transpose(reinterpret_cast(input_ptr.data()), + reinterpret_cast(output_ptr.data()), input_shape, element_type.size(), permutation.data(), @@ -421,15 +421,15 @@ void transpose_input(HostTensorVector& inputs, /// same shape of both operands in the common (or batch) dimensionsy. /// template -void broadcast_input(HostTensorVector& inputs, +void broadcast_input(ov::TensorVector& inputs, size_t input_ind, const Shape& new_common_shape, const Shape& separate_shape, const Shape& reduced_shape, bool is_separate_first) { - NGRAPH_CHECK(input_ind < inputs.size()); - HostTensorPtr& input = inputs[input_ind]; - const Shape old_shape = input->get_shape(); + OPENVINO_ASSERT(input_ind < inputs.size()); + ov::Tensor& input = inputs[input_ind]; + const Shape old_shape = input.get_shape(); Shape new_shape; new_shape.insert(new_shape.end(), new_common_shape.begin(), new_common_shape.end()); if (is_separate_first) { @@ -440,22 +440,22 @@ void broadcast_input(HostTensorVector& inputs, new_shape.insert(new_shape.end(), separate_shape.begin(), separate_shape.end()); } - if (input->get_shape() == new_shape) { + if (input.get_shape() == new_shape) { return; } - NGRAPH_CHECK(old_shape.size() <= new_shape.size()); + OPENVINO_ASSERT(old_shape.size() <= new_shape.size()); - HostTensorPtr output = std::shared_ptr(new HostTensor(input->get_element_type(), new_shape)); + auto output = ov::Tensor(input.get_element_type(), new_shape); std::vector broadcast_axes(old_shape.size()); std::iota(broadcast_axes.begin(), broadcast_axes.end(), new_shape.size() - old_shape.size()); - reference::broadcast(reinterpret_cast(input->get_data_ptr()), - reinterpret_cast(output->get_data_ptr()), - input->get_shape(), - output->get_shape(), + reference::broadcast(reinterpret_cast(input.data()), + reinterpret_cast(output.data()), + input.get_shape(), + output.get_shape(), broadcast_axes, - input->get_element_type().size()); + input.get_element_type().size()); input = output; } @@ -465,21 +465,21 @@ void broadcast_input(HostTensorVector& inputs, /// identity /// template -HostTensorPtr build_identity(const HostTensorPtr& input_ptr, const ov::TensorLabel& repeated_label_dims) { +ov::Tensor build_identity(const ov::Tensor& input, const ov::TensorLabel& repeated_label_dims) { // allocate HostTensor for building identity tensor - NGRAPH_CHECK(repeated_label_dims.size() > 1); - Shape input_shape = input_ptr->get_shape(); + OPENVINO_ASSERT(repeated_label_dims.size() > 1); + Shape input_shape = input.get_shape(); Shape identity_shape(input_shape.size(), 1); size_t repeated_label_dim_size = input_shape[repeated_label_dims[0]]; for (auto dim : repeated_label_dims) { - NGRAPH_CHECK(dim < input_shape.size()); - NGRAPH_CHECK(repeated_label_dim_size == input_shape[dim]); + OPENVINO_ASSERT(dim < input_shape.size()); + OPENVINO_ASSERT(repeated_label_dim_size == input_shape[dim]); identity_shape[dim] = repeated_label_dim_size; } - HostTensorPtr identity = std::shared_ptr(new HostTensor(input_ptr->get_element_type(), identity_shape)); + auto identity = ov::Tensor(input.get_element_type(), identity_shape); - T* identity_data_ptr = identity->get_data_ptr(); - size_t data_size = shape_size(identity_shape) * identity->get_element_type().size(); + T* identity_data_ptr = identity.data(); + size_t data_size = shape_size(identity_shape) * identity.get_element_type().size(); std::memset(identity_data_ptr, 0, data_size); // Identity[k,k,...,k] element is placed in k*p^(n-1) + ... + k*p + k position, @@ -507,35 +507,32 @@ HostTensorPtr build_identity(const HostTensorPtr& input_ptr, const ov::TensorLab /// repeated label /// template -HostTensorPtr build_multi_identity(const HostTensorPtr& input_ptr, - const std::vector& repeated_labels, - std::unordered_map& label_dim_map) { - Shape input_shape = input_ptr->get_shape(); +ov::Tensor build_multi_identity(const ov::Tensor& input, + const std::vector& repeated_labels, + std::unordered_map& label_dim_map) { + Shape input_shape = input.get_shape(); // initially set multi-identity with identity for the first repeated label - NGRAPH_CHECK(repeated_labels.size() > 0); + OPENVINO_ASSERT(repeated_labels.size() > 0); auto first_repeated_label = repeated_labels[0]; - NGRAPH_CHECK(label_dim_map.find(first_repeated_label) != label_dim_map.end()); + OPENVINO_ASSERT(label_dim_map.find(first_repeated_label) != label_dim_map.end()); auto repeated_label_dims = label_dim_map[first_repeated_label]; - HostTensorPtr multi_identity = build_identity(input_ptr, repeated_label_dims); + ov::Tensor multi_identity = build_identity(input, repeated_label_dims); for (size_t label_ind = 1; label_ind < repeated_labels.size(); ++label_ind) { - NGRAPH_CHECK(label_dim_map.find(repeated_labels[label_ind]) != label_dim_map.end()); + OPENVINO_ASSERT(label_dim_map.find(repeated_labels[label_ind]) != label_dim_map.end()); repeated_label_dims = label_dim_map[repeated_labels[label_ind]]; - HostTensorPtr identity = build_identity(input_ptr, repeated_label_dims); - - PartialShape output_shape = multi_identity->get_partial_shape(); - PartialShape::broadcast_merge_into(output_shape, - identity->get_partial_shape(), - ngraph::op::AutoBroadcastType::NUMPY); - HostTensorPtr mul_output = - std::shared_ptr(new HostTensor(identity->get_element_type(), output_shape.get_shape())); - reference::multiply(multi_identity->get_data_ptr(), - identity->get_data_ptr(), - mul_output->get_data_ptr(), - multi_identity->get_shape(), - identity->get_shape(), - ngraph::op::AutoBroadcastType::NUMPY); + ov::Tensor identity = build_identity(input, repeated_label_dims); + + PartialShape output_shape = multi_identity.get_shape(); + PartialShape::broadcast_merge_into(output_shape, identity.get_shape(), ov::op::AutoBroadcastType::NUMPY); + auto mul_output = ov::Tensor(identity.get_element_type(), output_shape.get_shape()); + reference::multiply(multi_identity.data(), + identity.data(), + mul_output.data(), + multi_identity.get_shape(), + identity.get_shape(), + ov::op::AutoBroadcastType::NUMPY); multi_identity = mul_output; } return multi_identity; @@ -545,28 +542,28 @@ HostTensorPtr build_multi_identity(const HostTensorPtr& input_ptr, /// labels) is diagonal /// template -void extract_diagonal(HostTensorVector& inputs, std::vector& input_subscripts, size_t input_ind) { +void extract_diagonal(ov::TensorVector& inputs, std::vector& input_subscripts, size_t input_ind) { // perform sanity check for arguments auto num_inputs = inputs.size(); - NGRAPH_CHECK(num_inputs == input_subscripts.size(), "Each input must have own subscript."); - NGRAPH_CHECK(input_ind < num_inputs, "Input index is out of range."); + OPENVINO_ASSERT(num_inputs == input_subscripts.size(), "Each input must have own subscript."); + OPENVINO_ASSERT(input_ind < num_inputs, "Input index is out of range."); const auto& input_ptr = inputs[input_ind]; const auto& input_subscript = input_subscripts[input_ind]; - const auto input_shape = input_ptr->get_shape(); + const auto input_shape = input_ptr.get_shape(); std::string resultant_subscript = ""; constexpr char ellipsis[] = "..."; - auto labels = ngraph::opset7::Einsum::extract_labels(input_subscript); - auto label_dim_map = compute_label_dim_map(input_ptr->get_partial_shape().rank(), input_subscript); + auto labels = ov::op::v7::Einsum::extract_labels(input_subscript); + auto label_dim_map = compute_label_dim_map(input_ptr.get_shape().size(), input_subscript); std::vector repeated_labels; Shape result_shape; AxisSet reduced_axes; for (const auto& label : labels) { if (resultant_subscript.find(label) == std::string::npos) { - NGRAPH_CHECK(label_dim_map.find(label) != label_dim_map.end()); + OPENVINO_ASSERT(label_dim_map.find(label) != label_dim_map.end()); auto dims = label_dim_map[label]; - NGRAPH_CHECK(dims.size() > 0); + OPENVINO_ASSERT(dims.size() > 0); if (label != ellipsis && dims.size() > 1) { // repeated label is found for (size_t dim_ind = 1; dim_ind < dims.size(); ++dim_ind) { @@ -578,7 +575,7 @@ void extract_diagonal(HostTensorVector& inputs, std::vector& input_ } resultant_subscript += label; for (auto dim : dims) { - NGRAPH_CHECK(dim < input_shape.size()); + OPENVINO_ASSERT(dim < input_shape.size()); result_shape.push_back(input_shape[dim]); } } @@ -587,18 +584,18 @@ void extract_diagonal(HostTensorVector& inputs, std::vector& input_ return; } - HostTensorPtr multi_identity = build_multi_identity(input_ptr, repeated_labels, label_dim_map); + ov::Tensor multi_identity = build_multi_identity(input_ptr, repeated_labels, label_dim_map); - HostTensorPtr mul_output = input_ptr; - reference::multiply(input_ptr->get_data_ptr(), - multi_identity->get_data_ptr(), - mul_output->get_data_ptr(), - input_ptr->get_shape(), - multi_identity->get_shape(), - ngraph::op::AutoBroadcastType::NUMPY); + ov::Tensor mul_output = input_ptr; + reference::multiply(input_ptr.data(), + multi_identity.data(), + mul_output.data(), + input_ptr.get_shape(), + multi_identity.get_shape(), + ov::op::AutoBroadcastType::NUMPY); - HostTensorPtr result = std::shared_ptr(new HostTensor(input_ptr->get_element_type(), result_shape)); - reference::sum(mul_output->get_data_ptr(), result->get_data_ptr(), mul_output->get_shape(), reduced_axes); + auto result = ov::Tensor(input_ptr.get_element_type(), result_shape); + reference::sum(mul_output.data(), result.data(), mul_output.get_shape(), reduced_axes); inputs[input_ind] = result; input_subscripts[input_ind] = resultant_subscript; } @@ -608,11 +605,11 @@ void extract_diagonal(HostTensorVector& inputs, std::vector& input_ /// acceptable by MatMul /// template -HostTensorPtr reshape_input_for_matmul(const HostTensorPtr& input, - const Shape& common_sub_shape, - const Shape& separate_sub_shape, - const Shape& reduced_sub_shape_prod, - bool is_separate_first) { +ov::Tensor reshape_input_for_matmul(const ov::Tensor& input, + const Shape& common_sub_shape, + const Shape& separate_sub_shape, + const Shape& reduced_sub_shape_prod, + bool is_separate_first) { Shape new_shape; new_shape.insert(new_shape.end(), common_sub_shape.begin(), common_sub_shape.end()); @@ -643,17 +640,19 @@ HostTensorPtr reshape_input_for_matmul(const HostTensorPtr& input, // when new shape is equal to the current one, // there is no need in reshape - if (new_shape == input->get_shape()) { + if (new_shape == input.get_shape()) { return input; } - const auto element_type = input->get_element_type(); - const auto input_shape = input->get_shape(); - HostTensorPtr output = std::shared_ptr(new HostTensor(element_type, new_shape)); + const auto element_type = input.get_element_type(); + const auto input_shape = input.get_shape(); + auto output = ov::Tensor(element_type, new_shape); + OPENVINO_SUPPRESS_DEPRECATED_START const AxisVector order = ngraph::get_default_order(input_shape); + OPENVINO_SUPPRESS_DEPRECATED_END - reference::reshape(reinterpret_cast(input->get_data_ptr()), - reinterpret_cast(output->get_data_ptr()), + reference::reshape(reinterpret_cast(input.data()), + reinterpret_cast(output.data()), input_shape, order, new_shape, @@ -667,7 +666,7 @@ HostTensorPtr reshape_input_for_matmul(const HostTensorPtr& input, /// inputs along with their input subscripts /// template -void contract_two_inputs(HostTensorVector& inputs, +void contract_two_inputs(ov::TensorVector& inputs, std::vector& input_subscripts, const std::string& output_subscript, size_t input_ind1, @@ -680,8 +679,8 @@ void contract_two_inputs(HostTensorVector& inputs, // perform sanity check for arguments auto num_inputs = inputs.size(); - NGRAPH_CHECK(num_inputs == input_subscripts.size(), "Each input must have own subscript."); - NGRAPH_CHECK(input_ind2 < num_inputs && input_ind1 != input_ind2, "Incorrect input index is specified."); + OPENVINO_ASSERT(num_inputs == input_subscripts.size(), "Each input must have own subscript."); + OPENVINO_ASSERT(input_ind2 < num_inputs && input_ind1 != input_ind2, "Incorrect input index is specified."); const auto& input1 = inputs[input_ind1]; const auto& input2 = inputs[input_ind2]; @@ -705,9 +704,9 @@ void contract_two_inputs(HostTensorVector& inputs, // corresponding label are met in neither the output subscript nor the input // subscripts for other Einsum inputs excluding two given inputs auto& input_subscript1 = input_subscripts[input_ind1]; - auto labels1 = ngraph::opset7::Einsum::extract_labels(input_subscript1); + auto labels1 = ov::op::v7::Einsum::extract_labels(input_subscript1); auto& input_subscript2 = input_subscripts[input_ind2]; - auto labels2 = ngraph::opset7::Einsum::extract_labels(input_subscript2); + auto labels2 = ov::op::v7::Einsum::extract_labels(input_subscript2); std::string common_part = ""; std::string separate_part1 = ""; std::string separate_part2 = ""; @@ -756,27 +755,27 @@ void contract_two_inputs(HostTensorVector& inputs, // for further unsqueezing transpose_input(inputs, input_subscripts, convenient_subscript, input_ind2); - auto separate_labels1 = ngraph::opset7::Einsum::extract_labels(separate_part1); - auto separate_labels2 = ngraph::opset7::Einsum::extract_labels(separate_part2); - auto label_to_dim_map1 = compute_label_dim_map(input1->get_partial_shape().rank(), input_subscript1); - auto label_to_dim_map2 = compute_label_dim_map(input2->get_partial_shape().rank(), input_subscript2); + auto separate_labels1 = ov::op::v7::Einsum::extract_labels(separate_part1); + auto separate_labels2 = ov::op::v7::Einsum::extract_labels(separate_part2); + auto label_to_dim_map1 = compute_label_dim_map(input1.get_shape().size(), input_subscript1); + auto label_to_dim_map2 = compute_label_dim_map(input2.get_shape().size(), input_subscript2); // unsqueeze the first operand with new dimensions in the tail // and the number of them is equal to the number of separate labels in the // second subscript - int64_t input_rank1 = input1->get_shape().size(); + int64_t input_rank1 = input1.get_shape().size(); int64_t unsqueeze_dim = input_rank1; std::vector unsqueeze_axis1; std::vector unsqueeze_axis2; for (const auto& sep_label2 : separate_labels2) { - NGRAPH_CHECK(label_to_dim_map2.find(sep_label2) != label_to_dim_map2.end()); + OPENVINO_ASSERT(label_to_dim_map2.find(sep_label2) != label_to_dim_map2.end()); auto label_dims = label_to_dim_map2[sep_label2]; for (size_t dim_ind = 0; dim_ind < label_dims.size(); ++dim_ind) { unsqueeze_axis1.push_back(unsqueeze_dim + static_cast(dim_ind)); } } for (const auto& sep_label1 : separate_labels1) { - NGRAPH_CHECK(label_to_dim_map1.find(sep_label1) != label_to_dim_map1.end()); + OPENVINO_ASSERT(label_to_dim_map1.find(sep_label1) != label_to_dim_map1.end()); auto label_dims = label_to_dim_map1[sep_label1]; for (auto label_dim : label_dims) { unsqueeze_axis2.push_back(label_dim); @@ -784,22 +783,21 @@ void contract_two_inputs(HostTensorVector& inputs, } // unsqueeze input operands for elementwise-multiplication with broadcasting - HostTensorPtr unsqueeze_output1 = unsqueeze_input(input1, unsqueeze_axis1); - HostTensorPtr unsqueeze_output2 = unsqueeze_input(input2, unsqueeze_axis2); + ov::Tensor unsqueeze_output1 = unsqueeze_input(input1, unsqueeze_axis1); + ov::Tensor unsqueeze_output2 = unsqueeze_input(input2, unsqueeze_axis2); // multiply both operands with broadcasting - PartialShape output_shape = unsqueeze_output1->get_partial_shape(); + PartialShape output_shape = unsqueeze_output1.get_shape(); PartialShape::broadcast_merge_into(output_shape, - unsqueeze_output2->get_partial_shape(), - ngraph::op::AutoBroadcastType::NUMPY); - HostTensorPtr mul_output = std::shared_ptr( - new HostTensor(unsqueeze_output1->get_element_type(), output_shape.get_shape())); - reference::multiply(unsqueeze_output1->get_data_ptr(), - unsqueeze_output2->get_data_ptr(), - mul_output->get_data_ptr(), - unsqueeze_output1->get_shape(), - unsqueeze_output2->get_shape(), - ngraph::op::AutoBroadcastType::NUMPY); + unsqueeze_output2.get_shape(), + ov::op::AutoBroadcastType::NUMPY); + auto mul_output = ov::Tensor(unsqueeze_output1.get_element_type(), output_shape.get_shape()); + reference::multiply(unsqueeze_output1.data(), + unsqueeze_output2.data(), + mul_output.data(), + unsqueeze_output1.get_shape(), + unsqueeze_output2.get_shape(), + ov::op::AutoBroadcastType::NUMPY); // update input operand and input subscript for Einsum operation update_operands(inputs, input_subscripts, input_ind1, input_ind2, mul_output, resultant_subscript); @@ -830,11 +828,11 @@ void contract_two_inputs(HostTensorVector& inputs, // [B1, ..., Bm, X1, Y] or [B1, ..., Bm, Y, X2], where B1, ..., Bm are common // dimensions, X1 and X2 are collapsed dimensions for separate labels and Y is // collapsed dimension for reduced labels - Shape input_shape1 = input1->get_shape(); - Shape input_shape2 = input2->get_shape(); + Shape input_shape1 = input1.get_shape(); + Shape input_shape2 = input2.get_shape(); size_t common_dims_begin, common_dims_end, reduced_dims_begin, reduced_dims_end, separate1_dims_begin, separate1_dims_end; - compute_ranges(input1->get_partial_shape().rank(), + compute_ranges(input1.get_shape().size(), input_subscript1, common_labels, sep_labels1, @@ -849,7 +847,7 @@ void contract_two_inputs(HostTensorVector& inputs, size_t common_dims_begin2, common_dims_end2, reduced_dims_begin2, reduced_dims_end2, separate2_dims_begin, separate2_dims_end; - compute_ranges(input2->get_partial_shape().rank(), + compute_ranges(input2.get_shape().size(), input_subscript2, common_labels, sep_labels2, @@ -888,29 +886,28 @@ void contract_two_inputs(HostTensorVector& inputs, reduced_sub_shape, is_separate_first2); - HostTensorPtr matmul_operand1 = reshape_input_for_matmul(input1, - common_sub_shape, - separate1_sub_shape, - reduced_sub_shape_prod, - is_separate_first1); - HostTensorPtr matmul_operand2 = reshape_input_for_matmul(input2, - common_sub_shape, - separate2_sub_shape, - reduced_sub_shape_prod, - is_separate_first2); + ov::Tensor matmul_operand1 = reshape_input_for_matmul(input1, + common_sub_shape, + separate1_sub_shape, + reduced_sub_shape_prod, + is_separate_first1); + ov::Tensor matmul_operand2 = reshape_input_for_matmul(input2, + common_sub_shape, + separate2_sub_shape, + reduced_sub_shape_prod, + is_separate_first2); // step 3. apply MatMul operation for formatted inputs Shape matmul_output_shape = compute_matmul_output_shape(common_sub_shape, separate1_sub_shape, separate2_sub_shape); - HostTensorPtr matmul_output = - std::shared_ptr(new HostTensor(matmul_operand1->get_element_type(), matmul_output_shape)); + auto matmul_output = ov::Tensor(matmul_operand1.get_element_type(), matmul_output_shape); bool transpose_a = (is_separate_first1 ? false : true); bool transpose_b = (is_separate_first2 ? true : false); - reference::matmul(matmul_operand1->get_data_ptr(), - matmul_operand2->get_data_ptr(), - matmul_output->get_data_ptr(), - matmul_operand1->get_shape(), - matmul_operand2->get_shape(), + reference::matmul(matmul_operand1.data(), + matmul_operand2.data(), + matmul_output.data(), + matmul_operand1.get_shape(), + matmul_operand2.get_shape(), matmul_output_shape, transpose_a, transpose_b); @@ -928,38 +925,39 @@ void contract_two_inputs(HostTensorVector& inputs, back_shape.insert(back_shape.end(), separate1_sub_shape.begin(), separate1_sub_shape.end()); back_shape.insert(back_shape.end(), separate2_sub_shape.begin(), separate2_sub_shape.end()); - HostTensorPtr contract_output = - std::shared_ptr(new HostTensor(matmul_output->get_element_type(), back_shape)); - const AxisVector order = ngraph::get_default_order(matmul_output->get_shape()); - reference::reshape(reinterpret_cast(matmul_output->get_data_ptr()), - reinterpret_cast(contract_output->get_data_ptr()), - matmul_output->get_shape(), + auto contract_output = ov::Tensor(matmul_output.get_element_type(), back_shape); + OPENVINO_SUPPRESS_DEPRECATED_START + const AxisVector order = ngraph::get_default_order(matmul_output.get_shape()); + OPENVINO_SUPPRESS_DEPRECATED_END + reference::reshape(reinterpret_cast(matmul_output.data()), + reinterpret_cast(contract_output.data()), + matmul_output.get_shape(), order, back_shape, - matmul_output->get_element_type().size()); + matmul_output.get_element_type().size()); update_operands(inputs, input_subscripts, input_ind1, input_ind2, contract_output, resultant_subscript); } template -void einsum_impl(const HostTensorVector& inputs, const HostTensorVector& outputs, const std::string& equation) { +void einsum_impl(const ov::TensorVector& inputs, ov::TensorVector& outputs, const std::string& equation) { std::vector input_subscripts; std::string output_subscript; - ngraph::opset7::Einsum::parse_equation(equation, input_subscripts, output_subscript); + ov::op::v7::Einsum::parse_equation(equation, input_subscripts, output_subscript); // compute einsum path that is used to contract a pair of operands // in more optimal order size_t num_inputs = inputs.size(); auto einsum_path = compute_einsum_path(num_inputs); - HostTensorVector int_inputs = inputs; + ov::TensorVector int_inputs = inputs; // contract inputs by Einsum until just one is remained for (auto const& inds_pair : einsum_path) { contract_two_inputs(int_inputs, input_subscripts, output_subscript, inds_pair.first, inds_pair.second); } - NGRAPH_CHECK(int_inputs.size() == 1); + OPENVINO_ASSERT(int_inputs.size() == 1); // extract diagonal for the single operand extract_diagonal(int_inputs, input_subscripts, 0); @@ -970,26 +968,23 @@ void einsum_impl(const HostTensorVector& inputs, const HostTensorVector& outputs // transpose dimensions to layout required by the output subscript transpose_input(int_inputs, input_subscripts, output_subscript, 0); - auto output_shape = int_inputs[0]->get_shape(); - const auto& element_type = int_inputs[0]->get_element_type(); - const auto& buf_size = shape_size(output_shape) * element_type.size(); - outputs[0]->write(int_inputs[0]->get_data_ptr(), buf_size); + int_inputs[0].copy_to(outputs[0]); } } // namespace -void einsum(const HostTensorVector& outputs, const HostTensorVector& inputs, const std::string& equation) { - NGRAPH_CHECK(inputs.size() > 0, "Einsum must accept at least one input."); - auto input_type = inputs[0]->get_element_type(); +void einsum(ov::TensorVector& outputs, const ov::TensorVector& inputs, const std::string& equation) { + OPENVINO_ASSERT(inputs.size() > 0, "Einsum must accept at least one input."); + auto input_type = inputs[0].get_element_type(); for (size_t input_ind = 1; input_ind < inputs.size(); ++input_ind) { - NGRAPH_CHECK(inputs[input_ind]->get_element_type() == input_type, "Input types must be the same."); + OPENVINO_ASSERT(inputs[input_ind].get_element_type() == input_type, "Input types must be the same."); } if (input_type == element::Type_t::f32) { einsum_impl(inputs, outputs, equation); } else if (input_type == element::Type_t::i32) { einsum_impl(inputs, outputs, equation); } else { - NGRAPH_CHECK(false, "Unsupported input type for Einsum operation."); + OPENVINO_ASSERT(false, "Unsupported input type for Einsum operation."); } } diff --git a/src/core/reference/src/op/fft.cpp b/src/core/reference/src/op/fft.cpp index 012969f278f8f5..9c88b21fd8d1b8 100644 --- a/src/core/reference/src/op/fft.cpp +++ b/src/core/reference/src/op/fft.cpp @@ -25,7 +25,6 @@ #include #include -#include "ngraph/shape.hpp" #include "openvino/reference/utils/fft_common.hpp" namespace ov { @@ -446,32 +445,30 @@ void fft(const float* input_data, } } -OPENVINO_SUPPRESS_DEPRECATED_START -void fft_postprocessing(const HostTensorVector& outputs, - const ngraph::element::Type output_type, +void fft_postprocessing(ov::TensorVector& outputs, + const ov::element::Type output_type, const std::vector& fft_result) { size_t fft_result_size = fft_result.size(); switch (output_type) { case element::Type_t::bf16: { - bfloat16* result_ptr = outputs[0]->get_data_ptr(); + bfloat16* result_ptr = outputs[0].data(); for (size_t i = 0; i < fft_result_size; ++i) { result_ptr[i] = bfloat16(fft_result[i]); } } break; case element::Type_t::f16: { - float16* result_ptr = outputs[0]->get_data_ptr(); + float16* result_ptr = outputs[0].data(); for (size_t i = 0; i < fft_result_size; ++i) { result_ptr[i] = float16(fft_result[i]); } } break; case element::Type_t::f32: { - float* result_ptr = outputs[0]->get_data_ptr(); + float* result_ptr = outputs[0].data(); memcpy(result_ptr, fft_result.data(), fft_result_size * sizeof(float)); } break; default:; } } -OPENVINO_SUPPRESS_DEPRECATED_END } // namespace reference } // namespace ov diff --git a/src/core/reference/src/op/function.cpp b/src/core/reference/src/op/function.cpp index cfd370e38a5c80..a0bedfa1241728 100644 --- a/src/core/reference/src/op/function.cpp +++ b/src/core/reference/src/op/function.cpp @@ -11,18 +11,21 @@ #include "ngraph/runtime/tensor.hpp" #include "openvino/core/deprecated.hpp" #include "openvino/reference/concat.hpp" +#include "shape_util.hpp" namespace ov { namespace reference { -OPENVINO_SUPPRESS_DEPRECATED_START -void function(const std::shared_ptr& function, const HostTensorVector& inputs, HostTensorVector& outputs) { +void function(const std::shared_ptr& function, const ov::TensorVector& inputs, ov::TensorVector& outputs) { const auto& results = function->get_results(); outputs.reserve(results.size()); for (size_t i = 0; i < results.size(); ++i) { - outputs.push_back(std::make_shared()); + OPENVINO_SUPPRESS_DEPRECATED_START + auto shape = results[i]->get_output_partial_shape(0).is_static() ? results[i]->get_output_shape(0) + : ov::util::make_dynamic_shape(); + OPENVINO_SUPPRESS_DEPRECATED_END + outputs.push_back(ov::Tensor(results[i]->get_element_type(), shape)); } function->evaluate(outputs, inputs); } -OPENVINO_SUPPRESS_DEPRECATED_END } // namespace reference } // namespace ov diff --git a/src/core/reference/src/op/if.cpp b/src/core/reference/src/op/if.cpp index 3fe42b61719809..4bffb99470e32d 100644 --- a/src/core/reference/src/op/if.cpp +++ b/src/core/reference/src/op/if.cpp @@ -7,20 +7,19 @@ #include "ngraph/op/if.hpp" #include "openvino/reference/function.hpp" -OPENVINO_SUPPRESS_DEPRECATED_START namespace ov { namespace reference { void if_reference(const std::vector>& bodies, const std::vector& out_descs, const std::vector& input_descs, - const HostTensorVector& out, - const HostTensorVector& args) { + ov::TensorVector& out, + const ov::TensorVector& args) { NGRAPH_CHECK(args.size() > 0, "If operation must have input condition value"); - auto condition_value = args[0]->get_data_ptr()[0]; + auto condition_value = args[0].data()[0]; auto branch_index = (condition_value) ? op::v8::If::THEN_BODY_INDEX : op::v8::If::ELSE_BODY_INDEX; - HostTensorVector inputs_to_body; - HostTensorVector outs_from_body; + ov::TensorVector inputs_to_body; + ov::TensorVector outs_from_body; inputs_to_body.resize(input_descs[branch_index].size()); auto inputs_size = args.size(); auto output_size = out.size(); @@ -36,8 +35,7 @@ void if_reference(const std::vector>& bodies, "Incorrect associating! If has not output with id ", out_descr->m_output_index); auto res = outs_from_body[out_descr->m_body_value_index]; - out[out_descr->m_output_index]->set_shape(res->get_shape()); - out[out_descr->m_output_index]->write(res->get_data_ptr(), res->get_size_in_bytes()); + res.copy_to(out[out_descr->m_output_index]); } } } // namespace reference diff --git a/src/core/reference/src/op/loop.cpp b/src/core/reference/src/op/loop.cpp index 0a1ee146664a74..39ad2c2d4f229f 100644 --- a/src/core/reference/src/op/loop.cpp +++ b/src/core/reference/src/op/loop.cpp @@ -4,12 +4,13 @@ #include "openvino/reference/loop.hpp" +#include + #include "openvino/op/tensor_iterator.hpp" #include "openvino/reference/concat.hpp" #include "openvino/reference/function.hpp" #include "openvino/reference/split.hpp" - -OPENVINO_SUPPRESS_DEPRECATED_START +#include "openvino/runtime/tensor.hpp" namespace ov { namespace reference { @@ -17,8 +18,8 @@ void loop(const std::shared_ptr& func, const op::util::OutputDescriptionVector& out_descs, const op::util::InputDescriptionVector& input_descs, const op::v5::Loop::SpecialBodyPorts& special_ports, - const HostTensorVector& out, - const HostTensorVector& args) { + ov::TensorVector& out, + const ov::TensorVector& args) { const auto& cur_iter_idx = special_ports.current_iteration_input_idx; auto val = std::find_if(input_descs.begin(), input_descs.end(), @@ -32,9 +33,9 @@ void loop(const std::shared_ptr& func, // should allocate input_descs.size() + 1 inputs and set default value (0) for // current_iteration input. int64_t inputs_count = input_descs.size() + (cur_iter_idx >= 0 ? !cur_iter_initial_value_exist : 0); - HostTensorVector inputs_to_body; + ov::TensorVector inputs_to_body; for (int64_t i = 0; i < inputs_count; ++i) - inputs_to_body.push_back(std::make_shared(element::dynamic, PartialShape::dynamic())); + inputs_to_body.push_back(ov::Tensor()); if (cur_iter_idx >= 0 && !cur_iter_initial_value_exist) { const auto& cur_iter = func->get_parameters().at(cur_iter_idx); if (cur_iter->get_partial_shape().is_dynamic()) { @@ -45,8 +46,10 @@ void loop(const std::shared_ptr& func, auto init = std::make_shared(func->get_parameters().at(cur_iter_idx)->get_element_type(), func->get_parameters().at(cur_iter_idx)->get_shape(), 0); - inputs_to_body.at(cur_iter_idx)->initialize(init); - // reinterpret_cast(inputs_to_body.at(cur_iter_idx).data())[0] = 0; + ov::Tensor in_tensor(func->get_parameters().at(cur_iter_idx)->get_element_type(), + func->get_parameters().at(cur_iter_idx)->get_shape()); + std::memset(in_tensor.data(), 0, in_tensor.get_byte_size()); + inputs_to_body.at(cur_iter_idx) = in_tensor; } // Port map processing: inputs and back edges @@ -65,19 +68,19 @@ void loop(const std::shared_ptr& func, // Get TripCount int64_t trip_count = 0; - if (args[0]->get_element_type() == ngraph::element::i32) { - auto* trip_count_p = args[0]->get_data_ptr(); + if (args[0].get_element_type() == ov::element::i32) { + auto* trip_count_p = args[0].data(); trip_count = trip_count_p[0]; - } else if (args[0]->get_element_type() == ngraph::element::i64) { - auto* trip_count_p = args[0]->get_data_ptr(); + } else if (args[0].get_element_type() == ov::element::i64) { + auto* trip_count_p = args[0].data(); trip_count = trip_count_p[0]; } else { - NGRAPH_CHECK(false, "Unsupported element type for trip_count input. Expected int32 or int64."); + OPENVINO_THROW("Unsupported element type for trip_count input. Expected int32 or int64."); } - NGRAPH_CHECK(trip_count != 0, "Zero count of iteration not supported"); + OPENVINO_ASSERT(trip_count != 0, "Zero count of iteration not supported"); // Loop iterations - auto exec_condition = args[1]->get_data_ptr(); + auto exec_condition = args[1].data(); if (exec_condition[0]) { // Find all ConcatOutputDescription std::vector> concat_outputs; @@ -89,28 +92,28 @@ void loop(const std::shared_ptr& func, // Slicing std::vector> slice_inputs; - std::vector sliced_values; + std::vector sliced_values; int slice_in_idx = 0; for (const auto& desc : input_descs) { if (const auto& slice_desc = std::dynamic_pointer_cast(desc)) { - const auto el_size = args[slice_desc->m_input_index]->get_element_type().size(); + const auto el_size = args[slice_desc->m_input_index].get_element_type().size(); slice_inputs.push_back(slice_desc); - auto shape = args[slice_desc->m_input_index]->get_shape(); + auto shape = args[slice_desc->m_input_index].get_shape(); uint64_t num_iterations = shape.at(slice_desc->m_axis); shape.at(slice_desc->m_axis) = 1; - sliced_values.emplace_back(HostTensorVector()); + sliced_values.emplace_back(ov::TensorVector()); for (uint64_t i = 0; i < num_iterations; ++i) { sliced_values.back().emplace_back( - std::make_shared(args[slice_desc->m_input_index]->get_element_type(), shape)); + ov::Tensor(args[slice_desc->m_input_index].get_element_type(), shape)); } std::vector pointers_to_data(num_iterations); for (size_t j = 0; j < pointers_to_data.size(); ++j) { pointers_to_data[slice_desc->m_stride > 0 ? j : (pointers_to_data.size() - j - 1)] = - sliced_values[slice_in_idx][j]->get_data_ptr(); + static_cast(sliced_values[slice_in_idx][j].data()); } - reference::split(args[slice_desc->m_input_index]->get_data_ptr(), - args[slice_desc->m_input_index]->get_shape(), + reference::split(static_cast(args[slice_desc->m_input_index].data()), + args[slice_desc->m_input_index].get_shape(), el_size, slice_desc->m_axis, num_iterations, @@ -120,8 +123,8 @@ void loop(const std::shared_ptr& func, } // Allocate vectors for store output values - std::vector values_to_concat(concat_outputs.size()); - HostTensorVector body_outputs; + std::vector values_to_concat(concat_outputs.size()); + ov::TensorVector body_outputs; // Negative value means infinity count of iterations trip_count = trip_count >= 0 ? trip_count : std::numeric_limits::max(); @@ -145,7 +148,7 @@ void loop(const std::shared_ptr& func, bool body_exec_condition(false); if (static_cast(body_outputs.size()) > special_ports.body_condition_output_idx && body_outputs[special_ports.body_condition_output_idx]) - body_outputs[special_ports.body_condition_output_idx]->read(&body_exec_condition, sizeof(bool)); + body_exec_condition = body_outputs[special_ports.body_condition_output_idx].data()[0]; if (!body_exec_condition) break; @@ -155,21 +158,24 @@ void loop(const std::shared_ptr& func, const auto& cur_iter_param = func->get_parameters().at(cur_iter_idx); int64_t iter_num = cur_iter + 1; if (cur_iter_param->get_element_type() == element::i64) - inputs_to_body.at(cur_iter_idx)->write(&iter_num, cur_iter_param->get_element_type().size()); + std::memcpy(inputs_to_body.at(cur_iter_idx).data(), + &iter_num, + cur_iter_param->get_element_type().size()); else if (cur_iter_param->get_element_type() == element::i32) { int32_t iter_num_i32 = static_cast(iter_num); - inputs_to_body.at(cur_iter_idx)->write(&iter_num_i32, cur_iter_param->get_element_type().size()); + std::memcpy(inputs_to_body.at(cur_iter_idx).data(), + &iter_num_i32, + cur_iter_param->get_element_type().size()); } else - NGRAPH_CHECK(false, - "Unsupported element type for current iteration " - "input. Expected int32 or int64."); + OPENVINO_THROW("Unsupported element type for current iteration " + "input. Expected int32 or int64."); } // Back-edge processing bool need_validate = false; for (auto& back_edge : back_edges) { - const auto& input_shape = inputs_to_body[back_edge.param_idx]->get_shape(); - const auto& result_shape = body_outputs[back_edge.result_idx]->get_shape(); + const auto& input_shape = inputs_to_body[back_edge.param_idx].get_shape(); + const auto& result_shape = body_outputs[back_edge.result_idx].get_shape(); // when output shape does not equal to input shape in a back-edge, such as // Parameter(out:1)->| // |->Concat(out:2)->Result(out:2) @@ -190,8 +196,7 @@ void loop(const std::shared_ptr& func, for (const auto& desc : out_descs) { if (const auto& body_desc = std::dynamic_pointer_cast(desc)) { const auto& res = body_outputs[body_desc->m_body_value_index]; - out[body_desc->m_output_index]->set_shape(res->get_shape()); - out[body_desc->m_output_index]->write(res->get_data_ptr(), res->get_size_in_bytes()); + res.copy_to(out[body_desc->m_output_index]); } } @@ -201,21 +206,21 @@ void loop(const std::shared_ptr& func, auto shape = func->get_results().at(concat_desc->m_body_value_index)->get_shape(); std::vector shapes_to_concat(values_to_concat[i].size(), shape); shape.at(concat_desc->m_axis) = values_to_concat[i].size(); - out[concat_desc->m_output_index]->set_shape(shape); + out[concat_desc->m_output_index].set_shape(shape); std::vector pointers_on_values; pointers_on_values.reserve(values_to_concat[i].size()); for (const auto& vec : values_to_concat[i]) { - pointers_on_values.push_back(vec->get_data_ptr()); + pointers_on_values.push_back(static_cast(vec.data())); } reference::concat(pointers_on_values, - out[concat_desc->m_output_index]->get_data_ptr(), + static_cast(out[concat_desc->m_output_index].data()), shapes_to_concat, shape, concat_desc->m_axis, - out[concat_desc->m_output_index]->get_element_type().size()); + out[concat_desc->m_output_index].get_element_type().size()); } } else { - NGRAPH_CHECK(false, "ExecutionCondition is false. Zero count of iteration not supported."); + OPENVINO_THROW("ExecutionCondition is false. Zero count of iteration not supported."); } } } // namespace reference diff --git a/src/core/reference/src/op/non_max_suppression.cpp b/src/core/reference/src/op/non_max_suppression.cpp index e0032c3f5b4ecb..51609fd8059377 100644 --- a/src/core/reference/src/op/non_max_suppression.cpp +++ b/src/core/reference/src/op/non_max_suppression.cpp @@ -2,15 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/non_max_suppression.hpp" +#include "openvino/op/non_max_suppression.hpp" #include #include #include #include -#include "ngraph/shape.hpp" #include "openvino/reference/non_max_suppression.hpp" +#include "openvino/runtime/tensor.hpp" namespace ov { namespace reference { @@ -366,35 +366,31 @@ void non_max_suppression(const float* boxes_data, } } -OPENVINO_SUPPRESS_DEPRECATED_START -void nms_postprocessing(const HostTensorVector& outputs, - const ngraph::element::Type output_type, +void nms_postprocessing(ov::TensorVector& outputs, + const ov::element::Type output_type, const std::vector& selected_indices, const std::vector& selected_scores, int64_t valid_outputs, - const ngraph::element::Type selected_scores_type) { - outputs[0]->set_element_type(output_type); - outputs[0]->set_shape(Shape{static_cast(valid_outputs), 3}); + const ov::element::Type selected_scores_type) { + outputs[0].set_shape(Shape{static_cast(valid_outputs), 3}); size_t num_of_outputs = outputs.size(); if (num_of_outputs >= 2) { - outputs[1]->set_element_type(selected_scores_type); - outputs[1]->set_shape(Shape{static_cast(valid_outputs), 3}); + outputs[1].set_shape(Shape{static_cast(valid_outputs), 3}); } if (num_of_outputs >= 3) { - outputs[2]->set_element_type(output_type); - outputs[2]->set_shape(Shape{1}); + outputs[2].set_shape(Shape{1}); } size_t selected_size = valid_outputs * 3; - if (output_type == ngraph::element::i64) { - int64_t* indices_ptr = outputs[0]->get_data_ptr(); + if (output_type == ov::element::i64) { + int64_t* indices_ptr = outputs[0].data(); memcpy(indices_ptr, selected_indices.data(), selected_size * sizeof(int64_t)); } else { - int32_t* indices_ptr = outputs[0]->get_data_ptr(); + int32_t* indices_ptr = outputs[0].data(); for (size_t i = 0; i < selected_size; ++i) { indices_ptr[i] = static_cast(selected_indices[i]); } @@ -408,19 +404,19 @@ void nms_postprocessing(const HostTensorVector& outputs, switch (selected_scores_type) { case element::Type_t::bf16: { - bfloat16* scores_ptr = outputs[1]->get_data_ptr(); + bfloat16* scores_ptr = outputs[1].data(); for (size_t i = 0; i < selected_scores_size; ++i) { scores_ptr[i] = bfloat16(selected_scores[i]); } } break; case element::Type_t::f16: { - float16* scores_ptr = outputs[1]->get_data_ptr(); + float16* scores_ptr = outputs[1].data(); for (size_t i = 0; i < selected_scores_size; ++i) { scores_ptr[i] = float16(selected_scores[i]); } } break; case element::Type_t::f32: { - float* scores_ptr = outputs[1]->get_data_ptr(); + float* scores_ptr = outputs[1].data(); memcpy(scores_ptr, selected_scores.data(), selected_size * sizeof(float)); } break; default:; @@ -430,21 +426,21 @@ void nms_postprocessing(const HostTensorVector& outputs, return; } - if (output_type == ngraph::element::i64) { - int64_t* valid_outputs_ptr = outputs[2]->get_data_ptr(); + if (output_type == ov::element::i64) { + int64_t* valid_outputs_ptr = outputs[2].data(); *valid_outputs_ptr = valid_outputs; } else { - int32_t* valid_outputs_ptr = outputs[2]->get_data_ptr(); + int32_t* valid_outputs_ptr = outputs[2].data(); *valid_outputs_ptr = static_cast(valid_outputs); } } -void nms5_postprocessing(const HostTensorVector& outputs, - const ngraph::element::Type output_type, +void nms5_postprocessing(ov::TensorVector& outputs, + const ov::element::Type output_type, const std::vector& selected_indices, const std::vector& selected_scores, int64_t valid_outputs, - const ngraph::element::Type selected_scores_type) { + const ov::element::Type selected_scores_type) { nms_postprocessing(outputs, output_type, selected_indices, selected_scores, valid_outputs, selected_scores_type); } } // namespace reference diff --git a/src/core/reference/src/op/tensor_iterator.cpp b/src/core/reference/src/op/tensor_iterator.cpp index 26646536372c98..d0466d7cef1044 100644 --- a/src/core/reference/src/op/tensor_iterator.cpp +++ b/src/core/reference/src/op/tensor_iterator.cpp @@ -4,23 +4,25 @@ #include "openvino/reference/tensor_iterator.hpp" +#include "openvino/op/loop.hpp" +#include "openvino/op/tensor_iterator.hpp" #include "openvino/reference/concat.hpp" #include "openvino/reference/function.hpp" #include "openvino/reference/split.hpp" +#include "openvino/runtime/tensor.hpp" -OPENVINO_SUPPRESS_DEPRECATED_START namespace ov { namespace reference { void tensor_iterator(uint64_t num_iterations, const std::shared_ptr& func, const op::util::OutputDescriptionVector& out_descs, const op::util::InputDescriptionVector& input_descs, - const HostTensorVector& out, - const HostTensorVector& args, + ov::TensorVector& out, + const ov::TensorVector& args, const custom_evaluate_function& evaluate) { - HostTensorVector inputs_to_body; + ov::TensorVector inputs_to_body; for (size_t i = 0; i < input_descs.size(); ++i) - inputs_to_body.push_back(std::make_shared(element::dynamic, PartialShape::dynamic())); + inputs_to_body.push_back(ov::Tensor()); // Port map processing: inputs and back edges struct BackEdge { @@ -30,12 +32,12 @@ void tensor_iterator(uint64_t num_iterations, std::vector back_edges; for (const auto& desc : input_descs) { inputs_to_body[desc->m_body_parameter_index] = args[desc->m_input_index]; - if (const auto& merged_desc = std::dynamic_pointer_cast(desc)) { + if (const auto& merged_desc = std::dynamic_pointer_cast(desc)) { back_edges.push_back({merged_desc->m_body_parameter_index, merged_desc->m_body_value_index}); } } // Find all ConcatOutputDescription - std::vector> concat_outputs; + std::vector> concat_outputs; for (const auto& desc : out_descs) { if (const auto& concat_desc = std::dynamic_pointer_cast(desc)) { @@ -45,26 +47,26 @@ void tensor_iterator(uint64_t num_iterations, // Slicing std::vector> slice_inputs; - std::vector sliced_values; + std::vector sliced_values; int slice_in_idx = 0; for (const auto& desc : input_descs) { if (const auto& slice_desc = std::dynamic_pointer_cast(desc)) { - const auto el_size = args[slice_desc->m_input_index]->get_element_type().size(); + const auto el_size = args[slice_desc->m_input_index].get_element_type().size(); slice_inputs.push_back(slice_desc); - auto shape = args[slice_desc->m_input_index]->get_shape(); + auto shape = args[slice_desc->m_input_index].get_shape(); shape.at(slice_desc->m_axis) = 1; - sliced_values.emplace_back(HostTensorVector()); + sliced_values.emplace_back(ov::TensorVector()); for (uint64_t i = 0; i < num_iterations; ++i) { sliced_values.back().emplace_back( - std::make_shared(args[slice_desc->m_input_index]->get_element_type(), shape)); + ov::Tensor(args[slice_desc->m_input_index].get_element_type(), shape)); } std::vector pointers_to_data(num_iterations); for (size_t j = 0; j < pointers_to_data.size(); ++j) { pointers_to_data[slice_desc->m_stride > 0 ? j : (pointers_to_data.size() - j - 1)] = - sliced_values[slice_in_idx][j]->get_data_ptr(); + static_cast(sliced_values[slice_in_idx][j].data()); } - reference::split(args[slice_desc->m_input_index]->get_data_ptr(), - args[slice_desc->m_input_index]->get_shape(), + reference::split(static_cast(args[slice_desc->m_input_index].data()), + args[slice_desc->m_input_index].get_shape(), el_size, slice_desc->m_axis, num_iterations, @@ -74,8 +76,8 @@ void tensor_iterator(uint64_t num_iterations, } // Allocate vectors for store output values - std::vector values_to_concat(concat_outputs.size()); - HostTensorVector body_outputs; + std::vector values_to_concat(concat_outputs.size()); + ov::TensorVector body_outputs; for (uint64_t cur_iter = 0; cur_iter < num_iterations; ++cur_iter) { // Copy new values for sliced inputs @@ -107,8 +109,7 @@ void tensor_iterator(uint64_t num_iterations, if (const auto& body_desc = std::dynamic_pointer_cast(desc)) { // Copy output values from the last iteration const auto& res = body_outputs[body_desc->m_body_value_index]; - out[body_desc->m_output_index]->set_shape(res->get_shape()); - out[body_desc->m_output_index]->write(res->get_data_ptr(), res->get_size_in_bytes()); + res.copy_to(out[body_desc->m_output_index]); } } @@ -120,20 +121,20 @@ void tensor_iterator(uint64_t num_iterations, auto shape = func->get_results().at(concat_desc->m_body_value_index)->get_shape(); std::vector shapes_to_concat(values_to_concat[i].size(), shape); shape.at(concat_desc->m_axis) = values_to_concat[i].size(); - out[concat_desc->m_output_index]->set_shape(shape); + out[concat_desc->m_output_index].set_shape(shape); std::vector pointers_on_values; pointers_on_values.reserve(values_to_concat[i].size()); for (size_t j = 0; j < values_to_concat[i].size(); ++j) { size_t idx = concat_desc->m_stride > 0 ? j : (values_to_concat[i].size() - j - 1); if (values_to_concat[i].size() > idx && values_to_concat[i][idx]) - pointers_on_values.push_back(values_to_concat[i][idx]->get_data_ptr()); + pointers_on_values.push_back(static_cast(values_to_concat[i][idx].data())); } reference::concat(pointers_on_values, - out[concat_desc->m_output_index]->get_data_ptr(), + static_cast(out[concat_desc->m_output_index].data()), shapes_to_concat, shape, concat_desc->m_axis, - out[concat_desc->m_output_index]->get_element_type().size()); + out[concat_desc->m_output_index].get_element_type().size()); } } } // namespace reference diff --git a/src/core/reference/src/op/tile.cpp b/src/core/reference/src/op/tile.cpp index b027613d980623..f88e56392a26a8 100644 --- a/src/core/reference/src/op/tile.cpp +++ b/src/core/reference/src/op/tile.cpp @@ -9,8 +9,6 @@ #include #include -#include "ngraph/check.hpp" - using namespace ov; namespace { diff --git a/src/core/shape_inference/include/slice_shape_inference.hpp b/src/core/shape_inference/include/slice_shape_inference.hpp index 83a4c4fef3dfa0..cd6c3b018bdf68 100644 --- a/src/core/shape_inference/include/slice_shape_inference.hpp +++ b/src/core/shape_inference/include/slice_shape_inference.hpp @@ -136,7 +136,8 @@ std::vector shape_infer(const Slice* op, } auto& last_dim = out[out.size() - 1]; - if (std::is_same::value && (last_dim == input_dim)) { + if (std::is_same::value && + (last_dim == input_dim && last_dim != Dimension::dynamic())) { // for equal ov::Dimension do merge to get input label (always success) DimType::merge(last_dim, last_dim, input_dim); } diff --git a/src/core/shape_inference/include/strided_slice_shape_inference.hpp b/src/core/shape_inference/include/strided_slice_shape_inference.hpp index 2a870f158bf113..4806cded4dce5e 100644 --- a/src/core/shape_inference/include/strided_slice_shape_inference.hpp +++ b/src/core/shape_inference/include/strided_slice_shape_inference.hpp @@ -176,7 +176,8 @@ std::vector shape_infer(const StridedSlice* op, const auto& stop = end_mask.count(axis) ? default_stop : (*end)[axis]; auto sliced_dim = slice::make_dim(input_dim, start, stop, stride); - if (std::is_same::value && (sliced_dim == input_dim)) { + if (std::is_same::value && + (sliced_dim == input_dim && sliced_dim != Dimension::dynamic())) { // for equal ov::Dimension do merge to get input label (always success) DimType::merge(sliced_dim, sliced_dim, input_dim); } diff --git a/src/core/src/bound_evaluate.cpp b/src/core/src/bound_evaluate.cpp index 49e32619766082..38d09dbcdf6fe4 100644 --- a/src/core/src/bound_evaluate.cpp +++ b/src/core/src/bound_evaluate.cpp @@ -7,6 +7,7 @@ #include "ngraph/validation_util.hpp" #include "openvino/core/dimension_tracker.hpp" #include "openvino/core/rt_info.hpp" +#include "openvino/op/util/symbolic_info.hpp" #include "openvino/opsets/opset10.hpp" #include "shape_util.hpp" #include "tensor_conversion_util.hpp" @@ -112,9 +113,16 @@ ov::Tensor evaluate_bound(const Output& output, bool is_upper, bool invali } } - if (node->evaluate_label(output_labels)) - for (size_t i = 0; i < outputs.size(); ++i) - node->get_output_tensor(i).set_value_label(output_labels[i]); + bool labels_evaluated = node->evaluate_label(output_labels); + for (size_t i = 0; i < outputs.size(); ++i) { + auto& out_tensor = node->get_output_tensor(i); + if (!out_tensor.get_value_label().empty()) + continue; + if (labels_evaluated) + out_tensor.set_value_label(output_labels[i]); + if (outputs[i]) + ov::populate_tensor_with_missing_labels(out_tensor); + } for (const auto& input : input_values) { auto& tensor = input.get_tensor(); @@ -329,8 +337,11 @@ std::pair ov::evaluate_both_bounds(const Output& o out_tensor.set_upper_value(outputs_upper[i]); } + if (!out_tensor.get_value_label().empty()) + continue; if (labels_evaluated) - node->get_output_tensor(i).set_value_label(output_labels[i]); + out_tensor.set_value_label(output_labels[i]); + ov::populate_tensor_with_missing_labels(node->get_output_tensor(i)); } for (const auto& input : node->input_values()) { auto& tensor = input.get_tensor(); diff --git a/src/core/src/descriptor/tensor.cpp b/src/core/src/descriptor/tensor.cpp index c0b2cb973f4a89..18c9fd3ff18f96 100644 --- a/src/core/src/descriptor/tensor.cpp +++ b/src/core/src/descriptor/tensor.cpp @@ -6,6 +6,7 @@ #include "openvino/core/except.hpp" #include "openvino/core/node.hpp" +#include "openvino/op/util/symbolic_info.hpp" ov::descriptor::Tensor::Tensor(const element::Type& element_type, const PartialShape& pshape, @@ -46,6 +47,8 @@ void ov::descriptor::Tensor::set_element_type(const element::Type& element_type) OPENVINO_SUPPRESS_DEPRECATED_END void ov::descriptor::Tensor::invalidate_values() { + if (ov::skip_invalidation(*this)) + return; m_upper_value = {}; m_lower_value = {}; m_value_label.clear(); diff --git a/src/core/src/dimension_tracker.cpp b/src/core/src/dimension_tracker.cpp index ec8d48472408a9..07bb1f52435657 100644 --- a/src/core/src/dimension_tracker.cpp +++ b/src/core/src/dimension_tracker.cpp @@ -86,4 +86,9 @@ void DimensionTracker::set_up_for_tracking(Dimension& d) { void DimensionTracker::set_up_for_tracking(Dimension& d, label_t label) const { set_label(d, label); set_table_of_equivalence(d); -} \ No newline at end of file +} + +void DimensionTracker::set_up_for_tracking(ov::PartialShape& shape) { + for (auto& d : shape) + set_up_for_tracking(d); +} diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index 10ebc91b976d92..7f20ce4b991b6c 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -740,7 +740,13 @@ inline void update_output_tensors(ov::TensorVector& output_values, const ngraph: if (!tensor && outputs[i]->get_partial_shape().is_static()) { tensor = ov::Tensor(outputs[i]->get_element_type(), outputs[i]->get_shape()); } - output_values[i] = tensor; + if (output_values[i]) { + // Copy value to the original tensor + tensor.copy_to(output_values[i]); + } else { + // Tensor is not initialized, so create the new tensor + output_values[i] = tensor; + } } } } diff --git a/src/core/src/op/loop.cpp b/src/core/src/op/loop.cpp index 419e564b19f35e..ef623af65814b6 100644 --- a/src/core/src/op/loop.cpp +++ b/src/core/src/op/loop.cpp @@ -12,6 +12,7 @@ #include "ngraph/graph_util.hpp" #include "ngraph/opsets/opset5.hpp" #include "openvino/reference/loop.hpp" +#include "openvino/runtime/tensor.hpp" using namespace std; using namespace ngraph; @@ -347,8 +348,7 @@ Output op::v5::Loop::get_concatenated_slices(const Output& value, return SubGraphOp::get_concatenated_slices(value, start, stride, part_size, end, axis); } -OPENVINO_SUPPRESS_DEPRECATED_START -bool op::v5::Loop::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { +bool op::v5::Loop::evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const { OV_OP_SCOPE(v5_Loop_evaluate); ov::reference::loop(m_bodies[0], m_output_descriptions[0], @@ -358,7 +358,6 @@ bool op::v5::Loop::evaluate(const HostTensorVector& outputs, const HostTensorVec inputs); return true; } -OPENVINO_SUPPRESS_DEPRECATED_END bool op::v5::Loop::has_evaluate() const { OV_OP_SCOPE(v5_Loop_has_evaluate); diff --git a/src/core/src/op/util/symbolic_info.cpp b/src/core/src/op/util/symbolic_info.cpp new file mode 100644 index 00000000000000..5dd2858a4ee507 --- /dev/null +++ b/src/core/src/op/util/symbolic_info.cpp @@ -0,0 +1,83 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/op/util/symbolic_info.hpp" + +#include + +#include "openvino/core/dimension_tracker.hpp" +#include "openvino/op/util/multi_subgraph_base.hpp" + +namespace { +std::shared_ptr get_table(const ov::RTMap& rt_info) { + const auto& type = ov::SymbolicInfo::get_type_info_static(); + if (!rt_info.count(type)) + return nullptr; + return rt_info.at(type).as().get_table(); +} +} // namespace + +void ov::set_up_symbolic_info(const std::shared_ptr& model, + const std::shared_ptr& table) { + auto& rt_info = model->get_rt_info(); + rt_info[ov::SymbolicInfo::get_type_info_static()] = ov::SymbolicInfo(true, table); +} + +void ov::set_up_symbolic_info(const ov::Output& output, + const std::shared_ptr& table) { + auto& rt_info = output.get_tensor().get_rt_info(); + rt_info[ov::SymbolicInfo::get_type_info_static()] = ov::SymbolicInfo(true, table); +} + +bool ov::skip_invalidation(const ov::descriptor::Tensor& tensor) { + const auto& rt_info = tensor.get_rt_info(); + const auto& type = ov::SymbolicInfo::get_type_info_static(); + return rt_info.count(type) && rt_info.at(type).as().get_skip_invalidation(); +} + +std::shared_ptr ov::table_of_equivalence(const ov::descriptor::Tensor& tensor) { + const auto& rt_info = tensor.get_rt_info(); + return get_table(rt_info); +} + +std::shared_ptr ov::table_of_equivalence(const std::shared_ptr& model) { + const auto& rt_info = model->get_rt_info(); + return get_table(rt_info); +} + +void ov::populate_tensor_with_missing_labels(ov::descriptor::Tensor& tensor) { + if (auto table = ov::table_of_equivalence(tensor)) { + auto label_values = tensor.get_value_label(); + if (label_values.empty()) { + const auto& pshape = tensor.get_partial_shape(); + if (pshape.is_dynamic()) + return; + label_values.resize(ov::shape_size(pshape.to_shape()), ov::no_label); + } + for (auto& label : label_values) + if (label == ov::no_label) + label = table->get_next_label(); + tensor.set_value_label(label_values); + } +} + +void ov::remove_symbolic_info(const std::shared_ptr& model, bool outermost_model) { + const auto& type = ov::SymbolicInfo::get_type_info_static(); + auto& model_rt_info = model->get_rt_info(); + if (model_rt_info.count(type)) + model_rt_info.erase(type); + for (const auto& op : model->get_ops()) { + if (auto multi_subgraph_op = std::dynamic_pointer_cast(op)) + for (const auto& sub_graph : multi_subgraph_op->get_functions()) + if (sub_graph) + remove_symbolic_info(sub_graph, false); + for (auto& output : op->outputs()) { + auto& rt_info = output.get_tensor().get_rt_info(); + if (rt_info.count(type)) + rt_info.erase(type); + } + } + if (outermost_model) + model->validate_nodes_and_infer_types(); +} diff --git a/src/core/src/pass/manager.cpp b/src/core/src/pass/manager.cpp index f55e0c8b57a4f2..804ee9af0bfea2 100644 --- a/src/core/src/pass/manager.cpp +++ b/src/core/src/pass/manager.cpp @@ -135,7 +135,8 @@ bool ov::pass::Manager::run_passes(shared_ptr func) { index++; pass_timer.stop(); if (profile_enabled) { - cout << setw(7) << pass_timer.get_milliseconds() << "ms " << pass->get_name() << "\n"; + cout << setw(7) << pass_timer.get_milliseconds() << "ms" << (pass_applied ? " + " : " ") + << pass->get_name() << "\n"; } function_changed = function_changed || pass_applied; needs_validate = pass_applied; diff --git a/src/core/src/pass/serialize.cpp b/src/core/src/pass/serialize.cpp index de2fcd8c46b8fe..70b91b7e97bf81 100644 --- a/src/core/src/pass/serialize.cpp +++ b/src/core/src/pass/serialize.cpp @@ -197,7 +197,7 @@ void ngfunction_2_ir(pugi::xml_node& node, bool deterministic); namespace rt_info { -const std::vector list_of_names{ +static const std::vector list_of_names{ "PrimitivesPriority", "alt_width", }; diff --git a/src/core/src/pass/visualize_tree.cpp b/src/core/src/pass/visualize_tree.cpp index 58622688b47c3e..15f1bc410b2189 100644 --- a/src/core/src/pass/visualize_tree.cpp +++ b/src/core/src/pass/visualize_tree.cpp @@ -11,6 +11,7 @@ #include "openvino/core/type.hpp" #include "openvino/op/constant.hpp" #include "openvino/op/parameter.hpp" +#include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/op/util/op_types.hpp" #include "openvino/util/common_util.hpp" #include "openvino/util/env_util.hpp" @@ -165,6 +166,19 @@ static std::string get_attribute_values(const std::map& at return ss.str(); } +static std::string name_of_subgraph_file(const std::shared_ptr op, + const std::string& current_file_name, + const size_t& i) { + // friendly is never empty it is either friendly (set by user) or unique (auto-generated) name + auto node_name = op->get_friendly_name(); + std::replace(node_name.begin(), node_name.end(), '/', '-'); + auto postfix = "_node_" + node_name + "_subgraph_#" + std::to_string(i); + auto file_name = current_file_name; + auto insert_pos = file_name.find_last_of('.'); + file_name.insert(insert_pos, postfix); + return file_name; +} + bool ov::pass::VisualizeTree::run_on_model(const std::shared_ptr& f) { RUN_ON_MODEL_SCOPE(VisualizeTree); std::unordered_map height_maps; @@ -181,6 +195,14 @@ bool ov::pass::VisualizeTree::run_on_model(const std::shared_ptr& f) for (auto it = nodes.rbegin(); it != nodes.rend(); ++it) { auto& node = *it; + if (auto multi_subgraph_op = std::dynamic_pointer_cast(node)) { + for (size_t i = 0; i < multi_subgraph_op->get_internal_subgraphs_size(); ++i) + if (const auto& sub_graph = multi_subgraph_op->get_function(i)) + ov::pass::VisualizeTree(name_of_subgraph_file(multi_subgraph_op, m_name, i), + m_node_modifiers, + m_dot_only) + .run_on_model(sub_graph); + } for (auto& output : node->outputs()) { for (auto& input : output.get_target_inputs()) { auto target_node = input.get_node(); @@ -213,8 +235,6 @@ ov::pass::VisualizeTree::VisualizeTree(const std::string& file_name, node_modifi void ov::pass::VisualizeTree::add_node_arguments(std::shared_ptr node, std::unordered_map& height_maps, size_t& fake_node_ctr) { - static const int const_max_elements = ov::util::getenv_int("OV_VISUALIZE_TREE_CONST_MAX_ELEMENTS", 7); - size_t arg_index = 0; for (auto input_value : node->input_values()) { auto arg = input_value.get_node_shared_ptr(); @@ -227,7 +247,7 @@ void ov::pass::VisualizeTree::add_node_arguments(std::shared_ptr node, "style=\"dashed\"", color, std::string("label=\"") + get_node_name(arg) + std::string("\n") + - get_constant_value(arg, const_max_elements) + std::string("\"")}; + get_constant_value(arg) + std::string("\"")}; if (m_node_modifiers && !arg->output(0).get_rt_info().empty()) { m_node_modifiers(*arg, attributes); @@ -280,13 +300,7 @@ std::string ov::pass::VisualizeTree::add_attributes(std::shared_ptr node) static std::string pretty_partial_shape(const ov::PartialShape& shape) { std::stringstream ss; - - if (shape.rank().is_dynamic()) { - ss << "?"; - } else { - ss << shape; - } - + ss << shape; return ss.str(); } @@ -327,21 +341,11 @@ static std::string pretty_min_max_denormal_value(const std::vector& values) { } template -static std::string pretty_value(const std::vector& values, size_t max_elements, bool allow_obfuscate = false) { +static std::string pretty_value(const std::vector& values, bool allow_obfuscate = false) { std::stringstream ss; for (size_t i = 0; i < values.size(); ++i) { - if (i < max_elements) { - if (i != 0 && i % 8 == 0) { - ss << std::endl; - } - } else { - bool all_same = std::all_of(values.begin(), values.end(), [&](const T& el) { - return el == values[0]; - }); - ss << "..." << (all_same ? " same" : ""); - break; - } - + if (i != 0 && i % 8 == 0) + ss << std::endl; const auto& value = values[i]; if (i > 0) ss << ", "; @@ -361,46 +365,46 @@ static std::string pretty_value(const std::vector& values, size_t max_element return ss.str(); } -static std::string get_value(const std::shared_ptr& constant, - size_t max_elements, - bool allow_obfuscate = false) { +static std::string get_value(const std::shared_ptr& constant, bool allow_obfuscate = false) { + static const int max_elements = ov::util::getenv_int("OV_VISUALIZE_TREE_CONST_MAX_ELEMENTS", 7); std::stringstream ss; + ss << "[ "; switch (constant->get_output_element_type(0)) { case ov::element::Type_t::undefined: - ss << "[ undefined value ]"; - break; case ov::element::Type_t::dynamic: - ss << "[ dynamic value ]"; - break; case ov::element::Type_t::u1: - ss << "[ u1 value ]"; - break; case ov::element::Type_t::u4: - ss << "[ u4 value ]"; - break; case ov::element::Type_t::i4: - ss << "[ i4 value ]"; + ss << constant->get_output_element_type(0).get_type_name() << " value"; break; case ov::element::Type_t::bf16: case ov::element::Type_t::f16: case ov::element::Type_t::f32: case ov::element::Type_t::f64: - ss << "[" << pretty_value(constant->cast_vector(), max_elements, allow_obfuscate) << "]"; + ss << pretty_value(constant->cast_vector(max_elements), allow_obfuscate); break; case ov::element::Type_t::i8: case ov::element::Type_t::i16: case ov::element::Type_t::i32: case ov::element::Type_t::i64: - ss << "[" << pretty_value(constant->cast_vector(), max_elements, allow_obfuscate) << "]"; + ss << pretty_value(constant->cast_vector(max_elements), allow_obfuscate); break; case ov::element::Type_t::boolean: case ov::element::Type_t::u8: case ov::element::Type_t::u16: case ov::element::Type_t::u32: case ov::element::Type_t::u64: - ss << "[" << pretty_value(constant->cast_vector(), max_elements, allow_obfuscate) << "]"; + ss << pretty_value(constant->cast_vector(max_elements), allow_obfuscate); break; } + const auto num_elements_in_constant = static_cast(shape_size(constant->get_shape())); + if (num_elements_in_constant == 0) + ss << "empty"; + else if (max_elements == 0) + ss << "suppressed"; + else if (num_elements_in_constant > max_elements) + ss << ", ..."; + ss << " ]"; return ss.str(); } @@ -418,12 +422,9 @@ static std::string get_bounds_and_label_info(const ov::Output output) if (size == 0) { label << "empty"; } else { - static const int const_max_elements = ov::util::getenv_int("OV_VISUALIZE_TREE_CONST_MAX_ELEMENTS", 7); - label << " lower: " - << (lower ? get_value(std::make_shared(lower), const_max_elements, true) : "NONE"); - label << " upper: " - << (upper ? get_value(std::make_shared(upper), const_max_elements, true) : "NONE"); - label << " label: " << (value_label.empty() ? "NONE" : pretty_value(value_label, const_max_elements)); + label << " lower: " << (lower ? get_value(std::make_shared(lower), true) : "NONE"); + label << " upper: " << (upper ? get_value(std::make_shared(upper), true) : "NONE"); + label << " label: " << (value_label.empty() ? "NONE" : pretty_value(value_label)); } return label.str(); } diff --git a/src/core/src/runtime/host_tensor.cpp b/src/core/src/runtime/host_tensor.cpp index 712b6df02203c0..28ad2526b82529 100644 --- a/src/core/src/runtime/host_tensor.cpp +++ b/src/core/src/runtime/host_tensor.cpp @@ -139,11 +139,13 @@ void runtime::HostTensor::set_element_type(const element::Type& element_type) { } void runtime::HostTensor::set_shape(const Shape& shape) { - NGRAPH_CHECK(PartialShape(shape).refines(get_partial_shape()), - "Allocation shape ", - shape, - " must be compatible with the partial shape: ", - get_partial_shape()); + NGRAPH_CHECK( + PartialShape(shape).refines(get_partial_shape()) || + (m_descriptor->m_partial_shape.is_static() && m_descriptor->m_partial_shape.to_shape() == ov::Shape{0}), + "Allocation shape ", + shape, + " must be compatible with the partial shape: ", + get_partial_shape()); m_descriptor->m_partial_shape = shape; m_descriptor->m_shape_changed = true; } diff --git a/src/core/src/runtime/itensor.cpp b/src/core/src/runtime/itensor.cpp index 9cde89a8cc1091..9149e3cb5c9bf2 100644 --- a/src/core/src/runtime/itensor.cpp +++ b/src/core/src/runtime/itensor.cpp @@ -10,6 +10,7 @@ #include "openvino/runtime/allocator.hpp" #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/properties.hpp" +#include "shape_util.hpp" namespace ov { @@ -63,8 +64,10 @@ void ITensor::copy_to(const std::shared_ptr& dst) const { " != dst: ", dst->get_element_type(), ")"); - if (dst->get_shape() == ov::Shape{0}) + OPENVINO_SUPPRESS_DEPRECATED_START + if (dst->get_shape() == ov::Shape{0} || ov::util::is_dynamic_shape(dst->get_shape())) dst->set_shape(get_shape()); + OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_ASSERT(shapes_equal(get_shape(), dst->get_shape()), "Tensor shapes are not equal. (src: ", get_shape(), diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index e42513bd939ebb..b5298b9e5eea3c 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -11,8 +11,6 @@ ov_try_use_gold_linker() add_definitions(-DSERIALIZED_ZOO=\"${TEST_MODEL_ZOO}/core/models\") -message(STATUS "OpenVINO Core unit tests enabled") - # For type relaxed types set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/threading.cpp PROPERTIES INCLUDE_DIRECTORIES $) diff --git a/src/core/tests/constant.cpp b/src/core/tests/constant.cpp index 810affda0a6c36..45ad60d153627a 100644 --- a/src/core/tests/constant.cpp +++ b/src/core/tests/constant.cpp @@ -1779,3 +1779,43 @@ TEST(constant, lazy_bitwise_identical) { // '10' times is guaranteed to be faster here (typical value is ~200'000) EXPECT_GT(bitwise_check_count_only, bitwise_check_count * 10); } + +TEST(constant, cast_vector) { + std::vector types = {element::boolean, + element::bf16, + element::f16, + element::f32, + element::f64, + element::i4, + element::i8, + element::i16, + element::i32, + element::i64, + element::u1, + element::u4, + element::u8, + element::u16, + element::u32, + element::u64}; + std::vector data = {0, 1, 0, 0, 1, 1, 0, 1}; + std::vector expected_partial_data = {0, 1, 0, 0, 1, 1}; + + for (const auto& type : types) { + const auto& constant = op::v0::Constant::create(type, Shape{data.size()}, data); + + const auto& default_casted = constant->cast_vector(); + EXPECT_EQ(default_casted, data) << "Constant::cast_vector failed default casting for type " << type; + + int64_t num_elements_for_partial_casting = static_cast(expected_partial_data.size()); + const auto& partially_casted = constant->cast_vector(num_elements_for_partial_casting); + EXPECT_EQ(partially_casted, expected_partial_data) + << "Constant::cast_vector failed partial casting for type " << type; + + int64_t num_elements_for_over_casting = static_cast(data.size()) + 10; + const auto& over_casted = constant->cast_vector(num_elements_for_over_casting); + EXPECT_EQ(over_casted, data) << "Constant::cast_vector failed for partial casting for type " << type; + + EXPECT_TRUE(constant->cast_vector(0).empty()) + << "Constant::cast_vector failed empty casting for type " << type; + } +} diff --git a/src/core/tests/type_prop/slice.cpp b/src/core/tests/type_prop/slice.cpp index b4ef1dcff27115..d62ce5001f928c 100644 --- a/src/core/tests/type_prop/slice.cpp +++ b/src/core/tests/type_prop/slice.cpp @@ -243,7 +243,7 @@ TEST(type_prop, slice_v8_basic_param_inputs_default_axes_labels_prop) { EXPECT_EQ(op->get_element_type(), et); EXPECT_EQ(op->get_output_partial_shape(0), expected_out_shape); EXPECT_THAT(get_shape_labels(op->get_output_partial_shape(0)), - ElementsAre(10, ov::no_label, ov::no_label, ov::no_label, 14, ov::no_label, 16, 17, 18)); + ElementsAre(10, ov::no_label, ov::no_label, ov::no_label, ov::no_label, ov::no_label, 16, 17, 18)); } TEST(type_prop, slice_v8_sss_param_inputs_mixed_neg_const_axes) { diff --git a/src/frontends/pytorch/src/op/min_max.cpp b/src/frontends/pytorch/src/op/min_max.cpp index ad5c8224d3e143..c6b63f11e25d9f 100644 --- a/src/frontends/pytorch/src/op/min_max.cpp +++ b/src/frontends/pytorch/src/op/min_max.cpp @@ -33,6 +33,7 @@ OutputVector translate_max(const NodeContext& context) { // torch.max(input, other) if (context.input_is_none(2)) { auto y = context.get_input(1); + align_eltwise_input_types(context, x, y, true); return {context.mark_node(std::make_shared(x, y))}; } // torch.max(input, dim, keepdim), returns values and indicies @@ -62,6 +63,7 @@ OutputVector translate_min(const NodeContext& context) { // torch.min(input, other) if (context.input_is_none(2)) { auto y = context.get_input(1); + align_eltwise_input_types(context, x, y, true); return {context.mark_node(std::make_shared(x, y))}; } // torch.min(input, dim, keepdim), returns values and indicies diff --git a/src/frontends/pytorch/src/op/repeat.cpp b/src/frontends/pytorch/src/op/repeat.cpp deleted file mode 100644 index 15dc03a466ec92..00000000000000 --- a/src/frontends/pytorch/src/op/repeat.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "openvino/frontend/pytorch/node_context.hpp" -#include "openvino/op/broadcast.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/shape_of.hpp" -#include "openvino/op/tile.hpp" -#include "utils.hpp" - -namespace ov { -namespace frontend { -namespace pytorch { -namespace op { - -using namespace ov::op; - -OutputVector translate_repeat(const NodeContext& context) { - num_inputs_check(context, 2, 2); - auto x = context.get_input(0); - auto repeats = context.get_input(1); - auto one = context.mark_node(v0::Constant::create(element::i32, Shape{}, {1})); - auto sizes_shape = context.mark_node(std::make_shared(repeats, element::i32)); - auto expand_shape = context.mark_node(std::make_shared(one, sizes_shape)); - auto expanded_input = - context.mark_node(std::make_shared(x, expand_shape, BroadcastType::BIDIRECTIONAL)); - return {context.mark_node(std::make_shared(expanded_input, repeats))}; -}; - -} // namespace op -} // namespace pytorch -} // namespace frontend -} // namespace ov \ No newline at end of file diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp index 87358dd4d42ee3..4dd60e01b71d7f 100644 --- a/src/frontends/pytorch/src/op_table.cpp +++ b/src/frontends/pytorch/src/op_table.cpp @@ -134,7 +134,6 @@ OP_CONVERTER(translate_randn_like); OP_CONVERTER(translate_reciprocal); OP_CONVERTER(translate_relu6); OP_CONVERTER(translate_remainder); -OP_CONVERTER(translate_repeat); OP_CONVERTER(translate_repeat_interleave); OP_CONVERTER(translate_reshape); OP_CONVERTER(translate_reshape_as); @@ -393,7 +392,7 @@ const std::map get_supported_ops_ts() { {"aten::relu_", op::inplace_op>}, {"aten::relu6", op::translate_relu6}, {"aten::remainder", op::translate_remainder}, - {"aten::repeat", op::translate_repeat}, + {"aten::repeat", op::inplace_op>}, {"aten::repeat_interleave", op::translate_repeat_interleave}, {"aten::reshape", op::translate_reshape}, {"aten::reshape_as", op::translate_reshape_as}, diff --git a/src/frontends/pytorch/src/utils.cpp b/src/frontends/pytorch/src/utils.cpp index 386ac393d35772..afd9b5ebf85216 100644 --- a/src/frontends/pytorch/src/utils.cpp +++ b/src/frontends/pytorch/src/utils.cpp @@ -422,9 +422,9 @@ void align_eltwise_input_types(const NodeContext& context, Output& lhs, Ou // if div we need to also align float types to highest bitness regardless of scalar if (!align_scalars) lhs_dst_type = element::f32; - rhs_dst_type = element::f32; + rhs_dst_type = lhs_type; } else if (is_rhs_scalar && !lhs_type.is_real() && rhs_type.is_real()) { - lhs_dst_type = element::f32; + lhs_dst_type = rhs_type; // if div we need to also align float types to highest bitness regardless of scalar if (!align_scalars) rhs_dst_type = element::f32; @@ -437,9 +437,9 @@ void align_eltwise_input_types(const NodeContext& context, Output& lhs, Ou } if (!lhs_dst_type.is_real() && rhs_dst_type.is_real()) { - lhs_dst_type = element::f32; + lhs_dst_type = rhs_dst_type; } else if (lhs_dst_type.is_real() && !rhs_dst_type.is_real()) { - rhs_dst_type = element::f32; + rhs_dst_type = lhs_dst_type; } // Align bool to other type if (lhs_dst_type == element::boolean) { diff --git a/src/frontends/tensorflow/src/input_model.cpp b/src/frontends/tensorflow/src/input_model.cpp index 9ed378b9373762..6e2103ca463a10 100644 --- a/src/frontends/tensorflow/src/input_model.cpp +++ b/src/frontends/tensorflow/src/input_model.cpp @@ -259,15 +259,26 @@ void InputModel::InputModelTFImpl::load_places() { return; } - // treat terminal nodes as the models outputs for the frozen TF1 format - std::set op_names_without_consumers; - std::set_difference(all_op_names.begin(), - all_op_names.end(), - op_names_with_consumers.begin(), - op_names_with_consumers.end(), - std::inserter(op_names_without_consumers, op_names_without_consumers.begin())); - - for (const auto& output_name : op_names_without_consumers) { + auto out_names = m_graph_iterator->get_output_names(); + if (!out_names.size()) { + // treat terminal nodes as the models outputs for the frozen TF1 format + std::set op_names_without_consumers; + std::set_difference(all_op_names.begin(), + all_op_names.end(), + op_names_with_consumers.begin(), + op_names_with_consumers.end(), + std::inserter(op_names_without_consumers, op_names_without_consumers.begin())); + for (const auto& output_name : op_names_without_consumers) { + auto output_place = std::make_shared(m_input_model, + ov::PartialShape({}), + ov::element::dynamic, + std::vector{output_name}); + m_tensor_places[output_name] = output_place; + m_outputs.push_back(output_place); + } + return; + } + for (const auto& output_name : out_names) { auto output_place = std::make_shared(m_input_model, ov::PartialShape({}), ov::element::dynamic, @@ -312,7 +323,6 @@ std::vector> InputModel::InputModelTFImpl::topologicall // TODO: implement logic to check direct cycles in the graph // and break them // probably not only NextIteration can generate cycles - for (const auto& output_place : m_outputs) { FRONT_END_GENERAL_CHECK(output_place->get_names().size() > 0, "TensorPlace must have at least one name."); auto output_place_name = output_place->get_names()[0]; diff --git a/src/frontends/tensorflow/src/translate_session.cpp b/src/frontends/tensorflow/src/translate_session.cpp index ebb0dee99923fa..933bc0d900eba5 100644 --- a/src/frontends/tensorflow/src/translate_session.cpp +++ b/src/frontends/tensorflow/src/translate_session.cpp @@ -25,17 +25,24 @@ std::vector reorder_ops_by_names(const std::vector& names, const "[TensorFlow Frontend] Internal error: cannot perform reordering of operations. The number " "of names mismatches the number of operations."); std::vector resulted_ops(ops.size(), nullptr); + for (const auto& op : ops) { const auto& op_name = op->get_friendly_name(); auto iter = std::find(names.begin(), names.end(), op_name); FRONT_END_GENERAL_CHECK(iter != names.end(), "[TensorFlow Frontend] Internal error: cannot perform reordering of operations. The " "requested name is not found among operations."); - auto ind = std::distance(names.begin(), iter); - FRONT_END_GENERAL_CHECK(resulted_ops[ind] == nullptr, - "[TensorFlow Frontend] Internal error: incorrect reordering of operations. " - "Found two operations that are mapped to the same name."); - resulted_ops[ind] = op; + } + + size_t ind = 0; + for (const auto& name : names) { + for (const auto& op : ops) { + if (op->get_friendly_name() == name) { + resulted_ops[ind] = op; + ind++; + break; + } + } } return resulted_ops; }; diff --git a/src/inference/src/cnn_network_ngraph_impl.cpp b/src/inference/src/cnn_network_ngraph_impl.cpp index cb26e63e7c2626..74f80f39e2f641 100644 --- a/src/inference/src/cnn_network_ngraph_impl.cpp +++ b/src/inference/src/cnn_network_ngraph_impl.cpp @@ -29,7 +29,6 @@ #include "transformations/common_optimizations/nop_elimination.hpp" #include "transformations/common_optimizations/remove_concat_zero_dim_input.hpp" #include "transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.hpp" -#include "transformations/fix_rt_info.hpp" #include "transformations/smart_reshape/set_batch_size.hpp" #include "transformations/smart_reshape/smart_reshape.hpp" #include "transformations/utils/utils.hpp" @@ -136,7 +135,6 @@ CNNNetworkNGraphImpl::CNNNetworkNGraphImpl(const std::shared_ptr& nGra { ov::pass::Manager m; using namespace ov::pass; - REGISTER_PASS(m, FixRtInfo) REGISTER_PASS(m, EliminateScatterUpdate) REGISTER_PASS(m, RemoveConcatZeroDimInput) REGISTER_PASS(m, RemoveMultiSubGraphOpDanglingParamsResults) @@ -199,12 +197,6 @@ CNNNetworkNGraphImpl::CNNNetworkNGraphImpl(const CNNNetwork& network) { } _ngraph_function = ngraph::clone_function(*network.getFunction()); - { - ov::pass::Manager m; - using namespace ov::pass; - REGISTER_PASS(m, FixRtInfo) - m.run_passes(_ngraph_function); - } validateFunctionNames(); InputsDataMap inputs = network.getInputsInfo(); OutputsDataMap outputs = network.getOutputsInfo(); diff --git a/src/inference/src/compilation_context.cpp b/src/inference/src/compilation_context.cpp index e4a373f1c0223b..7c0b9cfba869c6 100644 --- a/src/inference/src/compilation_context.cpp +++ b/src/inference/src/compilation_context.cpp @@ -18,7 +18,6 @@ #include "itt.hpp" #include "ngraph/opsets/opset6.hpp" #include "openvino/pass/manager.hpp" -#include "transformations/fix_rt_info.hpp" #include "transformations/hash.hpp" #include "transformations/rt_info/fused_names_attribute.hpp" #include "transformations/rt_info/primitives_priority_attribute.hpp" @@ -86,7 +85,6 @@ std::string ModelCache::compute_hash(const std::shared_ptr& mod uint64_t seed = 0; // 1. Calculate hash on function ov::pass::Manager m; - m.register_pass(); m.register_pass(seed); m.run_passes(std::const_pointer_cast(model)); diff --git a/src/inference/src/dev/make_tensor.cpp b/src/inference/src/dev/make_tensor.cpp index ddd0ea293966c0..94ddf723eabc4c 100644 --- a/src/inference/src/dev/make_tensor.cpp +++ b/src/inference/src/dev/make_tensor.cpp @@ -188,6 +188,8 @@ class AllocatedTensor : public ViewTensor { } void set_shape(ov::Shape new_shape) override { + if (m_shape == new_shape) + return; auto old_byte_size = get_byte_size(); m_shape = std::move(new_shape); if (get_byte_size() > old_byte_size) { diff --git a/src/inference/tests/unit/compilation_context_test.cpp b/src/inference/tests/unit/compilation_context_test.cpp index 1f3d5266729c61..2070f2302e29ef 100644 --- a/src/inference/tests/unit/compilation_context_test.cpp +++ b/src/inference/tests/unit/compilation_context_test.cpp @@ -189,8 +189,7 @@ TEST(NetworkContext, HashWithPrimitivesPriority) { op3["PrimitivesPriority"] = "testPriority"; ASSERT_NE(ModelCache::compute_hash(net1, {}), ModelCache::compute_hash(net2, {})); - - ASSERT_EQ(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); + ASSERT_NE(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); } TEST(NetworkContext, HashWithFusedNames) { diff --git a/src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.cpp b/src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.cpp index ce09fb071ea4ce..67159c582b0d33 100644 --- a/src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.cpp +++ b/src/plugins/intel_cpu/src/emitters/x64/jit_load_store_emitters.cpp @@ -234,6 +234,18 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 ®, int o has_xmm_block = true; } + // Cornerstone of partial load is combinaion of vpinsrb/w/d. + // As vpinsrb/w/d will not only write(insert) values into vmm, but also read values in vmm to copy from to positions that not in imm mask, + // this could introduce RAW false dependency(we actually do not care about values not in imm mask). + // To eliminate this false dependency, + // 1. For 1/2/3/4 bytes tails, replace vpinsrb/w/d with mov,shl etc instructions that don't read vmm. + // Besides eliminate RAW, these instructions have smaller latency, which also bring better perf, especially for small loop iteration case. + // 2. For 8/16 bytes, use vmovq/vmovdqu instructions to load, which also don't read src vmm. + // 3. For other size, insert vpxor before vpinsrb/w/d. vpxor and read vmm instructions in previous loop have WAR(write after read) relationship. + // CPU can identify this scenario and assign another physical vector register(register renameing) in next loop to eliminate RAW. + if (!one_of(bytes_to_load, 0, 1, 2, 3, 4, 8, 16)) { + h->uni_vpxor(vmm, vmm, vmm); + } if (bytes_to_load >= 8 && bytes_to_load < 16) h->uni_vmovq(xmm, addr(start_bytes)); else if (bytes_to_load == 16) diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index aa8acb24267c42..ed18c01c848af1 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -1308,17 +1308,16 @@ class UpdateNodes : public UpdateNodesBase { auto startCounter = m_prepareCounter.load(); #pragma omp parallel - #pragma omp single + #pragma omp sections { - #pragma omp task + #pragma omp section { updateDynParams(startCounter, stopIndx); } - #pragma omp task + #pragma omp section { updateShapes(startCounter, stopIndx); } - #pragma omp taskwait } } }; diff --git a/src/plugins/intel_cpu/src/mlas/sgemm.cpp b/src/plugins/intel_cpu/src/mlas/sgemm.cpp index 2b3ebf9ecf9a9f..b2c868ddb1ba55 100644 --- a/src/plugins/intel_cpu/src/mlas/sgemm.cpp +++ b/src/plugins/intel_cpu/src/mlas/sgemm.cpp @@ -54,7 +54,7 @@ void mlas_sgemm(const char* transa, sgemmParam.beta = beta; auto _transa = *transa == 'N' ? CblasNoTrans : CblasTrans; auto _transb = *transb == 'N' ? CblasNoTrans : CblasTrans; - ov::cpu::OVMlasThreadPool threadPool(0 == thread_num ? parallel_get_num_threads() : thread_num); + ov::cpu::OVMlasThreadPool threadPool(0 == thread_num ? parallel_get_max_threads() : thread_num); MlasGemmBatch(_transa, _transb, M, N, K, &sgemmParam, 1, &threadPool); } @@ -74,7 +74,7 @@ void mlas_sgemm_compute(const char* transa, const float* bias, size_t thread_num) { // C = alpha*op( A )op( B ) + beta * C - ov::cpu::OVMlasThreadPool threadPool(0 == thread_num ? parallel_get_num_threads() : thread_num); + ov::cpu::OVMlasThreadPool threadPool(0 == thread_num ? parallel_get_max_threads() : thread_num); MLAS_SGEMM_DATA_PARAMS sgemmParam; sgemmParam.BIsPacked = true; sgemmParam.A = A; diff --git a/src/plugins/intel_cpu/src/nodes/mha.cpp b/src/plugins/intel_cpu/src/nodes/mha.cpp index d4405be5d468d0..19cf63ac727cba 100644 --- a/src/plugins/intel_cpu/src/nodes/mha.cpp +++ b/src/plugins/intel_cpu/src/nodes/mha.cpp @@ -16,6 +16,7 @@ #include "common/cpu_convert.h" #include "transformations/cpu_opset/x64/op/mha.hpp" #include "dnnl_extension_utils.h" +#include "utils/bfloat16.hpp" #include using namespace InferenceEngine; diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp index c9b761470a133e..d941c247a005e8 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/ngram_fusion.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,12 @@ ov::intel_cpu::NgramFusion::NgramFusion() { auto idces_match = [](ov::Output output) -> bool { return ov::pass::pattern::rank_equals(2)(output) && ov::pass::pattern::type_matches(ov::element::i32)(output); }; + auto as_is_cropped_shape_match = [](ov::Output output) -> bool { + const auto& labels = output.get_tensor().get_value_label(); + return ov::pass::pattern::rank_equals(1)(output) && !labels.empty() && labels[0] != ov::no_label; + }; + + ov::label_t cropped_shape_label = ov::no_label; ov::Output tokens; // "as_is" input validation @@ -67,14 +74,11 @@ ov::intel_cpu::NgramFusion::NgramFusion() { ? ov::OutputVector{tokens_m, wrap_type()} : ov::OutputVector{wrap_type(), tokens_m, wrap_type()}; auto padded_tokens_m = wrap_type(padded_tokens_inputs); - auto shape_of_m = wrap_type({tokens_m}); - auto cropped_shape_m = wrap_type({shape_of_m, wrap_type(), wrap_type(), wrap_type()}); - + auto cropped_shape_m = any_input(as_is_cropped_shape_match); auto ss_bias = wrap_type(); auto ss_biased_shape_m = as_is_idx == 0 ? cropped_shape_m : wrap_type({cropped_shape_m, ss_bias}); - auto ss_end_reshape_m = wrap_type({ss_biased_shape_m, any_input()}); - auto cropped_tokens_m = wrap_type({padded_tokens_m, wrap_type(), ss_end_reshape_m, wrap_type()}); + auto cropped_tokens_m = wrap_type({padded_tokens_m, wrap_type(), ss_biased_shape_m, wrap_type()}); Matcher matcher(cropped_tokens_m); if (!matcher.match(inputs[as_is_idx])) { @@ -93,15 +97,27 @@ ov::intel_cpu::NgramFusion::NgramFusion() { concat_shape.rank() != tokens_shape.rank() || tokens_shape[1] * k != concat_shape[1]) { return false; } + // save label of cropped_shape and check it against first dimension of tokens shape + cropped_shape_label = pattern_map.at(cropped_shape_m).get_tensor().get_value_label()[0]; + if (ov::DimensionTracker::get_label(tokens_shape[0]) != cropped_shape_label) + return false; } + auto cropped_shape_label_match = [cropped_shape_label](ov::Output output) -> bool { + const auto& labels = output.get_tensor().get_value_label(); + return ov::pass::pattern::rank_equals(1)(output) && !labels.empty() && labels[0] != ov::no_label && labels[0] == cropped_shape_label; + }; + + auto tokens_label_match = [tokens_match, cropped_shape_label](ov::Output output) -> bool { + return tokens_match(output) && ov::DimensionTracker::get_label(output.get_partial_shape()[0]) == cropped_shape_label; + }; + ov::Output indices; // select branches validation { using namespace ov::opset1; - auto tokens_m = any_input(tokens_match); - auto shape_of_m = wrap_type({tokens_m}); - auto cropped_shape_m = wrap_type({shape_of_m, wrap_type(), wrap_type(), wrap_type()}); + auto tokens_m = any_input(tokens_label_match); + auto cropped_shape_m = any_input(cropped_shape_label_match); auto idces_m = any_input(idces_match); auto idces_concat_inputs = as_is_idx == 0 ? ov::OutputVector{idces_m, wrap_type()} @@ -111,17 +127,15 @@ ov::intel_cpu::NgramFusion::NgramFusion() { // left equal branch auto crop_left_bias_m = wrap_type(); auto crop_left_cropped_shape_m = std::make_shared( - ov::OutputVector{cropped_shape_m, wrap_type({cropped_shape_m, crop_left_bias_m})}); - auto crop_left_reshape_m = wrap_type({crop_left_cropped_shape_m, any_input()}); - auto idxes_crop_left_concat_m = wrap_type({crop_left_reshape_m, wrap_type()}); + ov::OutputVector{wrap_type({cropped_shape_m, crop_left_bias_m}), cropped_shape_m}); + auto idxes_crop_left_concat_m = wrap_type({crop_left_cropped_shape_m, wrap_type()}); auto idxes_crop_left_m = wrap_type({idces_concat_m, wrap_type(), idxes_crop_left_concat_m, wrap_type()}); // right equal branch auto crop_right_bias_m = wrap_type(); auto crop_right_cropped_shape_m = std::make_shared( - ov::OutputVector{cropped_shape_m, wrap_type({cropped_shape_m, crop_right_bias_m})}); - auto crop_right_reshape_m = wrap_type({crop_right_cropped_shape_m, any_input()}); - auto idxes_crop_right_concat_m = wrap_type({crop_right_reshape_m, wrap_type()}); + ov::OutputVector{wrap_type({cropped_shape_m, crop_right_bias_m}), cropped_shape_m}); + auto idxes_crop_right_concat_m = wrap_type({crop_right_cropped_shape_m, wrap_type()}); auto idxes_crop_right_m = wrap_type({idces_concat_m, wrap_type(), idxes_crop_right_concat_m, wrap_type()}); auto equal_m = wrap_type({idxes_crop_left_m, idxes_crop_right_m}); @@ -135,13 +149,11 @@ ov::intel_cpu::NgramFusion::NgramFusion() { auto then_cropped_shape_bias_m = wrap_type(); auto then_cropped_shape_m = std::make_shared( - ov::OutputVector{cropped_shape_m, wrap_type({cropped_shape_m, then_cropped_shape_bias_m})}); - auto then_crop_end_reshape_m = wrap_type({then_cropped_shape_m, wrap_type()}); - auto then_m = wrap_type({padded_tokens_m, wrap_type(), then_crop_end_reshape_m, wrap_type()}); + ov::OutputVector{wrap_type({cropped_shape_m, then_cropped_shape_bias_m}), cropped_shape_m}); + auto then_m = wrap_type({padded_tokens_m, wrap_type(), then_cropped_shape_m, wrap_type()}); // else branch - auto else_target_shape_reshape_m = wrap_type({cropped_shape_m, any_input()}); - auto else_target_shape_concat_m = wrap_type({else_target_shape_reshape_m, wrap_type()}); + auto else_target_shape_concat_m = wrap_type({cropped_shape_m, wrap_type()}); auto else_m = wrap_type({wrap_type(), else_target_shape_concat_m, wrap_type()}); auto select_m = wrap_type