Merge branch 'master' into an/new_deconv_slt

CMakeLists.txt

@@ -28,6 +28,11 @@ if(POLICY CMP0091)
     cmake_policy(SET CMP0091 NEW) # Enables use of MSVC_RUNTIME_LIBRARY
 endif()
 
+# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24:
+if(POLICY CMP0135)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
 project(OpenVINO DESCRIPTION "OpenVINO toolkit")
 
 find_package(OpenVINODeveloperScripts REQUIRED

cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake

@@ -206,6 +206,8 @@ set(CMAKE_POLICY_DEFAULT_CMP0025 NEW)
 set(CMAKE_POLICY_DEFAULT_CMP0026 NEW)
 # CMake 3.0+ (2.8.12): MacOS "@rpath" in target's install name
 set(CMAKE_POLICY_DEFAULT_CMP0042 NEW)
+# CMake 3.1+: Simplify variable reference and escape sequence evaluation.
+set(CMAKE_POLICY_DEFAULT_CMP0053 NEW)
 # CMake 3.9+: `RPATH` settings on macOS do not affect `install_name`.
 set(CMAKE_POLICY_DEFAULT_CMP0068 NEW)
 # CMake 3.12+: find_package() uses <PackageName>_ROOT variables.

cmake/developer_package/api_validator/api_validator.cmake

@@ -3,25 +3,25 @@
 #
 
 if(WIN32)
-    set(PROGRAMFILES_ENV "ProgramFiles(X86)")
+    set(PROGRAMFILES_ENV "ProgramFiles\(X86\)")
 
     # check that PROGRAMFILES_ENV is defined, because in case of cross-compilation for Windows
     # we don't have such variable
-    if(DEFINED ENV{PROGRAMFILES_ENV})
+    if(DEFINED ENV{${PROGRAMFILES_ENV}})
         file(TO_CMAKE_PATH $ENV{${PROGRAMFILES_ENV}} PROGRAMFILES)
 
         set(WDK_PATHS "${PROGRAMFILES}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64"
-                    "${PROGRAMFILES}/Windows Kits/10/bin/x64")
+                      "${PROGRAMFILES}/Windows Kits/10/bin/x64")
 
         message(STATUS "Trying to find apivalidator in: ")
         foreach(wdk_path IN LISTS WDK_PATHS)
             message("    * ${wdk_path}")
         endforeach()
 
         find_host_program(ONECORE_API_VALIDATOR
-                        NAMES apivalidator
-                        PATHS ${WDK_PATHS}
-                        DOC "ApiValidator for OneCore compliance")
+                          NAMES apivalidator
+                          PATHS ${WDK_PATHS}
+                          DOC "ApiValidator for OneCore compliance")
 
         if(ONECORE_API_VALIDATOR)
             message(STATUS "Found apivalidator: ${ONECORE_API_VALIDATOR}")

cmake/developer_package/plugins/create_plugins_hpp.cmake

@@ -42,10 +42,10 @@ foreach(dev_map IN LISTS OV_DEVICE_MAPPING)
 
         # declarations
         set(OV_PLUGINS_DECLARATIONS "${OV_PLUGINS_DECLARATIONS}
-        IE_DEFINE_PLUGIN_CREATE_FUNCTION_DECLARATION(${_OV_CREATE_PLUGIN_FUNC});")
+        OV_DEFINE_PLUGIN_CREATE_FUNCTION_DECLARATION(${_OV_CREATE_PLUGIN_FUNC});")
         if(${actual_dev_name}_AS_EXTENSION)
             set(OV_PLUGINS_DECLARATIONS "${OV_PLUGINS_DECLARATIONS}
-            IE_DEFINE_EXTENSION_CREATE_FUNCTION_DECLARATION(${_OV_CREATE_EXTENSION_FUNC});")
+            OV_DEFINE_EXTENSION_CREATE_FUNCTION_DECLARATION(${_OV_CREATE_EXTENSION_FUNC});")
         else()
             set(_OV_CREATE_EXTENSION_FUNC "nullptr")
         endif()

cmake/developer_package/plugins/plugins.cmake

@@ -80,7 +80,7 @@ function(ov_add_plugin)
             if(OV_PLUGIN_AS_EXTENSION)
                 # to distinguish functions creating extensions objects
                 target_compile_definitions(${OV_PLUGIN_NAME} PRIVATE
-                    IE_CREATE_EXTENSION=CreateExtensionShared${OV_PLUGIN_DEVICE_NAME})
+                    OV_CREATE_EXTENSION=CreateExtensionShared${OV_PLUGIN_DEVICE_NAME})
             endif()
         endif()
 

cmake/developer_package/plugins/plugins.hpp.in

@@ -9,13 +9,23 @@
 
 #ifdef OPENVINO_STATIC_LIBRARY
 
-#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
+// The Macro used to create extensions for static library
+#define OV_DEFINE_EXTENSION_CREATE_FUNCTION_DECLARATION(_OV_CREATE_EXTENSION_FUNC) \
+    OPENVINO_EXTENSION_C_API void                                                  \
+    _OV_CREATE_EXTENSION_FUNC(std::vector<::ov::Extension::Ptr>& ext)
+
+// The Macro used to create plugin for static library
+#define OV_DEFINE_PLUGIN_CREATE_FUNCTION_DECLARATION(_OV_CREATE_PLUGIN_FUNC) \
+    OPENVINO_PLUGIN_API void                                                 \
+    _OV_CREATE_PLUGIN_FUNC(::std::shared_ptr<::ov::IPlugin> &plugin) noexcept(false)
 
 @OV_PLUGINS_DECLARATIONS@
 
+using CreateExtensionFunc = void(std::vector<::ov::Extension::Ptr>&);
+using CreatePluginEngineFunc = void(std::shared_ptr<::ov::IPlugin>&);
 struct Value {
-    InferenceEngine::CreatePluginEngineFunc * m_create_plugin_func;
-    InferenceEngine::CreateExtensionFunc * m_create_extension_func;
+    CreatePluginEngineFunc * m_create_plugin_func;
+    CreateExtensionFunc * m_create_extension_func;
     std::map<std::string, std::string> m_default_config;
 };
 

cmake/features.cmake

@@ -177,6 +177,9 @@ ov_dependent_option (ENABLE_SYSTEM_SNAPPY "Enables use of system version of Snap
 ov_dependent_option (ENABLE_PYTHON_PACKAGING "Enables packaging of Python API in APT / YUM" OFF
     "ENABLE_PYTHON;UNIX" OFF)
 
+ov_dependent_option (ENABLE_JS "Enables JS API building" ON
+    "NOT WIN32" OFF)
+
 ov_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF)
 
 if(NOT BUILD_SHARED_LIBS AND ENABLE_OV_TF_FRONTEND)

docs/articles_en/openvino_workflow/model_preparation.rst

@@ -15,7 +15,6 @@ Model Preparation
    Convert to OpenVINO Model <openvino_docs_OV_Converter_UG_prepare_model_convert_model_Convert_Model_IR>
    Conversion Parameters <openvino_docs_OV_Converter_UG_Conversion_Options>
    Setting Input Shapes <openvino_docs_OV_Converter_UG_prepare_model_convert_model_Converting_Model>
-   PyVision preprocessing <pytorch_vision>
 
 
 You can obtain a model in one of supported formats, **PyTorch, TensorFlow, TensorFlow Lite, ONNX, and PaddlePaddle**,

docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide.rst

@@ -13,10 +13,11 @@ Optimize Inference
    openvino_docs_OV_UG_Precision_Control
    openvino_docs_deployment_optimization_guide_latency
    openvino_docs_deployment_optimization_guide_tput
-   openvino_docs_deployment_optimization_guide_tput_advanced
+   Advanced Throughput Options <openvino_docs_deployment_optimization_guide_tput_advanced>
    openvino_docs_OV_UG_Preprocessing_Overview
    openvino_docs_deployment_optimization_guide_internals
-   openvino_docs_memory_optimization_guide
+   Optimizing memory usage <openvino_docs_memory_optimization_guide>
+
 
 .. meta::
    :description: Improving inference performance involves model and runtime 

docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/dldt_deployment_optimization_tput.rst

@@ -10,8 +10,15 @@ Optimizing for Throughput
                  simultaneously which improves the device utilization.
 
 
-As described in the section on the :doc:`latency-specific considerations <openvino_docs_deployment_optimization_guide_latency>`, one of the possible use cases is *delivering every single request at the minimal delay*.
-Throughput, on the other hand, is about inference scenarios in which potentially **large number of inference requests are served simultaneously to improve the device utilization**.
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   Advanced Throughput Options <openvino_docs_deployment_optimization_guide_tput_advanced>
+
+
+As described in the section on the :doc:`latency-specific optimizations <openvino_docs_deployment_optimization_guide_latency>`, one of the possible use cases is delivering every single request with minimal delay.
+Throughput, on the other hand, is about inference scenarios in which potentially **large numbers of inference requests are served simultaneously to improve resource use**.
 
 The associated increase in latency is not linearly dependent on the number of requests executed in parallel.
 A trade-off between overall throughput and serial performance of individual requests can be achieved with the right performance configuration of OpenVINO.

docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/dldt_deployment_optimization_tput_advanced.rst

@@ -1,14 +1,14 @@
 .. {#openvino_docs_deployment_optimization_guide_tput_advanced}
 
-Using Advanced Throughput Options: Streams and Batching
+Advanced Throughput Options: Streams and Batching
 =======================================================
 
 
 .. meta::
    :description: With OpenVINO streams a device may handle processing multiple 
                  inference requests and the batching helps to saturate the 
                  device and leads to higher throughput.
-
+ 
 
 OpenVINO Streams
 ####################

docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/precision_control.rst

@@ -9,9 +9,8 @@ The choice of data types is essential to the inference runtime, which can have a
 1. Model storage precision (IR precision),
 2. Model inference precision.
 
-Previously, these 2 precisions were interrelated, and model storage precision could affect the inference precision in some devices (e.g. GPU did ``f16`` inference only for ``f16`` IRs).
-
-With the ``2023.0`` release this behavior has been changed and the inference precision no longer depends on the precision of IR. Now users have several knobs to find the balance between model performance and accuracy.
+Inference precision no longer depends on the precision of IR, which means that users
+have several options to find the balance between model performance and accuracy.
 
 Essentially, the IR precision becomes a way of compressing the model by reducing the precision of the weights, and it does not affect how the devices execute the model. This change clears up a lot of confusion where, for example, you couldn't execute a high-performance model on the GPU by default, and the behavior between devices was different. 
 

docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/preprocessing_overview.rst

@@ -11,6 +11,7 @@ Optimize Preprocessing
    openvino_docs_OV_UG_Preprocessing_Details
    openvino_docs_OV_UG_Layout_Overview
    openvino_docs_OV_UG_Preprocess_Usecase_save
+   Torchvision preprocessing converter <torchvision_preprocessing_converter>
 
 .. meta::
    :description: The preprocessing entails additional operations to transform 

docs/articles_en/openvino_workflow/running_inference_with_openvino/dldt_deployment_optimization_guide/preprocessing_overview/torchvision_preprocessing_converter.rst

@@ -0,0 +1,71 @@
+.. {#torchvision_preprocessing_converter}
+
+Torchvision preprocessing converter
+=======================================
+
+
+.. meta::
+   :description: See how OpenVINO™ enables torchvision preprocessing
+                 to optimize model inference.
+
+
+The Torchvision-to-OpenVINO converter enables automatic translation of operators from the torchvision
+preprocessing pipeline to the OpenVINO format and embed them in your model. It is often used to adjust
+images serving as input for AI models to have proper dimensions or data types.
+
+As the converter is fully based on the **openvino.preprocess** module, you can implement the **torchvision.transforms**
+feature easily and without the use of external libraries, reducing the overall application complexity
+and enabling additional performance optimizations.
+
+
+.. note::
+
+   Not all torchvision transforms are supported yet. The following operations are available:
+
+   .. code-block::
+
+      transforms.Compose
+      transforms.Normalize
+      transforms.ConvertImageDtype
+      transforms.Grayscale
+      transforms.Pad
+      transforms.ToTensor
+      transforms.CenterCrop
+      transforms.Resize
+
+
+Example
+###################
+
+.. code-block:: py
+
+   preprocess_pipeline = torchvision.transforms.Compose(
+       [
+           torchvision.transforms.Resize(256, interpolation=transforms.InterpolationMode.NEAREST),
+           torchvision.transforms.CenterCrop((216, 218)),
+           torchvision.transforms.Pad((2, 3, 4, 5), fill=3),
+           torchvision.transforms.ToTensor(),
+           torchvision.transforms.ConvertImageDtype(torch.float32),
+           torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+       ]
+   )
+
+   torch_model = SimpleConvnet(input_channels=3)
+
+   torch.onnx.export(torch_model, torch.randn(1, 3, 224, 224), "test_convnet.onnx", verbose=False, input_names=["input"], output_names=["output"])
+   core = Core()
+   ov_model = core.read_model(model="test_convnet.onnx")
+
+   test_input = np.random.randint(255, size=(260, 260, 3), dtype=np.uint16)
+   ov_model = PreprocessConverter.from_torchvision(
+       model=ov_model, transform=preprocess_pipeline, input_example=Image.fromarray(test_input.astype("uint8"), "RGB")
+   )
+   ov_model = core.compile_model(ov_model, "CPU")
+   ov_input = np.expand_dims(test_input, axis=0)
+   output = ov_model.output(0)
+   ov_result = ov_model(ov_input)[output]
+
+
+
+
+

docs/dev/get_started.md

@@ -18,4 +18,4 @@ Explore other resources to learn more about OpenVINO:
  * [OpenVINO Developer Documentation](./index.md)
  * [OpenVINO Samples](../../samples)
  * [OpenVINO Building Documentation](./building_documentation.md)
- * [CMake Options for Custom Compilation](./cmake_options_for_custom_comiplation.md)
+ * [CMake Options for Custom Compilation](./cmake_options_for_custom_compilation.md)

docs/requirements.txt

@@ -13,7 +13,7 @@ imagesize==1.2.0
 importlib-metadata==4.4.0
 iniconfig==1.1.1
 ipython==8.10.0
-Jinja2==3.1.2
+Jinja2==3.1.3
 lxml>=4.9.2
 MarkupSafe==2.1.1
 mistune==2.0.3

src/bindings/js/CMakeLists.txt

@@ -2,6 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+if(NOT ENABLE_JS)
+    return()
+endif()
+
 project(OpenVINO_JS_API)
 
 add_subdirectory(node)

src/bindings/js/node/CMakeLists.txt

@@ -2,10 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-if(WIN32)
-    return()
-endif()
-
 if(CMAKE_VERSION VERSION_LESS 3.14)
     message(WARNING "JS API is not available with CMake version less than 3.14, skipping")
     return()

src/common/snippets/src/lowered/pass/allocate_buffers.cpp

@@ -67,18 +67,19 @@ void AllocateBuffers::set_buffer_offset(const ExpressionPtr& buffer_expr, const
 bool AllocateBuffers::run(lowered::LinearIR& linear_ir) {
     OV_ITT_SCOPED_TASK(ov::pass::itt::domains::SnippetsTransform, "Snippets::AllocateBuffers");
     m_buffer_scratchpad_size = 0;
-    PassPipeline pipeline;
+
     if (m_is_optimized_mode) {
         BufferClusters buffer_clusters;
+        PassPipeline pipeline;
         pipeline.register_pass<EnumerateExpressions>();
         pipeline.register_pass<IdentifyBuffers>();
         pipeline.register_pass<DefineBufferClusters>(buffer_clusters);
         pipeline.register_pass<SolveBufferMemory>(m_buffer_scratchpad_size, buffer_clusters);
         pipeline.register_pass<NormalizeBufferIDs>();
+        pipeline.run(linear_ir);
     } else {
-        pipeline.register_pass<InitBuffersDefault>(m_buffer_scratchpad_size);
+        InitBuffersDefault(m_buffer_scratchpad_size).run(linear_ir);
     }
-    pipeline.run(linear_ir);
 
     return m_buffer_scratchpad_size > 0;
 }

src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp

@@ -15,6 +15,7 @@ class TRANSFORMATIONS_API EnableDecompressionConvertConstantFolding;
 class TRANSFORMATIONS_API DisableDecompressionConvertConstantFolding;
 class TRANSFORMATIONS_API KeepConstAndDecompression;
 class TRANSFORMATIONS_API KeepConstantsPrecisionAndAddConverts;
+class TRANSFORMATIONS_API MarkCompressedFloatConstants;
 
 }  // namespace pass
 }  // namespace ov
@@ -58,3 +59,16 @@ class ov::pass::KeepConstantsPrecisionAndAddConverts : public MatcherPass {
     OPENVINO_RTTI("KeepConstantsPrecisionAndAddConverts", "0");
     KeepConstantsPrecisionAndAddConverts();
 };
+
+/**
+ * @ingroup ie_transformation_common_api
+ * @brief Prevents ConstantFolding for f16/bf16 Const + Convert_To_FP32 to keep original FW float Constants.
+ * Original precision should be kept as long as possible, this prevents redundant conversions and saves memory.
+ * E.g. if original FW model was already compressed no need to upcast during CF, store intermediate f32 consts and
+ * then again compress them to f16 during save_model.
+ */
+class ov::pass::MarkCompressedFloatConstants : public MatcherPass {
+public:
+    OPENVINO_RTTI("KeepFWPrecisionFor16BitFloatConstants", "0");
+    MarkCompressedFloatConstants();
+};

src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp

@@ -119,3 +119,30 @@ pass::KeepConstantsPrecisionAndAddConverts::KeepConstantsPrecisionAndAddConverts
     auto m = std::make_shared<pass::pattern::Matcher>(const_pattern, matcher_name);
     this->register_matcher(m, callback);
 }
+
+pass::MarkCompressedFloatConstants::MarkCompressedFloatConstants() {
+    MATCHER_SCOPE(MarkCompressedFloatConstants);
+
+    auto constant = pattern::wrap_type<ov::op::v0::Constant>();
+    auto convert = pattern::wrap_type<ov::op::v0::Convert>({constant});
+
+    matcher_pass_callback callback = [=](pattern::Matcher& m) {
+        const auto& convert_node = as_type_ptr<ov::op::v0::Convert>(m.get_match_root());
+        const auto& const_node = convert_node->input_value(0).get_node_shared_ptr();
+        if (convert_node == nullptr || const_node == nullptr)
+            return false;
+        if (convert_node->get_destination_type() != element::f32)
+            return false;
+        if (const_node->get_output_element_type(0) != element::f16 &&
+            const_node->get_output_element_type(0) != element::bf16)
+            return false;
+
+        mark_as_decompression(convert_node);
+        disable_constant_folding(const_node);
+        disable_constant_folding(convert_node);
+        return true;
+    };
+
+    auto m = std::make_shared<pass::pattern::Matcher>(convert, matcher_name);
+    this->register_matcher(m, callback);
+}