From 2bf8d910f641f912ed883ad2cee99f59316e9bc0 Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Wed, 13 Sep 2023 21:05:24 +0200 Subject: [PATCH] [Docs][PyOV] update python snippets (#19367) * [Docs][PyOV] update python snippets * first snippet * Fix samples debug * Fix linter * part1 * Fix speech sample * update model state snippet * add serialize * add temp dir * CPU snippets update (#134) * snippets CPU 1/6 * snippets CPU 2/6 * snippets CPU 3/6 * snippets CPU 4/6 * snippets CPU 5/6 * snippets CPU 6/6 * make module TODO: REMEMBER ABOUT EXPORTING PYTONPATH ON CIs ETC * Add static model creation in snippets for CPU * export_comp_model done * leftovers * apply comments * apply comments -- properties * small fixes * rempve debug info * return IENetwork instead of Function * apply comments * revert precision change in common snippets * update opset * [PyOV] Edit docs for the rest of plugins (#136) * modify main.py * GNA snippets * GPU snippets * AUTO snippets * MULTI snippets * HETERO snippets * Added properties * update gna * more samples * Update docs/OV_Runtime_UG/model_state_intro.md * Update docs/OV_Runtime_UG/model_state_intro.md * attempt1 fix ci * new approach to test * temporary remove some files from run * revert cmake changes * fix ci * fix snippet * fix py_exclusive snippet * fix preprocessing snippet * clean-up main * remove numpy installation in gha * check for GPU * add logger * iexclude main * main update * temp * Temp2 * Temp2 * temp * Revert temp * add property execution devices * hide output from samples --------- Co-authored-by: p-wysocki Co-authored-by: Jan Iwaszkiewicz Co-authored-by: Karol Blaszczak --- .github/workflows/linux.yml | 8 + .../shellcheck/shellcheck.cmake | 1 - .../migration_ov_2_0/graph_construction.md | 2 +- docs/OV_Runtime_UG/model_state_intro.md | 36 +++- .../109-latency-tricks-with-output.rst | 2 +- .../109-throughput-tricks-with-output.rst | 2 +- docs/notebooks/115-async-api-with-output.rst | 2 +- .../203-meter-reader-with-output.rst | 2 +- .../nncf/ptq/code/ptq_aa_openvino.py | 6 +- .../nncf/ptq/code/ptq_onnx.py | 2 +- .../nncf/ptq/code/ptq_openvino.py | 2 +- .../nncf/ptq/code/ptq_tensorflow.py | 2 +- .../nncf/ptq/code/ptq_torch.py | 2 +- docs/snippets/ShapeInference.py | 16 +- docs/snippets/__init__.py | 7 + docs/snippets/cpu/Bfloat16Inference.py | 20 +- docs/snippets/cpu/__init__.py | 2 + docs/snippets/cpu/compile_model.py | 30 +-- docs/snippets/cpu/dynamic_shape.py | 10 +- docs/snippets/cpu/multi_threading.py | 41 ++-- docs/snippets/cpu/ov_execution_mode.py | 16 +- .../cpu/ov_sparse_weights_decompression.py | 10 +- docs/snippets/export_compiled_model.py | 16 +- docs/snippets/gna/__init__.py | 2 + docs/snippets/gna/configure.py | 17 +- docs/snippets/gna/import_export.py | 15 +- docs/snippets/gna/set_batch.py | 15 +- docs/snippets/gpu/__init__.py | 2 + docs/snippets/gpu/compile_model.py | 83 ++++---- docs/snippets/gpu/custom_kernels_api.py | 17 +- docs/snippets/gpu/dynamic_batch.py | 43 +++-- .../gpu/preprocessing_nv12_two_planes.py | 43 +++-- docs/snippets/ie_common.py | 18 +- docs/snippets/main.py | 44 +++++ .../snippets/{ngraph.py => ngraph_snippet.py} | 0 docs/snippets/ov_auto.py | 181 ++++++++++++------ docs/snippets/ov_auto_batching.py | 96 ++++++---- docs/snippets/ov_caching.py | 26 +-- docs/snippets/ov_common.py | 19 +- docs/snippets/ov_denormals.py | 7 +- docs/snippets/ov_dynamic_shapes.py | 42 ++-- docs/snippets/ov_extensions.py | 10 +- docs/snippets/ov_graph.py | 11 +- docs/snippets/ov_hetero.py | 93 ++++----- docs/snippets/ov_infer_request.py | 36 ++-- docs/snippets/ov_layout.py | 17 +- docs/snippets/ov_model_snippets.py | 45 +++-- docs/snippets/ov_model_with_state_infer.py | 84 ++++++++ docs/snippets/ov_multi.py | 109 ++++++----- docs/snippets/ov_network_state_intro.py | 172 ----------------- docs/snippets/ov_preprocessing.py | 99 +++++----- docs/snippets/ov_preprocessing_migration.py | 55 +++--- docs/snippets/ov_properties_api.py | 109 ++++++----- docs/snippets/ov_properties_migration.py | 163 ++++++++-------- docs/snippets/ov_python_exclusives.py | 52 ++--- docs/snippets/ov_python_inference.py | 13 +- docs/snippets/src/main.py | 2 +- docs/snippets/utils.py | 86 +++++++++ .../bert_benchmark/bert_benchmark.py | 9 +- .../sync_benchmark/sync_benchmark.py | 5 +- .../throughput_benchmark.py | 7 +- .../classification_sample_async.py | 19 +- .../hello_classification.py | 17 +- .../hello_query_device/hello_query_device.py | 4 +- .../hello_reshape_ssd/hello_reshape_ssd.py | 17 +- .../model_creation_sample.py | 45 +++-- samples/python/speech_sample/speech_sample.py | 19 +- src/bindings/python/docs/test_examples.md | 2 +- 68 files changed, 1269 insertions(+), 938 deletions(-) create mode 100644 docs/snippets/__init__.py create mode 100644 docs/snippets/cpu/__init__.py create mode 100644 docs/snippets/gna/__init__.py create mode 100644 docs/snippets/gpu/__init__.py create mode 100644 docs/snippets/main.py rename docs/snippets/{ngraph.py => ngraph_snippet.py} (100%) create mode 100644 docs/snippets/ov_model_with_state_infer.py delete mode 100644 docs/snippets/ov_network_state_intro.py create mode 100644 docs/snippets/utils.py diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ff109f5d5bc43f..22d9037988634b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -599,6 +599,14 @@ jobs: --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py \ --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_zoo_models.py \ --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_backend.py + + - name: Python API snippets + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:${{ github.workspace }}/openvino/docs/:$PYTHONPATH + export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH + + python3 ${{ github.workspace }}/openvino/docs/snippets/main.py - name: Model Optimizer UT run: | diff --git a/cmake/developer_package/shellcheck/shellcheck.cmake b/cmake/developer_package/shellcheck/shellcheck.cmake index b792a8e6c1a112..3b18ff02abf359 100644 --- a/cmake/developer_package/shellcheck/shellcheck.cmake +++ b/cmake/developer_package/shellcheck/shellcheck.cmake @@ -39,7 +39,6 @@ function(ie_shellcheck_process) continue() endif() - get_filename_component(dir_name "${script}" DIRECTORY) string(REPLACE "${IE_SHELLCHECK_DIRECTORY}" "${CMAKE_BINARY_DIR}/shellcheck" output_file ${script}) set(output_file "${output_file}.txt") get_filename_component(script_name "${script}" NAME) diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md index 17ceb9a55ecb81..9dc0a937b07e08 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md @@ -20,7 +20,7 @@ nGraph API .. tab-item:: Python :sync: py - .. doxygensnippet:: docs/snippets/ngraph.py + .. doxygensnippet:: docs/snippets/ngraph_snippet.py :language: Python :fragment: ngraph:graph diff --git a/docs/OV_Runtime_UG/model_state_intro.md b/docs/OV_Runtime_UG/model_state_intro.md index eca22cc6eb95f1..60acfaa79e730a 100644 --- a/docs/OV_Runtime_UG/model_state_intro.md +++ b/docs/OV_Runtime_UG/model_state_intro.md @@ -163,9 +163,21 @@ Example of Creating Model OpenVINO API In the following example, the ``SinkVector`` is used to create the ``ov::Model``. For a model with states, except inputs and outputs, the ``Assign`` nodes should also point to the ``Model`` to avoid deleting it during graph transformations. Use the constructor to do it, as shown in the example, or with the special ``add_sinks(const SinkVector& sinks)`` method. After deleting the node from the graph with the ``delete_sink()`` method, a sink can be deleted from ``ov::Model``. -.. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp - :language: cpp - :fragment: [model_create] +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp + :language: cpp + :fragment: [model_create] + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.py + :language: python + :fragment: ov:model_create .. _openvino-state-api: @@ -189,10 +201,22 @@ Based on the IR from the previous section, the example below demonstrates infere One infer request and one thread will be used in this example. Using several threads is possible if there are several independent sequences. Then, each sequence can be processed in its own infer request. Inference of one sequence in several infer requests is not recommended. In one infer request, a state will be saved automatically between inferences, but if the first step is done in one infer request and the second in another, a state should be set in a new infer request manually (using the ``ov::IVariableState::set_state`` method). +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp + :language: cpp + :fragment: [part1] + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.py + :language: python + :fragment: ov:part1 -.. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp - :language: cpp - :fragment: [part1] For more elaborate examples demonstrating how to work with models with states, diff --git a/docs/notebooks/109-latency-tricks-with-output.rst b/docs/notebooks/109-latency-tricks-with-output.rst index 4ce64f56c5afa5..af706167522e3b 100644 --- a/docs/notebooks/109-latency-tricks-with-output.rst +++ b/docs/notebooks/109-latency-tricks-with-output.rst @@ -206,7 +206,7 @@ benchmarking process. .. code:: ipython3 - import openvino.runtime as ov + import openvino as ov # initialize OpenVINO core = ov.Core() diff --git a/docs/notebooks/109-throughput-tricks-with-output.rst b/docs/notebooks/109-throughput-tricks-with-output.rst index c9259e4b395f1b..523bb307ed2cf3 100644 --- a/docs/notebooks/109-throughput-tricks-with-output.rst +++ b/docs/notebooks/109-throughput-tricks-with-output.rst @@ -193,7 +193,7 @@ benchmarking process. .. code:: ipython3 - import openvino.runtime as ov + import openvino as ov # initialize OpenVINO core = ov.Core() diff --git a/docs/notebooks/115-async-api-with-output.rst b/docs/notebooks/115-async-api-with-output.rst index 06a43e3aeb2550..f8b9eb906a5a50 100644 --- a/docs/notebooks/115-async-api-with-output.rst +++ b/docs/notebooks/115-async-api-with-output.rst @@ -54,7 +54,7 @@ Imports `⇑ <#top>`__ import time import numpy as np from openvino.runtime import Core, AsyncInferQueue - import openvino.runtime as ov + import openvino as ov from IPython import display import matplotlib.pyplot as plt diff --git a/docs/notebooks/203-meter-reader-with-output.rst b/docs/notebooks/203-meter-reader-with-output.rst index e426fec36bd693..0efdff356edd3e 100644 --- a/docs/notebooks/203-meter-reader-with-output.rst +++ b/docs/notebooks/203-meter-reader-with-output.rst @@ -54,7 +54,7 @@ Import `⇑ <#top>`__ import cv2 import tarfile import matplotlib.pyplot as plt - import openvino.runtime as ov + import openvino as ov sys.path.append("../utils") from notebook_utils import download_file, segmentation_map_to_image diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py b/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py index d759695ea8a4ba..7800d47b89a1f3 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py @@ -21,7 +21,7 @@ def transform_fn(data_item): import openvino from sklearn.metrics import accuracy_score -def validate(model: openvino.runtime.CompiledModel, +def validate(model: openvino.CompiledModel, validation_loader: torch.utils.data.DataLoader) -> float: predictions = [] references = [] @@ -39,7 +39,7 @@ def validate(model: openvino.runtime.CompiledModel, #! [validation] #! [quantization] -model = ... # openvino.runtime.Model object +model = ... # openvino.Model object quantized_model = nncf.quantize_with_accuracy_control(model, calibration_dataset=calibration_dataset, @@ -50,7 +50,7 @@ def validate(model: openvino.runtime.CompiledModel, #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov # compile the model to transform quantized operations to int8 model_int8 = ov.compile_model(quantized_model) diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py b/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py index b8104d449c566f..9588886bd09cf8 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py @@ -23,7 +23,7 @@ def transform_fn(data_item): #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov from openvino.tools.mo import convert_model # convert ONNX model to OpenVINO model diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py b/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py index 81e19fd99f5c8b..1a3b10df219e1b 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py @@ -15,7 +15,7 @@ def transform_fn(data_item): #! [dataset] #! [quantization] -import openvino.runtime as ov +import openvino as ov model = ov.Core().read_model("model_path") quantized_model = nncf.quantize(model, calibration_dataset) diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py b/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py index e4fc817ddb31d8..f605a5d8737468 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py @@ -22,7 +22,7 @@ def transform_fn(data_item): #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov from openvino.tools.mo import convert_model # convert TensorFlow model to OpenVINO model diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_torch.py b/docs/optimization_guide/nncf/ptq/code/ptq_torch.py index fbdf103b4f2bf9..b0147631c667ae 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_torch.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_torch.py @@ -22,7 +22,7 @@ def transform_fn(data_item): #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov from openvino.tools.mo import convert_model input_fp32 = ... # FP32 model input diff --git a/docs/snippets/ShapeInference.py b/docs/snippets/ShapeInference.py index f680c8053b25fe..e74565696db80a 100644 --- a/docs/snippets/ShapeInference.py +++ b/docs/snippets/ShapeInference.py @@ -1,23 +1,23 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core, Layout, set_batch -ov = Core() -model = ov.read_model("path/to/model") +import openvino as ov +from utils import get_model, get_image + +model = get_model() #! [picture_snippet] model.reshape([8, 3, 448, 448]) #! [picture_snippet] #! [set_batch] -model.get_parameters()[0].set_layout(Layout("N...")) -set_batch(model, 5) +model.get_parameters()[0].set_layout(ov.Layout("N...")) +ov.set_batch(model, 5) #! [set_batch] #! [simple_spatials_change] -from cv2 import imread -image = imread("path/to/image") -model.reshape({1, 3, image.shape[0], image.shape[1]}) +image = get_image() +model.reshape([1, 3, image.shape[0], image.shape[1]]) #! [simple_spatials_change] #! [obj_to_shape] diff --git a/docs/snippets/__init__.py b/docs/snippets/__init__.py new file mode 100644 index 00000000000000..b1b02362f645ac --- /dev/null +++ b/docs/snippets/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from .utils import get_image +from .utils import get_model +from .utils import get_ngraph_model +from .utils import get_path_to_model diff --git a/docs/snippets/cpu/Bfloat16Inference.py b/docs/snippets/cpu/Bfloat16Inference.py index c1d38c284f57fa..2d3fabfe6c4c29 100644 --- a/docs/snippets/cpu/Bfloat16Inference.py +++ b/docs/snippets/cpu/Bfloat16Inference.py @@ -2,22 +2,24 @@ # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core +import openvino as ov + +from snippets import get_model + +model = get_model() #! [part0] -core = Core() -cpu_optimization_capabilities = core.get_property("CPU", "OPTIMIZATION_CAPABILITIES") +core = ov.Core() +cpu_optimization_capabilities = core.get_property("CPU", ov.properties.device.capabilities()) #! [part0] -# TODO: enable part1 when property api will be supported in python #! [part1] -core = Core() -model = core.read_model("model.xml") +core = ov.Core() compiled_model = core.compile_model(model, "CPU") -inference_precision = core.get_property("CPU", "INFERENCE_PRECISION_HINT") +inference_precision = core.get_property("CPU", ov.properties.hint.inference_precision()) #! [part1] #! [part2] -core = Core() -core.set_property("CPU", {"INFERENCE_PRECISION_HINT": "f32"}) +core = ov.Core() +core.set_property("CPU", {ov.properties.hint.inference_precision(): ov.Type.f32}) #! [part2] diff --git a/docs/snippets/cpu/__init__.py b/docs/snippets/cpu/__init__.py new file mode 100644 index 00000000000000..cddd115d39715b --- /dev/null +++ b/docs/snippets/cpu/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/docs/snippets/cpu/compile_model.py b/docs/snippets/cpu/compile_model.py index 5cb06ae694d01e..b0dd9d7cefda6e 100644 --- a/docs/snippets/cpu/compile_model.py +++ b/docs/snippets/cpu/compile_model.py @@ -1,17 +1,23 @@ -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from snippets import get_model -#! [compile_model_default] -from openvino.runtime import Core -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "CPU") -#! [compile_model_default] +def main(): + model = get_model() -#! [compile_model_multi] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "MULTI:CPU,GPU.0") -#! [compile_model_multi] + #! [compile_model_default] + import openvino as ov + + core = ov.Core() + compiled_model = core.compile_model(model, "CPU") + #! [compile_model_default] + + if "GPU" not in core.available_devices: + return 0 + + #! [compile_model_multi] + core = ov.Core() + compiled_model = core.compile_model(model, "MULTI:CPU,GPU.0") + #! [compile_model_multi] diff --git a/docs/snippets/cpu/dynamic_shape.py b/docs/snippets/cpu/dynamic_shape.py index 8bfbf5d99eec2a..1b531d0a8eebb6 100644 --- a/docs/snippets/cpu/dynamic_shape.py +++ b/docs/snippets/cpu/dynamic_shape.py @@ -1,11 +1,13 @@ -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from snippets import get_model -from openvino.runtime import Core +model = get_model() #! [static_shape] -core = Core() -model = core.read_model("model.xml") +import openvino as ov + +core = ov.Core() model.reshape([10, 20, 30, 40]) #! [static_shape] diff --git a/docs/snippets/cpu/multi_threading.py b/docs/snippets/cpu/multi_threading.py index a8113390e9a12c..9a5baa1e7575b1 100644 --- a/docs/snippets/cpu/multi_threading.py +++ b/docs/snippets/cpu/multi_threading.py @@ -2,28 +2,47 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov -from openvino.runtime import Core, Type, OVAny, properties +from openvino import Core, properties +from snippets import get_model + +model = get_model() + +device_name = "CPU" +core = Core() +core.set_property("CPU", properties.intel_cpu.sparse_weights_decompression_rate(0.8)) -device_name = 'CPU' -xml_path = 'model.xml' -core = ov.Core() -core.set_property("CPU", ov.properties.intel_cpu.sparse_weights_decompression_rate(0.8)) -model = core.read_model(model=xml_path) # ! [ov:intel_cpu:multi_threading:part0] # Use one logical processor for inference -compiled_model_1 = core.compile_model(model=model, device_name=device_name, config={properties.inference_num_threads(1)}) +compiled_model_1 = core.compile_model( + model=model, + device_name=device_name, + config={properties.inference_num_threads(): 1}, +) # Use logical processors of Efficient-cores for inference on hybrid platform -compiled_model_2 = core.compile_model(model=model, device_name=device_name, config={properties.hint.scheduling_core_type(properties.hint.SchedulingCoreType.ECORE_ONLY)}) +compiled_model_2 = core.compile_model( + model=model, + device_name=device_name, + config={ + properties.hint.scheduling_core_type(): properties.hint.SchedulingCoreType.ECORE_ONLY, + }, +) # Use one logical processor per CPU core for inference when hyper threading is on -compiled_model_3 = core.compile_model(model=model, device_name=device_name, config={properties.hint.enable_hyper_threading(False)}) +compiled_model_3 = core.compile_model( + model=model, + device_name=device_name, + config={properties.hint.enable_hyper_threading(): False}, +) # ! [ov:intel_cpu:multi_threading:part0] # ! [ov:intel_cpu:multi_threading:part1] # Disable CPU threads pinning for inference when system supoprt it -compiled_model_4 = core.compile_model(model=model, device_name=device_name, config={properties.hint.enable_cpu_pinning(False)}) +compiled_model_4 = core.compile_model( + model=model, + device_name=device_name, + config={properties.hint.enable_cpu_pinning(): False}, +) # ! [ov:intel_cpu:multi_threading:part1] assert compiled_model_1 assert compiled_model_2 diff --git a/docs/snippets/cpu/ov_execution_mode.py b/docs/snippets/cpu/ov_execution_mode.py index 4abd0ccf3754d6..2feff2777f7d27 100644 --- a/docs/snippets/cpu/ov_execution_mode.py +++ b/docs/snippets/cpu/ov_execution_mode.py @@ -1,12 +1,18 @@ # Copyright (C) 2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core - #! [ov:execution_mode:part0] -core = Core() +import openvino as ov + +core = ov.Core() # in case of Accuracy -core.set_property("CPU", {"EXECUTION_MODE_HINT": "ACCURACY"}) +core.set_property( + "CPU", + {ov.properties.hint.execution_mode(): ov.properties.hint.ExecutionMode.ACCURACY}, +) # in case of Performance -core.set_property("CPU", {"EXECUTION_MODE_HINT": "PERFORMANCE"}) +core.set_property( + "CPU", + {ov.properties.hint.execution_mode(): ov.properties.hint.ExecutionMode.PERFORMANCE}, +) #! [ov:execution_mode:part0] diff --git a/docs/snippets/cpu/ov_sparse_weights_decompression.py b/docs/snippets/cpu/ov_sparse_weights_decompression.py index f7ea0772bf73ce..76de90311d28ad 100644 --- a/docs/snippets/cpu/ov_sparse_weights_decompression.py +++ b/docs/snippets/cpu/ov_sparse_weights_decompression.py @@ -2,14 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov +from snippets import get_model -device_name = 'CPU' -xml_path = 'model.xml' +model = get_model() + +device_name = "CPU" +xml_path = "model.xml" # ! [ov:intel_cpu:sparse_weights_decompression:part0] core = ov.Core() core.set_property("CPU", ov.properties.intel_cpu.sparse_weights_decompression_rate(0.8)) -model = core.read_model(model=xml_path) compiled_model = core.compile_model(model=model, device_name=device_name) # ! [ov:intel_cpu:sparse_weights_decompression:part0] assert compiled_model diff --git a/docs/snippets/export_compiled_model.py b/docs/snippets/export_compiled_model.py index 39bebffe1b86db..b89b9a1479b0fb 100644 --- a/docs/snippets/export_compiled_model.py +++ b/docs/snippets/export_compiled_model.py @@ -1,7 +1,19 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +from utils import get_path_to_model + +device = "CPU" +model_path = get_path_to_model() +properties = {} + #! [export_compiled_model] -from openvino.runtime import Core +import openvino as ov -ov.Core().compile_model(device, modelPath, properties).export_model(compiled_blob) +core = ov.Core() +compiled_model = core.compile_model(model_path, device, properties) +output_stream = compiled_model.export_model() #! [export_compiled_model] diff --git a/docs/snippets/gna/__init__.py b/docs/snippets/gna/__init__.py new file mode 100644 index 00000000000000..6a16273c024652 --- /dev/null +++ b/docs/snippets/gna/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/docs/snippets/gna/configure.py b/docs/snippets/gna/configure.py index 6f0ba6756654e4..ec94a54ae4110c 100644 --- a/docs/snippets/gna/configure.py +++ b/docs/snippets/gna/configure.py @@ -2,14 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 #! [import] -from openvino.runtime import Core +import openvino as ov #! [import] -model_path = "model.xml" +from snippets import get_model +model = get_model() + +# TODO: no GNA properties to replace strings +#! [ov_gna_exec_mode_hw_with_sw_fback] +core = ov.Core() +compiled_model = core.compile_model( + model, device_name="GNA", config={"GNA_DEVICE_MODE": "GNA_HW_WITH_SW_FBACK"} +) #! [ov_gna_exec_mode_hw_with_sw_fback] -core = Core() -model = core.read_model(model=model_path) -compiled_model = core.compile_model(model, device_name="GNA", - config={ 'GNA_DEVICE_MODE' : 'GNA_HW_WITH_SW_FBACK'}) -#! [ov_gna_exec_mode_hw_with_sw_fback] \ No newline at end of file diff --git a/docs/snippets/gna/import_export.py b/docs/snippets/gna/import_export.py index 8ecead1520dafe..8aa6cd11267679 100644 --- a/docs/snippets/gna/import_export.py +++ b/docs/snippets/gna/import_export.py @@ -2,25 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 #! [import] -from openvino.runtime import Core +import openvino as ov from io import BytesIO #! [import] -model_path = "model.xml" +from snippets import get_model + +model = get_model() blob_path = "compiled_model.blob" -core = Core() -model = core.read_model(model=model_path) +core = ov.Core() compiled_model = core.compile_model(model, device_name="GNA") #! [ov_gna_export] user_stream = compiled_model.export_model() -with open(blob_path, 'wb') as f: +with open(blob_path, "wb") as f: f.write(user_stream) #! [ov_gna_export] # [ov_gna_import] -with open(blob_path, 'rb') as f: +with open(blob_path, "rb") as f: buf = BytesIO(f.read()) compiled_model = core.import_model(buf, device_name="GNA") -# [ov_gna_import] \ No newline at end of file +# [ov_gna_import] diff --git a/docs/snippets/gna/set_batch.py b/docs/snippets/gna/set_batch.py index 6ae1cd2ab153ae..fac783923ca4e7 100644 --- a/docs/snippets/gna/set_batch.py +++ b/docs/snippets/gna/set_batch.py @@ -2,20 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 #! [import] -from openvino.runtime import Core, set_batch -from openvino.preprocess import PrePostProcessor +import openvino as ov #! [import] -model_path = "model.xml" +from snippets import get_path_to_model + batch_size = 8 +model_path = get_path_to_model([1, 32]) #! [ov_gna_read_model] -core = Core() +core = ov.Core() model = core.read_model(model=model_path) #! [ov_gna_read_model] #! [ov_gna_set_nc_layout] -ppp = PrePostProcessor(model) +ppp = ov.preprocess.PrePostProcessor(model) for i in range(len(model.inputs)): input_name = model.input(i).get_any_name() ppp.input(i).model().set_layout("N?") @@ -23,5 +24,5 @@ #! [ov_gna_set_nc_layout] #! [ov_gna_set_batch_size] -set_batch(model, batch_size) -#! [ov_gna_set_batch_size] \ No newline at end of file +ov.set_batch(model, batch_size) +#! [ov_gna_set_batch_size] diff --git a/docs/snippets/gpu/__init__.py b/docs/snippets/gpu/__init__.py new file mode 100644 index 00000000000000..6a16273c024652 --- /dev/null +++ b/docs/snippets/gpu/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/docs/snippets/gpu/compile_model.py b/docs/snippets/gpu/compile_model.py index f1cede42ce8eab..15e67d41ab0ca7 100644 --- a/docs/snippets/gpu/compile_model.py +++ b/docs/snippets/gpu/compile_model.py @@ -2,40 +2,49 @@ # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core - -#! [compile_model_default_gpu] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU") -#! [compile_model_default_gpu] - -#! [compile_model_gpu_with_id] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU.1") -#! [compile_model_gpu_with_id] - -#! [compile_model_gpu_with_id_and_tile] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU.1.0") -#! [compile_model_gpu_with_id_and_tile] - -#! [compile_model_multi] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "MULTI:GPU.1,GPU.0") -#! [compile_model_multi] - -#! [compile_model_batch_plugin] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "BATCH:GPU") -#! [compile_model_batch_plugin] - -#! [compile_model_auto_batch] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU", {"PERFORMANCE_HINT": "THROUGHPUT"}) -#! [compile_model_auto_batch] +import openvino as ov +from snippets import get_model + + +def main(): + model = get_model() + + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 + + #! [compile_model_default_gpu] + core = ov.Core() + compiled_model = core.compile_model(model, "GPU") + #! [compile_model_default_gpu] + + #! [compile_model_gpu_with_id] + core = ov.Core() + compiled_model = core.compile_model(model, "GPU.1") + #! [compile_model_gpu_with_id] + + #! [compile_model_gpu_with_id_and_tile] + core = ov.Core() + compiled_model = core.compile_model(model, "GPU.1.0") + #! [compile_model_gpu_with_id_and_tile] + + #! [compile_model_multi] + core = ov.Core() + compiled_model = core.compile_model(model, "MULTI:GPU.1,GPU.0") + #! [compile_model_multi] + + #! [compile_model_batch_plugin] + core = ov.Core() + compiled_model = core.compile_model(model, "BATCH:GPU") + #! [compile_model_batch_plugin] + + #! [compile_model_auto_batch] + core = ov.Core() + compiled_model = core.compile_model( + model, + "GPU", + { + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.THROUGHPUT, + }, + ) + #! [compile_model_auto_batch] diff --git a/docs/snippets/gpu/custom_kernels_api.py b/docs/snippets/gpu/custom_kernels_api.py index f5564f7ba3ccdc..2eaeb610b3fdeb 100644 --- a/docs/snippets/gpu/custom_kernels_api.py +++ b/docs/snippets/gpu/custom_kernels_api.py @@ -1,10 +1,17 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import openvino as ov -from openvino.runtime import Core -#! [part0] -core = Core() -core.set_property("GPU", {"CONFIG_FILE": ""}) -#! [part0] +def main(): + core = ov.Core() + + if "GPU" not in core.available_devices: + return 0 + + # TODO: missing property to replace string + #! [part0] + core = ov.Core() + core.set_property("GPU", {"CONFIG_FILE": ""}) + #! [part0] diff --git a/docs/snippets/gpu/dynamic_batch.py b/docs/snippets/gpu/dynamic_batch.py index 119244efa125dc..be4096febf576a 100644 --- a/docs/snippets/gpu/dynamic_batch.py +++ b/docs/snippets/gpu/dynamic_batch.py @@ -2,27 +2,38 @@ # SPDX-License-Identifier: Apache-2.0 -import openvino.runtime as ov +import openvino as ov +from snippets import get_model -#! [dynamic_batch] -core = ov.Core() -C = 3 -H = 224 -W = 224 +def main(): + model = get_model() + + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 -model = core.read_model("model.xml") -model.reshape([(1, 10), C, H, W]) + #! [dynamic_batch] + core = ov.Core() -# compile model and create infer request -compiled_model = core.compile_model(model, "GPU") -infer_request = compiled_model.create_infer_request() + C = 3 + H = 224 + W = 224 -# create input tensor with specific batch size -input_tensor = ov.Tensor(model.input().element_type, [2, C, H, W]) + model.reshape([(1, 10), C, H, W]) -# ... + # compile model and create infer request + compiled_model = core.compile_model(model, "GPU") + infer_request = compiled_model.create_infer_request() -infer_request.infer([input_tensor]) + # create input tensor with specific batch size + input_tensor = ov.Tensor(model.input().element_type, [2, C, H, W]) -#! [dynamic_batch] + # ... + + results = infer_request.infer([input_tensor]) + + #! [dynamic_batch] + + assert list(results.keys())[0].partial_shape == ov.PartialShape([(1, 10), 3, 224, 224]) + assert list(results.values())[0].shape == tuple(ov.Shape([2, 3, 224, 224])) diff --git a/docs/snippets/gpu/preprocessing_nv12_two_planes.py b/docs/snippets/gpu/preprocessing_nv12_two_planes.py index 7afa5b8eeca30a..fc4a03f964c15d 100644 --- a/docs/snippets/gpu/preprocessing_nv12_two_planes.py +++ b/docs/snippets/gpu/preprocessing_nv12_two_planes.py @@ -1,18 +1,31 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#! [init_preproc] -from openvino.runtime import Core, Type, Layout -from openvino.preprocess import PrePostProcessor, ColorFormat - -core = Core() -model = core.read_model("model.xml") - -p = PrePostProcessor(model) -p.input().tensor().set_element_type(Type.u8) \ - .set_color_format(ColorFormat.NV12_TWO_PLANES, ["y", "uv"]) \ - .set_memory_type("GPU_SURFACE") -p.input().preprocess().convert_color(ColorFormat.BGR) -p.input().model().set_layout(Layout("NCHW")) -model_with_preproc = p.build() -#! [init_preproc] +from snippets import get_model +import openvino as ov + + +def init_preproc(): + model = get_model() + #! [init_preproc] + import openvino as ov + from openvino.preprocess import PrePostProcessor, ColorFormat + + core = ov.Core() + + p = PrePostProcessor(model) + p.input().tensor().set_element_type(ov.Type.u8).set_color_format( + ColorFormat.NV12_TWO_PLANES, ["y", "uv"] + ).set_memory_type("GPU_SURFACE") + p.input().preprocess().convert_color(ColorFormat.BGR) + p.input().model().set_layout(ov.Layout("NCHW")) + model_with_preproc = p.build() + #! [init_preproc] + + +def main(): + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 + init_preproc() + diff --git a/docs/snippets/ie_common.py b/docs/snippets/ie_common.py index 74c3831ffd975d..ff9f97df53f908 100644 --- a/docs/snippets/ie_common.py +++ b/docs/snippets/ie_common.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # +from utils import get_path_to_model, get_image, get_path_to_extension_library #! [ie:create_core] import numpy as np @@ -9,8 +10,10 @@ core = ie.IECore() #! [ie:create_core] +model_path = get_path_to_model(True) + #! [ie:read_model] -network = core.read_network("model.xml") +network = core.read_network(model_path) #! [ie:read_model] #! [ie:compile_model] @@ -26,7 +29,7 @@ infer_request = exec_network.requests[0] # Get input blobs mapped to input layers names input_blobs = infer_request.input_blobs -data = input_blobs["data1"].buffer +data = input_blobs["data"].buffer # Original I64 precision was converted to I32 assert data.dtype == np.int32 # Fill the first blob ... @@ -36,7 +39,7 @@ results = infer_request.infer() #! [ie:inference] -input_data = iter(list()) +input_data = get_image() def process_results(results, frame_id): pass @@ -67,11 +70,11 @@ def callback(output_blobs, _): total_frames = 100 for _ in range(total_frames): # Wait for at least one free request - exec_network.wait(num_request=1) + exec_network.wait(num_requests=1) # Get idle id idle_id = exec_network.get_idle_request_id() # Start asynchronous inference on idle request - exec_network.start_async(request_id=idle_id, inputs=next(input_data)) + exec_network.start_async(request_id=idle_id, inputs={"data": input_data}) # Wait for all requests to complete exec_network.wait() #! [ie:start_async_and_wait] @@ -79,12 +82,13 @@ def callback(output_blobs, _): #! [ie:get_output_tensor] # Get output blobs mapped to output layers names output_blobs = infer_request.output_blobs -data = output_blobs["out1"].buffer +data = output_blobs["relu"].buffer # Original I64 precision was converted to I32 assert data.dtype == np.int32 # Process output data #! [ie:get_output_tensor] +path_to_extension_library = get_path_to_extension_library() #! [ie:load_old_extension] -core.add_extension("path_to_extension_library.so", "CPU") +core.add_extension(path_to_extension_library, "CPU") #! [ie:load_old_extension] diff --git a/docs/snippets/main.py b/docs/snippets/main.py new file mode 100644 index 00000000000000..1c9278548e3bf6 --- /dev/null +++ b/docs/snippets/main.py @@ -0,0 +1,44 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import sys +import os +import io +import importlib +from contextlib import redirect_stdout, redirect_stderr + + +skip_snippets = ["main.py", "__init__.py", "utils.py", "ie_common.py", "ov_common.py"] + +def import_python_modules(directory, subdirectory=""): + for item in os.listdir(directory): + if item.endswith('.py') and item not in skip_snippets: + imported_item = item[:-3] + if subdirectory != "": + imported_item=subdirectory + "." + imported_item + print(f"Snippet {item} is executing...") + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + mod = importlib.import_module(imported_item) + + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + mod.main() + except AttributeError as e: + pass + + print(f"Snippet {item} succesfully executed.") + + + if os.path.isdir(os.path.join(directory, item)): + dir_path = os.path.join(directory, item) + import_python_modules(dir_path, item) + + +def main(): + dir_path = os.path.dirname(os.path.realpath(__file__)) + import_python_modules(dir_path) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/docs/snippets/ngraph.py b/docs/snippets/ngraph_snippet.py similarity index 100% rename from docs/snippets/ngraph.py rename to docs/snippets/ngraph_snippet.py diff --git a/docs/snippets/ov_auto.py b/docs/snippets/ov_auto.py index 9ad3163c4eb421..e0dc58587a4df9 100644 --- a/docs/snippets/ov_auto.py +++ b/docs/snippets/ov_auto.py @@ -1,15 +1,15 @@ import sys -from openvino.runtime import Core +import openvino as ov from openvino.inference_engine import IECore -model_path = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" -path_to_model = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" +from utils import get_model, get_ngraph_model + +model = get_model() +net = get_ngraph_model() -def part0(): -#! [part0] - core = Core() - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) +def part0(): + #! [part0] + core = ov.Core() # compile a model on AUTO using the default list of device candidates. # The following lines are equivalent: @@ -20,21 +20,25 @@ def part0(): # You can also specify the devices to be used by AUTO. # The following lines are equivalent: compiled_model = core.compile_model(model=model, device_name="AUTO:GPU,CPU") - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.device.priorities(): "GPU,CPU"}, + ) # Optional # the AUTO plugin is pre-configured (globally) with the explicit option: - core.set_property(device_name="AUTO", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) -#! [part0] + core.set_property( + device_name="AUTO", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) + #! [part0] + def part1(): -#! [part1] + #! [part1] ### IE API ### ie = IECore() - # Read a network in IR, PaddlePaddle, or ONNX format: - net = ie.read_network(model=path_to_model) - # Load a network to AUTO using the default list of device candidates. # The following lines are equivalent: exec_net = ie.load_network(network=net) @@ -45,87 +49,148 @@ def part1(): # You can also specify the devices to be used by AUTO in its selection process. # The following lines are equivalent: exec_net = ie.load_network(network=net, device_name="AUTO:GPU,CPU") - exec_net = ie.load_network(network=net, device_name="AUTO", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) + exec_net = ie.load_network( + network=net, + device_name="AUTO", + config={ov.properties.device.priorities(): "GPU,CPU"}, + ) # Optional # the AUTO plugin is pre-configured (globally) with the explicit option: - ie.set_config(config={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}, device_name="AUTO") -#! [part1] + ie.set_config( + config={ov.properties.device.priorities(): "GPU,CPU"}, device_name="AUTO" + ) + #! [part1] + def part3(): -#! [part3] - core = Core() - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) + #! [part3] + core = ov.Core() + # Compile a model on AUTO with Performance Hints enabled: # To use the “THROUGHPUT” mode: - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"PERFORMANCE_HINT":"THROUGHPUT"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.THROUGHPUT + }, + ) # To use the “LATENCY” mode: - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"PERFORMANCE_HINT":"LATENCY"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.LATENCY + }, + ) # To use the “CUMULATIVE_THROUGHPUT” mode: - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"PERFORMANCE_HINT":"CUMULATIVE_THROUGHPUT"}) -#! [part3] + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.CUMULATIVE_THROUGHPUT + }, + ) + #! [part3] + def part4(): -#! [part4] - core = Core() - model = core.read_model(model_path) + #! [part4] + core = ov.Core() # Example 1 - compiled_model0 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"HIGH"}) - compiled_model1 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"MEDIUM"}) - compiled_model2 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"LOW"}) + compiled_model0 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.HIGH}, + ) + compiled_model1 = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.model_priority(): ov.properties.hint.Priority.MEDIUM + }, + ) + compiled_model2 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.LOW}, + ) # Assume that all the devices (CPU and GPUs) can support all the networks. # Result: compiled_model0 will use GPU.1, compiled_model1 will use GPU.0, compiled_model2 will use CPU. # Example 2 - compiled_model3 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"HIGH"}) - compiled_model4 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"MEDIUM"}) - compiled_model5 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"LOW"}) + compiled_model3 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.HIGH}, + ) + compiled_model4 = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.model_priority(): ov.properties.hint.Priority.MEDIUM + }, + ) + compiled_model5 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.LOW}, + ) # Assume that all the devices (CPU ang GPUs) can support all the networks. # Result: compiled_model3 will use GPU.1, compiled_model4 will use GPU.1, compiled_model5 will use GPU.0. -#! [part4] + #! [part4] + def part5(): -#! [part5] - core = Core() - model = core.read_model(model_path) + #! [part5] + core = ov.Core() + # gpu_config and cpu_config will load during compile_model() compiled_model = core.compile_model(model=model) compiled_model = core.compile_model(model=model, device_name="AUTO") -#! [part5] + #! [part5] + def part6(): -#! [part6] - core = Core() - # read a network in IR, PaddlePaddle, or ONNX format - model = core.read_model(model_path) + #! [part6] + core = ov.Core() + # compile a model on AUTO and set log level to debug - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"LOG_LEVEL":"LOG_DEBUG"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.log.level(): ov.properties.log.Level.DEBUG}, + ) # set log level with set_property and compile model - core.set_property(device_name="AUTO", properties={"LOG_LEVEL":"LOG_DEBUG"}) + core.set_property( + device_name="AUTO", + properties={ov.properties.log.level(): ov.properties.log.Level.DEBUG}, + ) compiled_model = core.compile_model(model=model, device_name="AUTO") -#! [part6] + #! [part6] + def part7(): -#! [part7] - core = Core() - # read a network in IR, PaddlePaddle, or ONNX format - model = core.read_model(model_path) + #! [part7] + core = ov.Core() + # compile a model on AUTO and set log level to debug compiled_model = core.compile_model(model=model, device_name="AUTO") # query the runtime target devices on which the inferences are being executed - execution_devices = compiled_model.get_property("EXECUTION_DEVICES") -#! [part7] + execution_devices = compiled_model.get_property(ov.properties.execution_devices()) + #! [part7] + def main(): - part0() - part1() part3() part4() part5() part6() part7() - -if __name__ == '__main__': - sys.exit(main()) + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 + part0() + part1() diff --git a/docs/snippets/ov_auto_batching.py b/docs/snippets/ov_auto_batching.py index cecf2f846a7b8e..5e4f3032f7d05f 100644 --- a/docs/snippets/ov_auto_batching.py +++ b/docs/snippets/ov_auto_batching.py @@ -1,41 +1,55 @@ -from openvino.runtime import Core - -core = Core() -model = core.read_model(model="sample.xml") - -# [compile_model] -config = {"PERFORMANCE_HINT": "THROUGHPUT"} -compiled_model = core.compile_model(model, "GPU", config) -# [compile_model] - -# [compile_model_no_auto_batching] -# disabling the automatic batching -# leaving intact other configurations options that the device selects for the 'throughput' hint -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "ALLOW_AUTO_BATCHING": False} -compiled_model = core.compile_model(model, "GPU", config) -# [compile_model_no_auto_batching] - -# [query_optimal_num_requests] -# when the batch size is automatically selected by the implementation -# it is important to query/create and run the sufficient requests -config = {"PERFORMANCE_HINT": "THROUGHPUT"} -compiled_model = core.compile_model(model, "GPU", config) -num_requests = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -# [query_optimal_num_requests] - -# [hint_num_requests] -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "PERFORMANCE_HINT_NUM_REQUESTS": "4"} -# limiting the available parallel slack for the 'throughput' -# so that certain parameters (like selected batch size) are automatically accommodated accordingly -compiled_model = core.compile_model(model, "GPU", config) -# [hint_num_requests] - -# [hint_plus_low_level] -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "INFERENCE_NUM_THREADS": "4"} -# limiting the available parallel slack for the 'throughput' -# so that certain parameters (like selected batch size) are automatically accommodated accordingly -compiled_model = core.compile_model(model, "CPU", config) -# [hint_plus_low_level]] \ No newline at end of file +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import openvino as ov +from utils import get_model + + +def main(): + + core = ov.Core() + model = get_model() + + if "GPU" not in core.available_devices: + return 0 + + # [compile_model] + import openvino.runtime.properties as props + import openvino.runtime.properties.hint as hints + + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT} + compiled_model = core.compile_model(model, "GPU", config) + # [compile_model] + + # [compile_model_no_auto_batching] + # disabling the automatic batching + # leaving intact other configurations options that the device selects for the 'throughput' hint + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.allow_auto_batching(): False} + compiled_model = core.compile_model(model, "GPU", config) + # [compile_model_no_auto_batching] + + # [query_optimal_num_requests] + # when the batch size is automatically selected by the implementation + # it is important to query/create and run the sufficient requests + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT} + compiled_model = core.compile_model(model, "GPU", config) + num_requests = compiled_model.get_property(props.optimal_number_of_infer_requests()) + # [query_optimal_num_requests] + + # [hint_num_requests] + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.num_requests(): "4"} + # limiting the available parallel slack for the 'throughput' + # so that certain parameters (like selected batch size) are automatically accommodated accordingly + compiled_model = core.compile_model(model, "GPU", config) + # [hint_num_requests] + + # [hint_plus_low_level] + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + props.inference_num_threads(): "4"} + # limiting the available parallel slack for the 'throughput' + # so that certain parameters (like selected batch size) are automatically accommodated accordingly + compiled_model = core.compile_model(model, "CPU", config) + # [hint_plus_low_level]] diff --git a/docs/snippets/ov_caching.py b/docs/snippets/ov_caching.py index 51014707769a40..ef3fd0b4861ae6 100644 --- a/docs/snippets/ov_caching.py +++ b/docs/snippets/ov_caching.py @@ -2,35 +2,39 @@ # SPDX-License-Identifier: Apache-2.0 # -from openvino.runtime import Core +from utils import get_path_to_model, get_temp_dir +import openvino as ov + +import openvino.runtime.properties as props device_name = 'GNA' -xml_path = '/tmp/myModel.xml' +model_path = get_path_to_model() +path_to_cache_dir = get_temp_dir() # ! [ov:caching:part0] -core = Core() -core.set_property({'CACHE_DIR': '/path/to/cache/dir'}) -model = core.read_model(model=xml_path) +core = ov.Core() +core.set_property({props.cache_dir(): path_to_cache_dir}) +model = core.read_model(model=model_path) compiled_model = core.compile_model(model=model, device_name=device_name) # ! [ov:caching:part0] assert compiled_model # ! [ov:caching:part1] -core = Core() -compiled_model = core.compile_model(model_path=xml_path, device_name=device_name) +core = ov.Core() +compiled_model = core.compile_model(model=model_path, device_name=device_name) # ! [ov:caching:part1] assert compiled_model # ! [ov:caching:part2] -core = Core() -core.set_property({'CACHE_DIR': '/path/to/cache/dir'}) -compiled_model = core.compile_model(model_path=xml_path, device_name=device_name) +core = ov.Core() +core.set_property({props.cache_dir(): path_to_cache_dir}) +compiled_model = core.compile_model(model=model_path, device_name=device_name) # ! [ov:caching:part2] assert compiled_model # ! [ov:caching:part3] # Find 'EXPORT_IMPORT' capability in supported capabilities -caching_supported = 'EXPORT_IMPORT' in core.get_property(device_name, 'OPTIMIZATION_CAPABILITIES') +caching_supported = 'EXPORT_IMPORT' in core.get_property(device_name, props.device.capabilities()) # ! [ov:caching:part3] diff --git a/docs/snippets/ov_common.py b/docs/snippets/ov_common.py index c6ea0a8f97ddd5..b3d07524391da3 100644 --- a/docs/snippets/ov_common.py +++ b/docs/snippets/ov_common.py @@ -4,14 +4,17 @@ import numpy as np +from utils import get_image, get_path_to_extension_library, get_path_to_model #! [ov_api_2_0:create_core] -import openvino.runtime as ov +import openvino as ov core = ov.Core() #! [ov_api_2_0:create_core] +model_path = get_path_to_model() + #! [ov_api_2_0:read_model] -model = core.read_model("model.xml") +model = core.read_model(model_path) #! [ov_api_2_0:read_model] #! [ov_api_2_0:compile_model] @@ -30,7 +33,7 @@ # Fill the first data ... # Get input tensor by tensor name -input_tensor2 = infer_request.get_tensor("data2_t") +input_tensor2 = infer_request.get_tensor("input") assert input_tensor2.data.dtype == np.int64 # Fill the second data ... #! [ov_api_2_0:get_input_tensor_aligned] @@ -43,7 +46,7 @@ # Fill the first data ... # Get input tensor by tensor name -input_tensor2 = infer_request.get_tensor("data2_t") +input_tensor2 = infer_request.get_tensor("input") # IR v10 works with converted precisions (i64 -> i32) assert input_tensor2.data.dtype == np.int32 # Fill the second data .. @@ -53,7 +56,7 @@ results = infer_request.infer() #! [ov_api_2_0:inference] -input_data = iter(list()) +input_data = get_image() def process_results(results, frame_id): pass @@ -83,7 +86,7 @@ def callback(request, frame_id): total_frames = 100 for i in range(total_frames): # Wait for at least one available infer request and start asynchronous inference - infer_queue.start_async(next(input_data), userdata=i) + infer_queue.start_async(input_data, userdata=i) # Wait for all requests to complete infer_queue.wait_all() #! [ov_api_2_0:start_async_and_wait] @@ -104,6 +107,8 @@ def callback(request, frame_id): # process output data ... #! [ov_api_2_0:get_output_tensor_v10] +path_to_extension_library = get_path_to_extension_library() + #! [ov_api_2_0:load_old_extension] -core.add_extension("path_to_extension_library.so") +core.add_extension(path_to_extension_library) #! [ov_api_2_0:load_old_extension] diff --git a/docs/snippets/ov_denormals.py b/docs/snippets/ov_denormals.py index 20f54e7eced910..45b77483fcc882 100644 --- a/docs/snippets/ov_denormals.py +++ b/docs/snippets/ov_denormals.py @@ -2,14 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov +from utils import get_model device_name = 'CPU' -xml_path = 'modelWithDenormals.xml' +model = get_model() + # ! [ov:intel_cpu:denormals_optimization:part0] core = ov.Core() core.set_property("CPU", ov.properties.intel_cpu.denormals_optimization(True)) -model = core.read_model(model=xml_path) compiled_model = core.compile_model(model=model, device_name=device_name) # ! [ov:intel_cpu:denormals_optimization:part0] assert compiled_model diff --git a/docs/snippets/ov_dynamic_shapes.py b/docs/snippets/ov_dynamic_shapes.py index 7d90076a189373..ce9474b6750f7a 100644 --- a/docs/snippets/ov_dynamic_shapes.py +++ b/docs/snippets/ov_dynamic_shapes.py @@ -2,13 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +from utils import get_dynamic_model + #! [import] -import openvino.runtime as ov +import openvino as ov #! [import] +model = get_dynamic_model() + #! [reshape_undefined] -Core = ov.Core() -model = core.read_model(“model.xml”) +core = ov.Core() # Set first dimension to be dynamic while keeping others static model.reshape([-1, 3, 224, 224]) @@ -25,7 +28,7 @@ model.reshape([(1, 8), 3, (112, 448), (112, 448)]) #! [reshape_bounds] -model = core.read_model("model.xml") +model = get_dynamic_model() #! [print_dynamic] # Print output partial shape @@ -35,41 +38,43 @@ print(model.input().partial_shape) #! [print_dynamic] +model = get_dynamic_model() + #! [detect_dynamic] -model = core.read_model("model.xml") -if model.input(0).partial_shape.is_dynamic(): +if model.input(0).partial_shape.is_dynamic: # input is dynamic pass -if model.output(0).partial_shape.is_dynamic(): +if model.output(0).partial_shape.is_dynamic: # output is dynamic pass -if model.output(0).partial_shape[1].is_dynamic(): +if model.output(0).partial_shape[1].is_dynamic: # 1-st dimension of output is dynamic pass #! [detect_dynamic] executable = core.compile_model(model) infer_request = executable.create_infer_request() +input_tensor_name = "input" #! [set_input_tensor] # For first inference call, prepare an input tensor with 1x128 shape and run inference request -Input_data1 = np.ones(shape=[1,128]) -infer_request.infer([input_data1]) +input_data1 = np.ones(shape=[1,128]) +infer_request.infer({input_tensor_name: input_data1}) # Get resulting outputs -Output_tensor1 = infer_request.get_output_tensor() -Output_data1 = output_tensor.data[:] +output_tensor1 = infer_request.get_output_tensor() +output_data1 = output_tensor1.data[:] # For second inference call, prepare a 1x200 input tensor and run inference request -Input_data2 = np.ones(shape=[1,200]) -infer_request.infer([input_data2]) +input_data2 = np.ones(shape=[1,200]) +infer_request.infer({input_tensor_name: input_data2}) # Get resulting outputs -Output_tensor2 = infer_request.get_output_tensor() -Output_data2 = output_tensor.data[:] +output_tensor2 = infer_request.get_output_tensor() +output_data2 = output_tensor2.data[:] #! [set_input_tensor] infer_request = executable.create_infer_request() @@ -96,10 +101,9 @@ data2 = output_tensor.data[:] #! [get_input_tensor] -#! [check_inputs] -core = ov.Core() -model = core.read_model("model.xml") +model = get_dynamic_model() +#! [check_inputs] # Print model input layer info for input_layer in model.inputs: print(input_layer.names, input_layer.partial_shape) diff --git a/docs/snippets/ov_extensions.py b/docs/snippets/ov_extensions.py index 3b0d320eba31a4..99e1412e00eaa1 100644 --- a/docs/snippets/ov_extensions.py +++ b/docs/snippets/ov_extensions.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov #! [py_frontend_extension_ThresholdedReLU_header] -import openvino.runtime.opset8 as ops +import openvino.runtime.opset12 as ops from openvino.frontend import ConversionExtension #! [py_frontend_extension_ThresholdedReLU_header] @@ -17,10 +17,14 @@ # Not implemented #! [add_frontend_extension] +from utils import get_path_to_extension_library + +path_to_extension_lib = get_path_to_extension_library() + #! [add_extension_lib] core = ov.Core() # Load extensions library to ov.Core -core.add_extension("libopenvino_template_extension.so") +core.add_extension(path_to_extension_lib) #! [add_extension_lib] #! [py_frontend_extension_MyRelu] diff --git a/docs/snippets/ov_graph.py b/docs/snippets/ov_graph.py index e3eb6e9db4567e..f0d86336391b3b 100644 --- a/docs/snippets/ov_graph.py +++ b/docs/snippets/ov_graph.py @@ -25,21 +25,22 @@ # | result | # |_____________| -import openvino.runtime as ov +import openvino as ov +import openvino.runtime.opset12 as ops -data1 = ov.opset8.parameter([1, 3, 2, 2], ov.Type.i64) +data1 = ops.parameter([1, 3, 2, 2], ov.Type.i64) data1.friendly_name = "data1" # operation name data1.output(0).name = "data1_t" # tensor name -data2 = ov.opset8.parameter([1, 2, 2, 2], ov.Type.i64) +data2 = ops.parameter([1, 2, 2, 2], ov.Type.i64) data2.friendly_name = "data2" # operation name data2.output(0).name = "data2_t" # tensor name -concat = ov.opset8.concat([data1, data2], 1) +concat = ops.concat([data1, data2], 1) concat.friendly_name = "concat" # operation name concat.output(0).name = "concat_t" # tensor name -result = ov.opset8.result(concat) +result = ops.result(concat) result.friendly_name = "result" # operation name model = ov.Model(result, [data1, data2], "model_name") diff --git a/docs/snippets/ov_hetero.py b/docs/snippets/ov_hetero.py index 2d5bbd2d454880..95623a1f74a14c 100644 --- a/docs/snippets/ov_hetero.py +++ b/docs/snippets/ov_hetero.py @@ -1,43 +1,50 @@ -import openvino.runtime as ov - -core = ov.Core() -model = core.read_model("sample.xml") - -#! [set_manual_affinities] -for op in model.get_ops(): - rt_info = op.get_rt_info() - rt_info["affinity"] = "CPU" -#! [set_manual_affinities] - -#! [fix_automatic_affinities] -# This example demonstrates how to perform default affinity initialization and then -# correct affinity manually for some layers -device = "HETERO:GPU,CPU" - -# query_model result contains mapping of supported operations to devices -supported_ops = core.query_model(model, device) - -# update default affinities manually for specific operations -supported_ops["operation_name"] = "CPU" - -# set affinities to a model -for node in model.get_ops(): - affinity = supported_ops[node.get_friendly_name()] - node.get_rt_info()["affinity"] = "CPU" - -# load model with manually set affinities -compiled_model = core.compile_model(model, device) -#! [fix_automatic_affinities] - -#! [compile_model] -compiled_model = core.compile_model(model, device_name="HETERO:GPU,CPU") -# device priorities via configuration property -compiled_model = core.compile_model(model, device_name="HETERO", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) -#! [compile_model] - -#! [configure_fallback_devices] -core.set_property("HETERO", {"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) -core.set_property("GPU", {"PERF_COUNT": "YES"}) -core.set_property("CPU", {"INFERENCE_PRECISION_HINT": "f32"}) -compiled_model = core.compile_model(model=model, device_name="HETERO") -#! [configure_fallback_devices] +import openvino as ov +from utils import get_model + +def main(): + model = get_model() + core = ov.Core() + + #! [set_manual_affinities] + for op in model.get_ops(): + rt_info = op.get_rt_info() + rt_info["affinity"] = "CPU" + #! [set_manual_affinities] + + if "GPU" not in core.available_devices: + return 0 + + #! [fix_automatic_affinities] + # This example demonstrates how to perform default affinity initialization and then + # correct affinity manually for some layers + device = "HETERO:GPU,CPU" + + # query_model result contains mapping of supported operations to devices + supported_ops = core.query_model(model, device) + + # update default affinities manually for specific operations + supported_ops["operation_name"] = "CPU" + + # set affinities to a model + for node in model.get_ops(): + affinity = supported_ops[node.get_friendly_name()] + node.get_rt_info()["affinity"] = "CPU" + + # load model with manually set affinities + compiled_model = core.compile_model(model, device) + #! [fix_automatic_affinities] + + #! [compile_model] + compiled_model = core.compile_model(model, device_name="HETERO:GPU,CPU") + # device priorities via configuration property + compiled_model = core.compile_model( + model, device_name="HETERO", config={ov.properties.device.priorities(): "GPU,CPU"} + ) + #! [compile_model] + + #! [configure_fallback_devices] + core.set_property("HETERO", {ov.properties.device.priorities(): "GPU,CPU"}) + core.set_property("GPU", {ov.properties.enable_profiling(): True}) + core.set_property("CPU", {ov.properties.hint.inference_precision(): ov.Type.f32}) + compiled_model = core.compile_model(model=model, device_name="HETERO") + #! [configure_fallback_devices] diff --git a/docs/snippets/ov_infer_request.py b/docs/snippets/ov_infer_request.py index c770684e2da9ab..f2effba5fd4dc0 100644 --- a/docs/snippets/ov_infer_request.py +++ b/docs/snippets/ov_infer_request.py @@ -1,13 +1,14 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import numpy as np +from utils import get_model + #! [import] -import openvino.runtime as ov +import openvino as ov #! [import] core = ov.Core() -model = core.read_model("model.xml") +model = get_model() compiled_model = core.compile_model(model, "AUTO") #! [create_infer_request] @@ -31,10 +32,12 @@ #! [wait_for] #! [set_callback] -def callback(request, userdata): +def callback(request, _): request.start_async() -infer_request.set_callback(callback) +callbacks_info = {} +callbacks_info["finished"] = 0 +infer_request.set_callback(callback, callbacks_info) #! [set_callback] #! [cancel] @@ -48,24 +51,21 @@ def callback(request, userdata): #! [get_set_index_tensor] input_tensor = infer_request.get_input_tensor(0) -output_tensor = infer_request.get_output_tensor(1) +output_tensor = infer_request.get_output_tensor(0) #! [get_set_index_tensor] -#! [get_set_name_tensor] -input_tensor = infer_request.get_tensor("input_name") -output_tensor = infer_request.get_tensor("output_name") -#! [get_set_name_tensor] +input_tensor_name = "input" #! [get_set_tensor] -tensor1 = infer_request.get_tensor("tensor_name1") -tensor2 = ov.Tensor() -infer_request.set_tensor("tensor_name2", tensor2) +tensor1 = infer_request.get_tensor("result") +tensor2 = ov.Tensor(ov.Type.f32, [1, 3, 32, 32]) +infer_request.set_tensor(input_tensor_name, tensor2) #! [get_set_tensor] #! [get_set_tensor_by_port] input_port = model.input(0) -output_port = model.input("tensor_name") -input_tensor = ov.Tensor() +output_port = model.input(input_tensor_name) +input_tensor = ov.Tensor(ov.Type.f32, [1, 3, 32, 32]) infer_request.set_tensor(input_port, input_tensor) output_tensor = infer_request.get_tensor(output_port) #! [get_set_tensor_by_port] @@ -81,15 +81,15 @@ def callback(request, userdata): #! [roi_tensor] # input_tensor points to input of a previous network and # cropROI contains coordinates of output bounding box **/ -input_tensor = ov.Tensor(type=ov.Type.f32, shape=ov.Shape([1, 3, 20, 20])) +input_tensor = ov.Tensor(type=ov.Type.f32, shape=ov.Shape([1, 3, 100, 100])) begin = [0, 0, 0, 0] -end = [1, 2, 3, 3] +end = [1, 3, 32, 32] # ... # roi_tensor uses shared memory of input_tensor and describes cropROI # according to its coordinates **/ roi_tensor = ov.Tensor(input_tensor, begin, end) -infer_request2.set_tensor("input_name", roi_tensor) +infer_request2.set_tensor(input_tensor_name, roi_tensor) #! [roi_tensor] #! [remote_tensor] diff --git a/docs/snippets/ov_layout.py b/docs/snippets/ov_layout.py index 5ce91e74a500e2..79547045ebde81 100644 --- a/docs/snippets/ov_layout.py +++ b/docs/snippets/ov_layout.py @@ -1,10 +1,11 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov +import openvino.runtime.opset12 as ops # ! [ov:layout:simple] -from openvino.runtime import Layout +from openvino import Layout layout = Layout('NCHW') # ! [ov:layout:simple] # ! [ov:layout:complex] @@ -61,12 +62,12 @@ def create_simple_model(): # Parameter--->Multiply--->Add--->Result # Constant---' / # Constant---' - data = ov.opset8.parameter([3, 1, 2], ov.Type.f32) - mul_constant = ov.opset8.constant([1.5], ov.Type.f32) - mul = ov.opset8.multiply(data, mul_constant) - add_constant = ov.opset8.constant([0.5], ov.Type.f32) - add = ov.opset8.add(mul, add_constant) - res = ov.opset8.result(add) + data = ops.parameter([3, 1, 2], ov.Type.f32, name="input_tensor_name") + mul_constant = ops.constant([1.5], ov.Type.f32) + mul = ops.multiply(data, mul_constant) + add_constant = ops.constant([0.5], ov.Type.f32) + add = ops.add(mul, add_constant) + res = ops.result(add) return ov.Model([res], [data], "model") model = create_simple_model() diff --git a/docs/snippets/ov_model_snippets.py b/docs/snippets/ov_model_snippets.py index 08da43fc2ab28b..0bddd24f142e86 100644 --- a/docs/snippets/ov_model_snippets.py +++ b/docs/snippets/ov_model_snippets.py @@ -3,10 +3,13 @@ import numpy as np #! [import] -import openvino.runtime as ov +import openvino as ov +#! [import] +import openvino.runtime.opset12 as ops #! [import] import openvino.runtime.passes as passes + # ! [ov:create_simple_model] def create_simple_model(): # This example shows how to create ov::Function @@ -14,12 +17,12 @@ def create_simple_model(): # Parameter--->Multiply--->Add--->Result # Constant---' / # Constant---' - data = ov.opset8.parameter([3, 1, 2], ov.Type.f32) - mul_constant = ov.opset8.constant([1.5], ov.Type.f32) - mul = ov.opset8.multiply(data, mul_constant) - add_constant = ov.opset8.constant([0.5], ov.Type.f32) - add = ov.opset8.add(mul, add_constant) - res = ov.opset8.result(add) + data = ops.parameter([3, 1, 2], ov.Type.f32) + mul_constant = ops.constant([1.5], ov.Type.f32) + mul = ops.multiply(data, mul_constant) + add_constant = ops.constant([0.5], ov.Type.f32) + add = ops.add(mul, add_constant) + res = ops.result(add) return ov.Model([res], [data], "model") # ! [ov:create_simple_model] @@ -30,24 +33,24 @@ def create_advanced_model(): # Parameter->Split---0-->Result # | `--1-->Relu-->Result # `----2-->Result - data = ov.opset8.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32) + data = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32) # Create Constant for axis value - axis_const = ov.opset8.constant(ov.Type.i64, ov.Shape({}), [1]) + axis_const = ops.constant(1, dtype=ov.Type.i64) - # Create opset8::Split operation that splits input to three slices across 1st dimension - split = ov.opset8.split(data, axis_const, 3) + # Create opset12::Split operation that splits input to three slices across 1st dimension + split = ops.split(data, axis_const, 3) - # Create opset8::Relu operation that takes 1st Split output as input - relu = ov.opset8.relu(split.output(1)) + # Create opset12::Relu operation that takes 1st Split output as input + relu = ops.relu(split.output(1)) # Results operations will be created automatically based on provided OutputVector - return ov.Model([split.output(0), relu, split.output[2]], [data], "model") + return ov.Model([split.output(0), relu.output(0), split.output(2)], [data], "model") # ! [ov:create_advanced_model] def ov_api_examples(): # Doesn't work # node = ov.opset8.parameter(ov.PartialShape([ov.Dimension.dynamic(), 3, 64, 64]), np.float32) - node = ov.opset8.parameter(ov.PartialShape([ov.Dimension.dynamic(), ov.Dimension(3), ov.Dimension(64), ov.Dimension(64)]), np.float32) + node = ops.parameter(ov.PartialShape([ov.Dimension.dynamic(), ov.Dimension(3), ov.Dimension(64), ov.Dimension(64)]), np.float32) # it doesn't work: # static_shape = ov.Shape() @@ -59,8 +62,7 @@ def ov_api_examples(): # ! [ov:serialize] def serialize_example(m : ov.Model): - from openvino.runtime import serialize - serialize(m, xml_path='model.xml', bin_path='model.bin') + ov.serialize(m, xml_path='model.xml', bin_path='model.bin') # ! [ov:serialize] # ! [ov:visualize] @@ -68,7 +70,7 @@ def visualize_example(m : ov.Model): # Need import: # * import openvino.runtime.passes as passes pass_manager = passes.Manager() - pass_manager.register_pass(pass_name="VisualTree", file_name='image.svg') + pass_manager.register_pass(passes.VisualizeTree(file_name='image.svg')) pass_manager.run_passes(m) # ! [ov:visualize] @@ -79,7 +81,10 @@ def model_inputs_outputs(model : ov.Model): #! [all_inputs_ouputs] -if __name__ == '__main__': +def main(): ov_api_examples() create_simple_model() - create_advanced_model() + model = create_advanced_model() + serialize_example(model) + visualize_example(model) + model_inputs_outputs(model) diff --git a/docs/snippets/ov_model_with_state_infer.py b/docs/snippets/ov_model_with_state_infer.py new file mode 100644 index 00000000000000..13122c492f2786 --- /dev/null +++ b/docs/snippets/ov_model_with_state_infer.py @@ -0,0 +1,84 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging as log +import numpy as np +import sys + +import openvino as ov +from openvino.runtime import opset12 as ops + + +def main(): + # 1. Load inference engine + log.info("Loading OpenVINO") + core = ov.Core() + + #! [ov:model_create] + # 2. Creating ov.Model + input = ops.parameter([1, 1], dtype=np.float32, name="data") + init_const = ops.constant([0], dtype=np.float32) + read = ops.read_value(init_const, "variable0") + add = ops.add(input, read) + assign = ops.assign(add, "variable0") + add2 = ops.add(add, read) + result = ops.result(add2) + model = ov.Model(results=[result], sinks=[assign], parameters=[input], name="model") + #! [ov:model_create] + + log.info("Loading network files") + + # 3. Load network to CPU + compiled_model = core.compile_model(model, "CPU") + # 4. Create Infer Request + infer_request = compiled_model.create_infer_request() + + # 5. Prepare inputs + input_tensors = [] + for input in compiled_model.inputs: + input_tensors.append(infer_request.get_tensor(input)) + + # 6. Prepare outputs + output_tensors = [] + for output in compiled_model.outputs: + output_tensors.append(infer_request.get_tensor(output)) + + # 7. Initialize memory state before starting + for state in infer_request.query_state(): + state.reset() + + #! [ov:part1] + # input data + input_data = np.arange(start=1, stop=7, dtype=np.float32) + log.info("Infer the first utterance") + for next_input in range(int(len(input_data)/2)): + infer_request.infer({"data" : np.asarray([input_data[next_input]]).reshape([1,1])}) + # check states + states = infer_request.query_state() + if len(states) == 0: + log.error("Queried states are empty") + return -1 + mstate = states[0].state + if not mstate: + log.error("Can't cast state to MemoryBlob") + return -1 + state_buf = mstate.data + log.info(state_buf[0]) + + log.info("\nReset state between utterances...\n") + for state in infer_request.query_state(): + state.reset() + + log.info("Infer the second utterance") + for next_input in range(int(len(input_data)/2), len(input_data)): + infer_request.infer({0 : np.asarray([input_data[next_input]]).reshape([1,1])}) + # check states + states = infer_request.query_state() + mstate = states[0].state + state_buf = mstate.data + log.info(state_buf[0]) + #! [ov:part1] + + log.info("Execution successful") + + return 0 diff --git a/docs/snippets/ov_multi.py b/docs/snippets/ov_multi.py index f4e538c1b842ba..19ec07d8f9a8c5 100644 --- a/docs/snippets/ov_multi.py +++ b/docs/snippets/ov_multi.py @@ -1,108 +1,121 @@ import sys -from openvino.runtime import Core -model_path = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" -path_to_model = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" +import openvino as ov +from utils import get_model + +model = get_model() + + def MULTI_0(): -#! [MULTI_0] - core = Core() + #! [MULTI_0] + core = ov.Core() - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) - # Option 1 # Pre-configure MULTI globally with explicitly defined devices, # and compile the model on MULTI using the newly specified default device list. - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) compiled_model = core.compile_model(model=model, device_name="MULTI") # Option 2 # Specify the devices to be used by MULTI explicitly at compilation. # The following lines are equivalent: compiled_model = core.compile_model(model=model, device_name="MULTI:GPU,CPU") - compiled_model = core.compile_model(model=model, device_name="MULTI", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) + compiled_model = core.compile_model( + model=model, + device_name="MULTI", + config={ov.properties.device.priorities(): "GPU,CPU"}, + ) + #! [MULTI_0] -#! [MULTI_0] def MULTI_1(): -#! [MULTI_1] - core = Core() - model = core.read_model(model_path) - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"CPU,GPU"}) + #! [MULTI_1] + core = ov.Core() + + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "CPU,GPU"} + ) # Once the priority list is set, you can alter it on the fly: # reverse the order of priorities - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) - + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) + # exclude some devices (in this case, CPU) - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU"}) - + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU"} + ) + # bring back the excluded devices - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) # You cannot add new devices on the fly! # Attempting to do so will trigger the following exception: # [ ERROR ] [NOT_FOUND] You can only change device # priorities but not add new devices with the model's # ov::device::priorities. CPU device was not in the original device list! - -#! [MULTI_1] + #! [MULTI_1] # the following two pieces of code appear not to be used anywhere # they should be considered for removal + def available_devices_1(): -#! [available_devices_1] + #! [available_devices_1] all_devices = "MULTI:" - core = Core() - model = core.read_model(model_path) + core = ov.Core() + all_devices += ",".join(core.available_devices) compiled_model = core.compile_model(model=model, device_name=all_devices) -#! [available_devices_1] + #! [available_devices_1] + def available_devices_2(): -#! [available_devices_2] + #! [available_devices_2] match_list = [] all_devices = "MULTI:" dev_match_str = "GPU" - core = Core() - model = core.read_model(model_path) + core = ov.Core() + for d in core.available_devices: if dev_match_str in d: match_list.append(d) all_devices += ",".join(match_list) compiled_model = core.compile_model(model=model, device_name=all_devices) -#! [available_devices_2] - - - - - - + #! [available_devices_2] def MULTI_4(): -#! [MULTI_4] - core = Core() + #! [MULTI_4] + core = ov.Core() cpu_config = {} gpu_config = {} - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) - - # When compiling the model on MULTI, configure CPU and GPU + # When compiling the model on MULTI, configure CPU and GPU # (devices, priorities, and device configurations; gpu_config and cpu_config will load during compile_model() ): - compiled_model = core.compile_model(model=model, device_name="MULTI:GPU,CPU", config={"CPU":"NUM_STREAMS 4", "GPU":"NUM_STREAMS 8"}) + compiled_model = core.compile_model( + model=model, + device_name="MULTI:GPU,CPU", + config={"CPU": "NUM_STREAMS 4", "GPU": "NUM_STREAMS 8"}, + ) # Optionally, query the optimal number of requests: - nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -#! [MULTI_4] + nireq = compiled_model.get_property( + ov.properties.optimal_number_of_infer_requests() + ) + #! [MULTI_4] + def main(): + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 MULTI_0() MULTI_1() available_devices_1() available_devices_2() MULTI_4() - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docs/snippets/ov_network_state_intro.py b/docs/snippets/ov_network_state_intro.py deleted file mode 100644 index 1ec8c31433dc70..00000000000000 --- a/docs/snippets/ov_network_state_intro.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import logging as log -import numpy as np -import sys - -from openvino.runtime import opset10 as ops -from openvino.runtime import Core, Model, PartialShape, Tensor, Type -from openvino.runtime.passes import LowLatency2, MakeStateful, Manager - - -def state_network_example(): - #! [ov:state_network] - input = ops.parameter([1, 1], dtype=np.float32) - read = ops.read_value(input, "variable0") - add = ops.add(read, input) - save = ops.assign(add, "variable0") - result = ops.result(add) - model = Model(results=[result], sinks=[save], parameters=[input]) - #! [ov:state_network] - - -def low_latency_2_example(): - #! [ov:low_latency_2] - # Precondition for Model. - # TensorIterator and Parameter are created in body of TensorIterator with names - tensor_iterator_name = "TI_name" - body_parameter_name = "body_parameter_name" - idx = "0" # this is a first variable in the network - - # The State will be named "TI_name/param_name/variable_0" - state_name = tensor_iterator_name + "//" + body_parameter_name + "//" + "variable_" + idx # todo - - #! [ov:get_ov_model] - core = Core() - ov_model = core.read_model("path_to_the_model") - #! [ov:get_ov_model] - - # reshape input if needed - - #! [ov:reshape_ov_model] - ov_model.reshape({"X": PartialShape([1, 1, 16])}) - #! [ov:reshape_ov_model] - - #! [ov:apply_low_latency_2] - manager = Manager() - manager.register_pass(LowLatency2()) - manager.run_passes(ov_model) - #! [ov:apply_low_latency_2] - hd_specific_model = core.compile_model(ov_model) - # Try to find the Variable by name - infer_request = hd_specific_model.create_infer_request() - states = infer_request.query_state() - for state in states: - name = state.get_name() - if (name == state_name): - # some actions - #! [ov:low_latency_2] - - #! [ov:low_latency_2_use_parameters] - manager.register_pass(LowLatency2(False)) - #! [ov:low_latency_2_use_parameters] - - -def apply_make_stateful_tensor_names(): - #! [ov:make_stateful_tensor_names] - core = Core() - ov_model = core.read_model("path_to_the_model") - tensor_names = {"tensor_name_1": "tensor_name_4", - "tensor_name_3": "tensor_name_6"} - manager = Manager() - manager.register_pass(MakeStateful(tensor_names)) - manager.run_passes(ov_model) - #! [ov:make_stateful_tensor_names] - - -def apply_make_stateful_ov_nodes(): - #! [ov:make_stateful_ov_nodes] - core = Core() - ov_model = core.read_model("path_to_the_model") - # Parameter_1, Result_1, Parameter_3, Result_3 are - # ops.parameter/ops.result in the ov_model - pairs = ["""(Parameter_1, Result_1), (Parameter_3, Result_3)"""] - manager = Manager() - manager.register_pass(MakeStateful(pairs)) - manager.run_passes(ov_model) - #! [ov:make_stateful_ov_nodes] - - -def main(): - #! [ov:state_api_usage] - # 1. Load inference engine - log.info("Loading Inference Engine") - core = Core() - - # 2. Read a model - log.info("Loading network files") - model = core.read_model("path_to_the_model") - - - # 3. Load network to CPU - hw_specific_model = core.compile_model(model, "CPU") - - # 4. Create Infer Request - infer_request = hw_specific_model.create_infer_request() - - # 5. Reset memory states before starting - states = infer_request.query_state() - if (states.size() != 1): - log.error(f"Invalid queried state number. Expected 1, but got {str(states.size())}") - return -1 - - for state in states: - state.reset() - - # 6. Inference - input_data = np.arange(start=1, stop=12, dtype=np.float32) - - # This example demonstrates how to work with OpenVINO State API. - # Input_data: some array with 12 float numbers - - # Part1: read the first four elements of the input_data array sequentially. - # Expected output for the first utterance: - # sum of the previously processed elements [ 1, 3, 6, 10] - - # Part2: reset state value (set to 0) and read the next four elements. - # Expected output for the second utterance: - # sum of the previously processed elements [ 5, 11, 18, 26] - - # Part3: set state value to 5 and read the next four elements. - # Expected output for the third utterance: - # sum of the previously processed elements + 5 [ 14, 24, 35, 47] - target_state = states[0] - - # Part 1 - log.info("Infer the first utterance") - for next_input in range(len(input_data)/3): - infer_request.infer({0 : input_data[next_input]}) - state_buf = target_state.state.data - log.info(state_buf[0]) - - # Part 2 - log.info("\nReset state between utterances...\n") - target_state.reset() - - log.info("Infer the second utterance") - for next_input in range(len(input_data)/3, (len(input_data)/3 * 2)): - infer_request.infer({0 : input_data[next_input]}) - state_buf = target_state.state.data - log.info(state_buf[0]) - - # Part 3 - log.info("\nSet state value between utterances to 5...\n") - v = np.asarray([5], dtype=np.float32) - tensor = Tensor(v, shared_memory=True) - target_state.state = tensor - - log.info("Infer the third utterance") - for next_input in range((input_data.size()/3 * 2), input_data.size()): - infer_request.infer({0 : input_data[next_input]}) - - state_buf = target_state.state.data - log.info(state_buf[0]) - - log.info("Execution successful") - #! [ov:state_api_usage] - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docs/snippets/ov_preprocessing.py b/docs/snippets/ov_preprocessing.py index 45229d37a344c4..23cd30548115ad 100644 --- a/docs/snippets/ov_preprocessing.py +++ b/docs/snippets/ov_preprocessing.py @@ -2,24 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 # +import openvino.runtime.properties as props from openvino.preprocess import ResizeAlgorithm, ColorFormat -from openvino.runtime import Layout, Type, serialize +from openvino import Layout, Type, serialize +import openvino as ov +import openvino.runtime.opset12 as ops +from utils import get_model, get_temp_dir, get_path_to_model, get_advanced_model -xml_path = '' -input_name = '' +input_name = "input" +model = get_advanced_model() + # ! [ov:preprocess:create] from openvino.preprocess import PrePostProcessor -from openvino.runtime import Core -core = Core() -model = core.read_model(model=xml_path) ppp = PrePostProcessor(model) # ! [ov:preprocess:create] # ! [ov:preprocess:tensor] from openvino.preprocess import ColorFormat -from openvino.runtime import Layout, Type +from openvino import Layout, Type ppp.input(input_name).tensor() \ .set_element_type(Type.u8) \ .set_shape([1, 480, 640, 3]) \ @@ -52,10 +54,13 @@ # ! [ov:preprocess:input_name] -ppp.input('image') +ppp.input(input_name) ppp.output('result') # ! [ov:preprocess:input_name] +model = get_model() +ppp = PrePostProcessor(model) + # ! [ov:preprocess:input_1] # no index/name is needed if model has one input ppp.input().preprocess().scale(50.) @@ -65,75 +70,75 @@ .postprocess().convert_element_type(Type.u8) # ! [ov:preprocess:input_1] # ! [ov:preprocess:mean_scale] -ppp.input('input').preprocess().mean(128).scale(127) +ppp.input(input_name).preprocess().mean(128).scale(127) # ! [ov:preprocess:mean_scale] # ! [ov:preprocess:mean_scale_array] # Suppose model's shape is {1, 3, 224, 224} # N=1, C=3, H=224, W=224 -ppp.input('input').model().set_layout(Layout('NCHW')) +ppp.input(input_name).model().set_layout(Layout('NCHW')) # Mean/Scale has 3 values which matches with C=3 -ppp.input('input').preprocess() \ +ppp.input(input_name).preprocess() \ .mean([103.94, 116.78, 123.68]).scale([57.21, 57.45, 57.73]) # ! [ov:preprocess:mean_scale_array] # ! [ov:preprocess:convert_element_type] # First define data type for your tensor -ppp.input('input').tensor().set_element_type(Type.u8) +ppp.input(input_name).tensor().set_element_type(Type.u8) # Then define preprocessing step -ppp.input('input').preprocess().convert_element_type(Type.f32) +ppp.input(input_name).preprocess().convert_element_type(Type.f32) # If conversion is needed to `model's` element type, 'f32' can be omitted -ppp.input('input').preprocess().convert_element_type() +ppp.input(input_name).preprocess().convert_element_type() # ! [ov:preprocess:convert_element_type] # ! [ov:preprocess:convert_layout] # First define layout for your tensor -ppp.input('input').tensor().set_layout(Layout('NHWC')) +ppp.input(input_name).tensor().set_layout(Layout('NHWC')) # Then define layout of model -ppp.input('input').model().set_layout(Layout('NCHW')) +ppp.input(input_name).model().set_layout(Layout('NCHW')) print(ppp) # Will print 'implicit layout conversion step' # ! [ov:preprocess:convert_layout] # ! [ov:preprocess:convert_layout_2] -ppp.input('input').tensor().set_shape([1, 480, 640, 3]) +ppp.input(input_name).tensor().set_shape([1, 480, 640, 3]) # Model expects shape {1, 3, 480, 640} -ppp.input('input').preprocess()\ +ppp.input(input_name).preprocess()\ .convert_layout([0, 3, 1, 2]) # 0 -> 0; 3 -> 1; 1 -> 2; 2 -> 3 # ! [ov:preprocess:convert_layout_2] # ! [ov:preprocess:resize_1] -ppp.input('input').tensor().set_shape([1, 3, 960, 1280]) -ppp.input('input').model().set_layout(Layout('??HW')) -ppp.input('input').preprocess()\ +ppp.input(input_name).tensor().set_shape([1, 3, 960, 1280]) +ppp.input(input_name).model().set_layout(Layout('??HW')) +ppp.input(input_name).preprocess()\ .resize(ResizeAlgorithm.RESIZE_LINEAR, 480, 640) # ! [ov:preprocess:resize_1] # ! [ov:preprocess:resize_2] -ppp.input('input').tensor().set_shape([1, 3, 960, 1280]) +ppp.input(input_name).tensor().set_shape([1, 3, 960, 1280]) # Model accepts {1, 3, 480, 640} shape, thus last dimensions are 'H' and 'W' -ppp.input('input').model().set_layout(Layout('??HW')) +ppp.input(input_name).model().set_layout(Layout('??HW')) # Resize to model's dimension -ppp.input('input').preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) +ppp.input(input_name).preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) # ! [ov:preprocess:resize_2] # ! [ov:preprocess:convert_color_1] -ppp.input('input').tensor().set_color_format(ColorFormat.BGR) +ppp.input(input_name).tensor().set_color_format(ColorFormat.BGR) -ppp.input('input').preprocess().convert_color(ColorFormat.RGB) +ppp.input(input_name).preprocess().convert_color(ColorFormat.RGB) # ! [ov:preprocess:convert_color_1] # ! [ov:preprocess:convert_color_2] # This will split original `input` to 2 separate inputs: `input/y' and 'input/uv' -ppp.input('input').tensor()\ +ppp.input(input_name).tensor()\ .set_color_format(ColorFormat.NV12_TWO_PLANES) -ppp.input('input').preprocess()\ +ppp.input(input_name).preprocess()\ .convert_color(ColorFormat.RGB) print(ppp) # Dump preprocessing steps to see what will happen # ! [ov:preprocess:convert_color_2] # ! [ov:preprocess:custom] # It is possible to insert some custom operations -import openvino.runtime.opset8 as ops +import openvino.runtime.opset12 as ops from openvino.runtime import Output from openvino.runtime.utils.decorators import custom_preprocess_function @@ -142,22 +147,22 @@ def custom_abs(output: Output): # Custom nodes can be inserted as Preprocessing steps return ops.abs(output) -ppp.input("input_image").preprocess() \ +ppp.input("input").preprocess() \ .custom(custom_abs) # ! [ov:preprocess:custom] # ! [ov:preprocess:postprocess] # Model's output has 'NCHW' layout -ppp.output('result_image').model().set_layout(Layout('NCHW')) +ppp.output('result').model().set_layout(Layout('NCHW')) # Set target user's tensor to U8 type + 'NHWC' layout # Precision & layout conversions will be done implicitly -ppp.output('result_image').tensor()\ +ppp.output('result').tensor()\ .set_layout(Layout("NHWC"))\ .set_element_type(Type.u8) # Also it is possible to insert some custom operations -import openvino.runtime.opset8 as ops +import openvino.runtime.opset12 as ops from openvino.runtime import Output from openvino.runtime.utils.decorators import custom_preprocess_function @@ -166,23 +171,23 @@ def custom_abs(output: Output): # Custom nodes can be inserted as Post-processing steps return ops.abs(output) -ppp.output("result_image").postprocess()\ +ppp.output("result").postprocess()\ .custom(custom_abs) # ! [ov:preprocess:postprocess] # ! [ov:preprocess:save_headers] from openvino.preprocess import PrePostProcessor, ColorFormat, ResizeAlgorithm -from openvino.runtime import Core, Layout, Type, set_batch -# First method - imports -from openvino.runtime import serialize -# Second method - imports -from openvino.runtime.passes import Manager, Serialize +from openvino import Core, Layout, Type, set_batch +from openvino import serialize # ! [ov:preprocess:save_headers] +model_path = get_path_to_model() +serialized_model_path = get_path_to_model() + # ! [ov:preprocess:save] # ======== Step 0: read original model ========= core = Core() -model = core.read_model(model='/path/to/some_model.onnx') +model = core.read_model(model=model_path) # ======== Step 1: Preprocessing ================ ppp = PrePostProcessor(model) @@ -213,20 +218,16 @@ def custom_abs(output: Output): set_batch(model, 2) # ======== Step 3: Save the model ================ -# First method - using serialize runtime wrapper -serialize(model, '/path/to/some_model_saved.xml', '/path/to/some_model_saved.bin') - -# Second method - using Manager and Serialize pass -manager = Manager() -manager.register_pass(Serialize('/path/to/some_model_saved.xml', '/path/to/some_model_saved.bin')) -manager.run_passes(model) +serialize(model, serialized_model_path) # ! [ov:preprocess:save] +path_to_cache_dir = get_temp_dir() + # ! [ov:preprocess:save_load] core = Core() -core.set_property({'CACHE_DIR': '/path/to/cache/dir'}) +core.set_property({props.cache_dir(): path_to_cache_dir}) # In case that no preprocessing is needed anymore, we can load model on target device directly # With cached model available, it will also save some time on reading original model -compiled_model = core.compile_model('/path/to/some_model_saved.xml', 'CPU') +compiled_model = core.compile_model(serialized_model_path, 'CPU') # ! [ov:preprocess:save_load] diff --git a/docs/snippets/ov_preprocessing_migration.py b/docs/snippets/ov_preprocessing_migration.py index 6426cbed8c82ed..86d28bad6a3c4c 100644 --- a/docs/snippets/ov_preprocessing_migration.py +++ b/docs/snippets/ov_preprocessing_migration.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # +from utils import get_model, get_ngraph_model + #! [ov_imports] -from openvino.runtime import Core, Layout, Type +from openvino import Core, Layout, Type from openvino.preprocess import ColorFormat, PrePostProcessor, ResizeAlgorithm #! [ov_imports] @@ -13,11 +15,9 @@ #include "inference_engine.hpp" -model_path = '' -tensor_name = '' - +tensor_name="input" core = Core() -model = core.read_model(model=model_path) +model = get_model([1,32,32,3]) #! [ov_mean_scale] ppp = PrePostProcessor(model) @@ -30,6 +30,8 @@ model = ppp.build() #! [ov_mean_scale] +model = get_model() + #! [ov_conversions] ppp = PrePostProcessor(model) input = ppp.input(tensor_name) @@ -50,9 +52,11 @@ model = ppp.build() #! [ov_color_space] +model = get_model([1, 3, 448, 448]) + #! [ov_image_scale] ppp = PrePostProcessor(model) -input = ppp.input(tensor_name) +input = ppp.input("input") # need to specify H and W dimensions in model, others are not important input.model().set_layout(Layout('??HW')) # scale to model shape @@ -61,39 +65,38 @@ model = ppp.build() #! [ov_image_scale] +import openvino.inference_engine as ie +import ngraph as ng +operation_name = "data" -model_path = '' -operation_name = '' - -core = Core() -network = core.ReadNetwork(model_path) +core = ie.IECore() +network = get_ngraph_model() #! [mean_scale] -preProcess = network.getInputsInfo()[operation_name].getPreProcess() -preProcess.init(3) -preProcess[0].meanValue = 116.78 -preProcess[1].meanValue = 116.78 -preProcess[2].meanValue = 116.78 -preProcess[0].stdScale = 57.21 -preProcess[1].stdScale = 57.45 -preProcess[2].stdScale = 57.73 -preProcess.setVariant(ie.MEAN_VALUE) +preprocess_info = network.input_info[operation_name].preprocess_info +preprocess_info.init(3) +preprocess_info[0].mean_value = 116.78 +preprocess_info[1].mean_value = 116.78 +preprocess_info[2].mean_value = 116.78 +preprocess_info[0].std_scale = 57.21 +preprocess_info[1].std_scale = 57.45 +preprocess_info[2].std_scale = 57.73 +preprocess_info.mean_variant = ie.MeanVariant.MEAN_VALUE #! [mean_scale] #! [conversions] -inputInfo = network.getInputsInfo()[operation_name] -inputInfo.setPrecision(ie.Precision.U8) -inputInfo.setLayout(ie.Layout.NHWC) +input_info = network.input_info[operation_name] +input_info.precision = "U8" +input_info.layout = "NHWC" # model input layout is always NCHW in Inference Engine # for shapes with 4 dimensions #! [conversions] #! [image_scale] -preProcess = network.getInputsInfo()[operation_name].getPreProcess() +preprocess_info = network.input_info[operation_name].preprocess_info # Inference Engine supposes input for resize is always in NCHW layout # while for OpenVINO Runtime API 2.0 `H` and `W` dimensions must be specified # Also, current code snippet supposed resize from dynamic shapes -preProcess.setResizeAlgorithm(ie.ResizeAlgorithm.RESIZE_BILINEAR) -#! [image_scale] +preprocess_info.resize_algorithm = ie.ResizeAlgorithm.RESIZE_BILINEAR diff --git a/docs/snippets/ov_properties_api.py b/docs/snippets/ov_properties_api.py index f9c4b8913294f9..cc9645a7682e92 100644 --- a/docs/snippets/ov_properties_api.py +++ b/docs/snippets/ov_properties_api.py @@ -2,53 +2,62 @@ # SPDX-License-Identifier: Apache-2.0 # -from openvino.runtime import Core - -# [get_available_devices] -core = Core() -available_devices = core.available_devices -# [get_available_devices] - -# [hetero_priorities] -device_priorites = core.get_property("HETERO", "MULTI_DEVICE_PRIORITIES") -# [hetero_priorities] - -# [cpu_device_name] -cpu_device_name = core.get_property("CPU", "FULL_DEVICE_NAME") -# [cpu_device_name] - -model = core.read_model(model="sample.xml") -# [compile_model_with_property] -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "INFERENCE_PRECISION_HINT": "f32"} -compiled_model = core.compile_model(model, "CPU", config) -# [compile_model_with_property] - -# [optimal_number_of_infer_requests] -compiled_model = core.compile_model(model, "CPU") -nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -# [optimal_number_of_infer_requests] - - -# [core_set_property_then_compile] -# latency hint is a default for CPU -core.set_property("CPU", {"PERFORMANCE_HINT": "LATENCY"}) -# compiled with latency configuration hint -compiled_model_latency = core.compile_model(model, "CPU") -# compiled with overriden performance hint value -config = {"PERFORMANCE_HINT": "THROUGHPUT"} -compiled_model_thrp = core.compile_model(model, "CPU", config) -# [core_set_property_then_compile] - - -# [inference_num_threads] -compiled_model = core.compile_model(model, "CPU") -nthreads = compiled_model.get_property("INFERENCE_NUM_THREADS") -# [inference_num_threads] - -# [multi_device] -config = {"MULTI_DEVICE_PRIORITIES": "CPU,GPU"} -compiled_model = core.compile_model(model, "MULTI", config) -# change the order of priorities -compiled_model.set_property({"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) -# [multi_device] +import openvino as ov +import openvino.runtime.properties as props +import openvino.runtime.properties.hint as hints + +from utils import get_model + + +def main(): + # [get_available_devices] + core = ov.Core() + available_devices = core.available_devices + # [get_available_devices] + + # [hetero_priorities] + device_priorites = core.get_property("HETERO", props.device.priorities()) + # [hetero_priorities] + + # [cpu_device_name] + cpu_device_name = core.get_property("CPU", props.device.full_name()) + # [cpu_device_name] + + model = get_model() + # [compile_model_with_property] + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.inference_precision(): ov.Type.f32} + compiled_model = core.compile_model(model, "CPU", config) + # [compile_model_with_property] + + # [optimal_number_of_infer_requests] + compiled_model = core.compile_model(model, "CPU") + nireq = compiled_model.get_property(props.optimal_number_of_infer_requests()) + # [optimal_number_of_infer_requests] + + + # [core_set_property_then_compile] + # latency hint is a default for CPU + core.set_property("CPU", {hints.performance_mode(): hints.PerformanceMode.LATENCY}) + # compiled with latency configuration hint + compiled_model_latency = core.compile_model(model, "CPU") + # compiled with overriden performance hint value + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT} + compiled_model_thrp = core.compile_model(model, "CPU", config) + # [core_set_property_then_compile] + + + # [inference_num_threads] + compiled_model = core.compile_model(model, "CPU") + nthreads = compiled_model.get_property(props.inference_num_threads()) + # [inference_num_threads] + + if "GPU" not in available_devices: + return 0 + + # [multi_device] + config = {props.device.priorities(): "CPU,GPU"} + compiled_model = core.compile_model(model, "MULTI", config) + # change the order of priorities + compiled_model.set_property({props.device.priorities(): "GPU,CPU"}) + # [multi_device] diff --git a/docs/snippets/ov_properties_migration.py b/docs/snippets/ov_properties_migration.py index 82f6a37b9e1a0b..a590e3053e8312 100644 --- a/docs/snippets/ov_properties_migration.py +++ b/docs/snippets/ov_properties_migration.py @@ -2,81 +2,88 @@ # SPDX-License-Identifier: Apache-2.0 # -from openvino.runtime import Core - - -core = Core() - -# ! [core_set_property] -core.set_property(device_name="CPU", properties={"PERF_COUNT": "YES"}) -# ! [core_set_property] - -model = core.read_model("sample.xml") - -# ! [core_compile_model] -compiled_model = core.compile_model(model=model, device_name="MULTI", config= - { - "MULTI_DEVICE_PRIORITIES": "GPU,CPU", - "PERFORMANCE_HINT": "THROUGHPUT", - "INFERENCE_PRECISION_HINT": "f32" - }) -# ! [core_compile_model] - -# ! [compiled_model_set_property] -# turn CPU off for multi-device execution -compiled_model.set_property(properties={"MULTI_DEVICE_PRIORITIES": "GPU"}) -# ! [compiled_model_set_property] - -# ! [core_get_rw_property] -num_streams = core.get_property("CPU", "NUM_STREAMS") -# ! [core_get_rw_property] - -# ! [core_get_ro_property] -full_device_name = core.get_property("CPU", "FULL_DEVICE_NAME") -# ! [core_get_ro_property] - -# ! [compiled_model_get_rw_property] -perf_mode = compiled_model.get_property("PERFORMANCE_HINT") -# ! [compiled_model_get_rw_property] - -# ! [compiled_model_get_ro_property] -nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -# ! [compiled_model_get_ro_property] - - -from openvino.inference_engine import IECore - - -core = IECore() -#! [core_get_metric] -full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME") -#! [core_get_metric] - -#! [core_get_config] -num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS") -#! [core_get_config] - -#! [core_set_config] -core.set_config({"PERF_COUNT": "YES"}, "CPU") -#! [core_set_config] - -net = core.read_network("sample.xml") - -#! [core_load_network] -exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU", - "PERFORMANCE_HINT": "THROUGHPUT", - "ENFORCE_BF16": "NO"}) -#! [core_load_network] - -#! [executable_network_set_config] -# turn CPU off for multi-device execution -exec_network.set_config({"DEVICE_PRIORITIES": "GPU"}) -#! [executable_network_set_config] - -#! [executable_network_get_metric] -nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -#! [executable_network_get_metric] - -#! [executable_network_get_config] -perf_hint = exec_network.get_config("PERFORMANCE_HINT") -#! [executable_network_get_config] +import openvino as ov +import openvino.runtime.properties as props +import openvino.runtime.properties.hint as hints + +from utils import get_model + +def main(): + core = ov.Core() + + # ! [core_set_property] + core.set_property(device_name="CPU", properties={props.enable_profiling(): True}) + # ! [core_set_property] + + model = get_model() + + if "GPU" not in core.available_devices: + return 0 + + # ! [core_compile_model] + compiled_model = core.compile_model(model=model, device_name="MULTI", config= + { + props.device.priorities(): "GPU,CPU", + hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.inference_precision(): ov.Type.f32 + }) + # ! [core_compile_model] + + # ! [compiled_model_set_property] + # turn CPU off for multi-device execution + compiled_model.set_property(properties={props.device.priorities(): "GPU"}) + # ! [compiled_model_set_property] + + # ! [core_get_rw_property] + num_streams = core.get_property("CPU", props.streams.num()) + # ! [core_get_rw_property] + + # ! [core_get_ro_property] + full_device_name = core.get_property("CPU", props.device.full_name()) + # ! [core_get_ro_property] + + # ! [compiled_model_get_rw_property] + perf_mode = compiled_model.get_property(hints.performance_mode()) + # ! [compiled_model_get_rw_property] + + # ! [compiled_model_get_ro_property] + nireq = compiled_model.get_property(props.optimal_number_of_infer_requests()) + # ! [compiled_model_get_ro_property] + + import ngraph as ng + import openvino.inference_engine as ie + from utils import get_ngraph_model + + core = ie.IECore() + #! [core_get_metric] + full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME") + #! [core_get_metric] + + #! [core_get_config] + num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS") + #! [core_get_config] + + #! [core_set_config] + core.set_config({"PERF_COUNT": "YES"}, "CPU") + #! [core_set_config] + + net = get_ngraph_model() + + #! [core_load_network] + exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU", + "PERFORMANCE_HINT": "THROUGHPUT", + "ENFORCE_BF16": "NO"}) + #! [core_load_network] + + #! [executable_network_set_config] + # turn CPU off for multi-device execution + exec_network.set_config({"DEVICE_PRIORITIES": "GPU"}) + #! [executable_network_set_config] + + #! [executable_network_get_metric] + nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS") + #! [executable_network_get_metric] + + #! [executable_network_get_config] + perf_hint = exec_network.get_config("PERFORMANCE_HINT") + #! [executable_network_get_config] diff --git a/docs/snippets/ov_python_exclusives.py b/docs/snippets/ov_python_exclusives.py index a8c35285969fb0..556735215dcd5d 100644 --- a/docs/snippets/ov_python_exclusives.py +++ b/docs/snippets/ov_python_exclusives.py @@ -2,18 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +import openvino as ov -#! [auto_compilation] -import openvino.runtime as ov - -compiled_model = ov.compile_model("model.xml") -#! [auto_compilation] +from utils import get_path_to_model, get_image #! [properties_example] +import openvino.runtime.opset12 as ops + core = ov.Core() -input_a = ov.opset11.parameter([8], name="input_a") -res = ov.opset11.absolute(input_a) +input_a = ops.parameter([8], name="input_a") +res = ops.absolute(input_a) model = ov.Model(res, [input_a]) compiled = core.compile_model(model, "CPU") model.outputs[0].tensor.set_names({"result_0"}) # Add name for Output @@ -25,6 +24,10 @@ print(compiled.outputs) #! [properties_example] +#! [auto_compilation] +compiled_model = ov.compile_model(model) +#! [auto_compilation] + #! [tensor_basics] data_float64 = np.ones(shape=(2,8)) @@ -37,6 +40,7 @@ assert tensor.element_type == ov.Type.i32 #! [tensor_basics] + #! [tensor_shared_mode] data_to_share = np.ones(shape=(2,8)) @@ -98,9 +102,9 @@ core = ov.Core() # Simple model that adds two inputs together -input_a = ov.opset8.parameter([8]) -input_b = ov.opset8.parameter([8]) -res = ov.opset8.add(input_a, input_b) +input_a = ops.parameter([8]) +input_b = ops.parameter([8]) +res = ops.add(input_a, input_b) model = ov.Model(res, [input_a, input_b]) compiled = core.compile_model(model, "CPU") @@ -137,14 +141,14 @@ def f(request, userdata): assert all(data_done) #! [asyncinferqueue_set_callback] -unt8_data = np.ones([100]) +unt8_data = np.ones([100], dtype=np.uint8) #! [packing_data] from openvino.helpers import pack_data packed_buffer = pack_data(unt8_data, ov.Type.u4) # Create tensor with shape in element types -t = ov.Tensor(packed_buffer, [1, 128], ov.Type.u4) +t = ov.Tensor(packed_buffer, [100], ov.Type.u4) #! [packing_data] #! [unpacking] @@ -154,9 +158,12 @@ def f(request, userdata): assert np.array_equal(unpacked_data , unt8_data) #! [unpacking] +model_path = get_path_to_model() +image = get_image() + + #! [releasing_gil] -import openvino.runtime as ov -import cv2 as cv +import openvino as ov from threading import Thread input_data = [] @@ -164,23 +171,20 @@ def f(request, userdata): # Processing input data will be done in a separate thread # while compilation of the model and creation of the infer request # is going to be executed in the main thread. -def prepare_data(input, image_path): - image = cv.imread(image_path) - h, w = list(input.shape)[-2:] - image = cv.resize(image, (h, w)) - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, 0) - input_data.append(image) +def prepare_data(input, image): + shape = list(input.shape) + resized_img = np.resize(image, shape) + input_data.append(resized_img) core = ov.Core() -model = core.read_model("model.xml") +model = core.read_model(model_path) # Create thread with prepare_data function as target and start it -thread = Thread(target=prepare_data, args=[model.input(), "path/to/image"]) +thread = Thread(target=prepare_data, args=[model.input(), image]) thread.start() # The GIL will be released in compile_model. # It allows a thread above to start the job, # while main thread is running in the background. -compiled = core.compile_model(model, "GPU") +compiled = core.compile_model(model, "CPU") # After returning from compile_model, the main thread acquires the GIL # and starts create_infer_request which releases it once again. request = compiled.create_infer_request() diff --git a/docs/snippets/ov_python_inference.py b/docs/snippets/ov_python_inference.py index d55afeb5bf177e..9d2fb0fb139e2b 100644 --- a/docs/snippets/ov_python_inference.py +++ b/docs/snippets/ov_python_inference.py @@ -2,14 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np -import openvino.runtime as ov +import openvino as ov +import openvino.runtime.opset12 as ops INPUT_SIZE = 1_000_000 # Use bigger values if necessary, i.e.: 300_000_000 -input_0 = ov.opset11.parameter([INPUT_SIZE], name="input_0") -input_1 = ov.opset11.parameter([INPUT_SIZE], name="input_1") -add_inputs = ov.opset11.add(input_0, input_1) -res = ov.opset11.reduce_sum(add_inputs, reduction_axes=0, name="reduced") +input_0 = ops.parameter([INPUT_SIZE], name="input_0") +input_1 = ops.parameter([INPUT_SIZE], name="input_1") +add_inputs = ops.add(input_0, input_1) +res = ops.reduce_sum(add_inputs, reduction_axes=0, name="reduced") model = ov.Model(res, [input_0, input_1], name="my_model") model.outputs[0].tensor.set_names({"reduced_result"}) # Add name for Output @@ -56,7 +57,7 @@ def run(time_in_sec): # Hiding latency request.start_async({"input_0": data_0, "input_1": data_1}) -run() +run(time_in_sec) request.wait() results = request.get_output_tensor(0).data # Gather data from InferRequest #! [hiding_latency] diff --git a/docs/snippets/src/main.py b/docs/snippets/src/main.py index fe14021bbabc60..bb9dd4b1185edf 100644 --- a/docs/snippets/src/main.py +++ b/docs/snippets/src/main.py @@ -3,7 +3,7 @@ import numpy as np #! [import] -import openvino.runtime as ov +import openvino as ov #! [import] #! [part1] diff --git a/docs/snippets/utils.py b/docs/snippets/utils.py new file mode 100644 index 00000000000000..de96f631ed8bf5 --- /dev/null +++ b/docs/snippets/utils.py @@ -0,0 +1,86 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import tempfile +import numpy as np +from sys import platform + +import openvino as ov +import openvino.runtime.opset12 as ops + +import ngraph as ng +from ngraph.impl import Function +import openvino.inference_engine as ie + + +def get_dynamic_model(): + param = ops.parameter(ov.PartialShape([-1, -1]), name="input") + return ov.Model(ops.relu(param), [param]) + + +def get_model(input_shape = None, input_dtype=np.float32) -> ov.Model: + if input_shape is None: + input_shape = [1, 3, 32, 32] + param = ops.parameter(input_shape, input_dtype, name="input") + relu = ops.relu(param, name="relu") + relu.output(0).get_tensor().set_names({"result"}) + model = ov.Model([relu], [param], "test_model") + + assert model is not None + return model + + +def get_advanced_model(): + data = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32, name="input") + data1 = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32, name="input2") + axis_const = ops.constant(1, dtype=ov.Type.i64) + split = ops.split(data, axis_const, 3) + relu = ops.relu(split.output(1)) + relu.output(0).get_tensor().set_names({"result"}) + return ov.Model([split.output(0), relu.output(0), split.output(2)], [data, data1], "model") + + +def get_ngraph_model(input_shape = None, input_dtype=np.float32): + if input_shape is None: + input_shape = [1, 3, 32, 32] + param = ng.opset11.parameter(input_shape, input_dtype, name="data") + relu = ng.opset11.relu(param, name="relu") + func = Function([relu], [param], "test_model") + caps = ng.Function.to_capsule(func) + net = ie.IENetwork(caps) + + assert net is not None + return net + + +def get_image(shape = (1, 3, 32, 32), dtype = "float32"): + np.random.seed(42) + return np.random.rand(*shape).astype(dtype) + + +def get_path_to_extension_library(): + library_path="" + if platform == "win32": + library_path="openvino_template_extension.dll" + else: + library_path="libopenvino_template_extension.so" + return library_path + + +def get_path_to_model(input_shape = None, is_old_api=False): + if input_shape is None: + input_shape = [1, 3, 32, 32] + path_to_xml = tempfile.NamedTemporaryFile(suffix="_model.xml").name + if is_old_api: + net = get_ngraph_model(input_shape, input_dtype=np.int64) + net.serialize(path_to_xml) + else: + model = get_model(input_shape) + ov.serialize(model, path_to_xml) + return path_to_xml + + +def get_temp_dir(): + temp_dir = tempfile.TemporaryDirectory() + return temp_dir diff --git a/samples/python/benchmark/bert_benchmark/bert_benchmark.py b/samples/python/benchmark/bert_benchmark/bert_benchmark.py index eb50fb7bb52e0a..b15bbd48b34c84 100755 --- a/samples/python/benchmark/bert_benchmark/bert_benchmark.py +++ b/samples/python/benchmark/bert_benchmark/bert_benchmark.py @@ -10,7 +10,8 @@ from time import perf_counter import datasets -from openvino.runtime import Core, get_version, AsyncInferQueue, PartialShape +import openvino as ov +from openvino.runtime import get_version from transformers import AutoTokenizer from transformers.onnx import export from transformers.onnx.features import FeaturesManager @@ -28,7 +29,7 @@ def main(): # Download the tokenizer tokenizer = AutoTokenizer.from_pretrained(model_name) - core = Core() + core = ov.Core() with tempfile.TemporaryDirectory() as tmp: onnx_path = Path(tmp) / f'{model_name}.onnx' @@ -39,7 +40,7 @@ def main(): # Enforce dynamic input shape try: - model.reshape({model_input.any_name: PartialShape([1, '?']) for model_input in model.inputs}) + model.reshape({model_input.any_name: ov.PartialShape([1, '?']) for model_input in model.inputs}) except RuntimeError: log.error("Can't set dynamic shape") raise @@ -50,7 +51,7 @@ def main(): # It is possible to set CUMULATIVE_THROUGHPUT as PERFORMANCE_HINT for AUTO device compiled_model = core.compile_model(model, 'CPU', tput) # AsyncInferQueue creates optimal number of InferRequest instances - ireqs = AsyncInferQueue(compiled_model) + ireqs = ov.AsyncInferQueue(compiled_model) sst2 = datasets.load_dataset('glue', 'sst2') sst2_sentences = sst2['validation']['sentence'] diff --git a/samples/python/benchmark/sync_benchmark/sync_benchmark.py b/samples/python/benchmark/sync_benchmark/sync_benchmark.py index 367776356d389f..e270d25a64fbbd 100755 --- a/samples/python/benchmark/sync_benchmark/sync_benchmark.py +++ b/samples/python/benchmark/sync_benchmark/sync_benchmark.py @@ -9,7 +9,8 @@ from time import perf_counter import numpy as np -from openvino.runtime import Core, get_version +import openvino as ov +from openvino.runtime import get_version from openvino.runtime.utils.types import get_dtype @@ -40,7 +41,7 @@ def main(): # Pick a device by replacing CPU, for example AUTO:GPU,CPU. # Using MULTI device is pointless in sync scenario # because only one instance of openvino.runtime.InferRequest is used - core = Core() + core = ov.Core() compiled_model = core.compile_model(sys.argv[1], 'CPU', latency) ireq = compiled_model.create_infer_request() # Fill input data for the ireq diff --git a/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py b/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py index 3867e785b31ac9..c934a7650172e6 100755 --- a/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py +++ b/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py @@ -9,7 +9,8 @@ from time import perf_counter import numpy as np -from openvino.runtime import Core, get_version, AsyncInferQueue +import openvino as ov +from openvino.runtime import get_version from openvino.runtime.utils.types import get_dtype @@ -39,10 +40,10 @@ def main(): # Create Core and use it to compile a model. # Pick a device by replacing CPU, for example MULTI:CPU(4),GPU(8). # It is possible to set CUMULATIVE_THROUGHPUT as PERFORMANCE_HINT for AUTO device - core = Core() + core = ov.Core() compiled_model = core.compile_model(sys.argv[1], 'CPU', tput) # AsyncInferQueue creates optimal number of InferRequest instances - ireqs = AsyncInferQueue(compiled_model) + ireqs = ov.AsyncInferQueue(compiled_model) # Fill input data for ireqs for ireq in ireqs: for model_input in compiled_model.inputs: diff --git a/samples/python/classification_sample_async/classification_sample_async.py b/samples/python/classification_sample_async/classification_sample_async.py index 8e0b8d244d3a0c..92180f0534eb4e 100755 --- a/samples/python/classification_sample_async/classification_sample_async.py +++ b/samples/python/classification_sample_async/classification_sample_async.py @@ -9,8 +9,7 @@ import cv2 import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import AsyncInferQueue, Core, InferRequest, Layout, Type +import openvino as ov def parse_args() -> argparse.Namespace: @@ -32,7 +31,7 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def completion_callback(infer_request: InferRequest, image_path: str) -> None: +def completion_callback(infer_request: ov.InferRequest, image_path: str) -> None: predictions = next(iter(infer_request.results.values())) # Change a shape of a numpy.ndarray with results to get another one with one dimension @@ -61,7 +60,7 @@ def main() -> int: # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {args.model}') @@ -88,22 +87,22 @@ def main() -> int: input_tensors = [np.expand_dims(image, 0) for image in resized_images] # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() @@ -115,7 +114,7 @@ def main() -> int: # --------------------------- Step 6. Create infer request queue ------------------------------------------------------ log.info('Starting inference in asynchronous mode') # create async queue with optimal number of infer requests - infer_queue = AsyncInferQueue(compiled_model) + infer_queue = ov.AsyncInferQueue(compiled_model) infer_queue.set_callback(completion_callback) # --------------------------- Step 7. Do inference -------------------------------------------------------------------- diff --git a/samples/python/hello_classification/hello_classification.py b/samples/python/hello_classification/hello_classification.py index 135422ef96cf21..9e391c322196b1 100755 --- a/samples/python/hello_classification/hello_classification.py +++ b/samples/python/hello_classification/hello_classification.py @@ -8,8 +8,7 @@ import cv2 import numpy as np -from openvino.preprocess import PrePostProcessor, ResizeAlgorithm -from openvino.runtime import Core, Layout, Type +import openvino as ov def main(): @@ -26,7 +25,7 @@ def main(): # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {model_path}') @@ -48,7 +47,7 @@ def main(): input_tensor = np.expand_dims(image, 0) # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) _, h, w, _ = input_tensor.shape @@ -58,19 +57,19 @@ def main(): # - layout of data is 'NHWC' ppp.input().tensor() \ .set_shape(input_tensor.shape) \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: ECE001, N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: ECE001, N400 # 2) Adding explicit preprocessing steps: # - apply linear resize from tensor spatial dims to model spatial dims - ppp.input().preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) + ppp.input().preprocess().resize(ov.preprocess.ResizeAlgorithm.RESIZE_LINEAR) # 3) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 4) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 5) Apply preprocessing modifying the original 'model' model = ppp.build() diff --git a/samples/python/hello_query_device/hello_query_device.py b/samples/python/hello_query_device/hello_query_device.py index 5da48752427982..8efadc1466eff4 100755 --- a/samples/python/hello_query_device/hello_query_device.py +++ b/samples/python/hello_query_device/hello_query_device.py @@ -5,7 +5,7 @@ import logging as log import sys -from openvino.runtime import Core +import openvino as ov def param_to_string(parameters) -> str: @@ -20,7 +20,7 @@ def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) # --------------------------- Step 1. Initialize OpenVINO Runtime Core -------------------------------------------- - core = Core() + core = ov.Core() # --------------------------- Step 2. Get metrics of available devices -------------------------------------------- log.info('Available devices:') diff --git a/samples/python/hello_reshape_ssd/hello_reshape_ssd.py b/samples/python/hello_reshape_ssd/hello_reshape_ssd.py index e2e81d67bcc5cd..40d4afdbcf0819 100755 --- a/samples/python/hello_reshape_ssd/hello_reshape_ssd.py +++ b/samples/python/hello_reshape_ssd/hello_reshape_ssd.py @@ -9,8 +9,7 @@ import cv2 import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import Core, Layout, PartialShape, Type +import openvino as ov def main(): @@ -27,7 +26,7 @@ def main(): # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {model_path}') @@ -50,25 +49,25 @@ def main(): log.info('Reshaping the model to the height and width of the input image') n, h, w, c = input_tensor.shape - model.reshape({model.input().get_any_name(): PartialShape((n, c, h, w))}) + model.reshape({model.input().get_any_name(): ov.PartialShape((n, c, h, w))}) # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() diff --git a/samples/python/model_creation_sample/model_creation_sample.py b/samples/python/model_creation_sample/model_creation_sample.py index 27792fef5c0c57..d76d222c5f870e 100755 --- a/samples/python/model_creation_sample/model_creation_sample.py +++ b/samples/python/model_creation_sample/model_creation_sample.py @@ -8,14 +8,13 @@ from functools import reduce import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import (Core, Layout, Model, Shape, Type, op, opset1, - opset8, set_batch) +import openvino as ov +from openvino.runtime import op, opset1, opset8 from data import digits -def create_ngraph_function(model_path: str) -> Model: +def create_ngraph_function(model_path: str) -> ov.Model: """Create a model on the fly from the source code using ngraph.""" def shape_and_length(shape: list) -> typing.Tuple[list, int]: @@ -28,17 +27,17 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: # input input_shape = [64, 1, 28, 28] - param_node = op.Parameter(Type.f32, Shape(input_shape)) + param_node = op.Parameter(ov.Type.f32, ov.Shape(input_shape)) # convolution 1 conv_1_kernel_shape, conv_1_kernel_length = shape_and_length([20, 1, 5, 5]) - conv_1_kernel = op.Constant(Type.f32, Shape(conv_1_kernel_shape), weights[0:conv_1_kernel_length].tolist()) + conv_1_kernel = op.Constant(ov.Type.f32, ov.Shape(conv_1_kernel_shape), weights[0:conv_1_kernel_length].tolist()) weights_offset += conv_1_kernel_length conv_1_node = opset8.convolution(param_node, conv_1_kernel, [1, 1], padding_begin, padding_end, [1, 1]) # add 1 add_1_kernel_shape, add_1_kernel_length = shape_and_length([1, 20, 1, 1]) - add_1_kernel = op.Constant(Type.f32, Shape(add_1_kernel_shape), + add_1_kernel = op.Constant(ov.Type.f32, ov.Shape(add_1_kernel_shape), weights[weights_offset : weights_offset + add_1_kernel_length]) weights_offset += add_1_kernel_length add_1_node = opset8.add(conv_1_node, add_1_kernel) @@ -48,7 +47,7 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: # convolution 2 conv_2_kernel_shape, conv_2_kernel_length = shape_and_length([50, 20, 5, 5]) - conv_2_kernel = op.Constant(Type.f32, Shape(conv_2_kernel_shape), + conv_2_kernel = op.Constant(ov.Type.f32, ov.Shape(conv_2_kernel_shape), weights[weights_offset : weights_offset + conv_2_kernel_length], ) weights_offset += conv_2_kernel_length @@ -56,7 +55,7 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: # add 2 add_2_kernel_shape, add_2_kernel_length = shape_and_length([1, 50, 1, 1]) - add_2_kernel = op.Constant(Type.f32, Shape(add_2_kernel_shape), + add_2_kernel = op.Constant(ov.Type.f32, ov.Shape(add_2_kernel_shape), weights[weights_offset : weights_offset + add_2_kernel_length], ) weights_offset += add_2_kernel_length @@ -72,13 +71,13 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: weights[weights_offset : weights_offset + 2 * reshape_1_length], dtype=np.int64, ) - reshape_1_kernel = op.Constant(Type.i64, Shape(list(dtype_weights.shape)), dtype_weights) + reshape_1_kernel = op.Constant(ov.Type.i64, ov.Shape(list(dtype_weights.shape)), dtype_weights) weights_offset += 2 * reshape_1_length reshape_1_node = opset8.reshape(maxpool_2_node, reshape_1_kernel, True) # matmul 1 matmul_1_kernel_shape, matmul_1_kernel_length = shape_and_length([500, 800]) - matmul_1_kernel = op.Constant(Type.f32, Shape(matmul_1_kernel_shape), + matmul_1_kernel = op.Constant(ov.Type.f32, ov.Shape(matmul_1_kernel_shape), weights[weights_offset : weights_offset + matmul_1_kernel_length], ) weights_offset += matmul_1_kernel_length @@ -86,7 +85,7 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: # add 3 add_3_kernel_shape, add_3_kernel_length = shape_and_length([1, 500]) - add_3_kernel = op.Constant(Type.f32, Shape(add_3_kernel_shape), + add_3_kernel = op.Constant(ov.Type.f32, ov.Shape(add_3_kernel_shape), weights[weights_offset : weights_offset + add_3_kernel_length], ) weights_offset += add_3_kernel_length @@ -96,12 +95,12 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: relu_node = opset8.relu(add_3_node) # reshape 2 - reshape_2_kernel = op.Constant(Type.i64, Shape(list(dtype_weights.shape)), dtype_weights) + reshape_2_kernel = op.Constant(ov.Type.i64, ov.Shape(list(dtype_weights.shape)), dtype_weights) reshape_2_node = opset8.reshape(relu_node, reshape_2_kernel, True) # matmul 2 matmul_2_kernel_shape, matmul_2_kernel_length = shape_and_length([10, 500]) - matmul_2_kernel = op.Constant(Type.f32, Shape(matmul_2_kernel_shape), + matmul_2_kernel = op.Constant(ov.Type.f32, ov.Shape(matmul_2_kernel_shape), weights[weights_offset : weights_offset + matmul_2_kernel_length], ) weights_offset += matmul_2_kernel_length @@ -109,7 +108,7 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: # add 4 add_4_kernel_shape, add_4_kernel_length = shape_and_length([1, 10]) - add_4_kernel = op.Constant(Type.f32, Shape(add_4_kernel_shape), + add_4_kernel = op.Constant(ov.Type.f32, ov.Shape(add_4_kernel_shape), weights[weights_offset : weights_offset + add_4_kernel_length], ) weights_offset += add_4_kernel_length @@ -119,7 +118,7 @@ def shape_and_length(shape: list) -> typing.Tuple[list, int]: softmax_axis = 1 softmax_node = opset8.softmax(add_4_node, softmax_axis) - return Model(softmax_node, [param_node], 'lenet') + return ov.Model(softmax_node, [param_node], 'lenet') def main(): @@ -135,35 +134,35 @@ def main(): number_top = 1 # ---------------------------Step 1. Initialize OpenVINO Runtime Core-------------------------------------------------- log.info('Creating OpenVINO Runtime Core') - core = Core() # ---------------------------Step 2. Read a model in OpenVINO Intermediate Representation------------------------------ log.info(f'Loading the model using ngraph function with weights from {model_path}') model = create_ngraph_function(model_path) # ---------------------------Step 3. Apply preprocessing---------------------------------------------------------- # Get names of input and output blobs - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() # Set a batch size equal to number of input images - set_batch(model, digits.shape[0]) + ov.set_batch(model, digits.shape[0]) # ---------------------------Step 4. Loading model to the device------------------------------------------------------- log.info('Loading the model to the plugin') + core = ov.Core() compiled_model = core.compile_model(model, device_name) # ---------------------------Step 5. Prepare input--------------------------------------------------------------------- diff --git a/samples/python/speech_sample/speech_sample.py b/samples/python/speech_sample/speech_sample.py index 8c2352defaa98a..e9ac1a821dc51a 100755 --- a/samples/python/speech_sample/speech_sample.py +++ b/samples/python/speech_sample/speech_sample.py @@ -9,8 +9,7 @@ from typing import Dict import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import Core, InferRequest, Layout, Type, set_batch +import openvino as ov from arg_parser import parse_args from file_options import read_utterance_file, write_utterance_file @@ -20,7 +19,7 @@ set_scale_factors) -def do_inference(data: Dict[str, np.ndarray], infer_request: InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray: +def do_inference(data: Dict[str, np.ndarray], infer_request: ov.InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray: """Do a synchronous matrix inference.""" frames_to_infer = {} result = {} @@ -69,7 +68,7 @@ def main(): # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- if args.model: @@ -83,19 +82,19 @@ def main(): if args.layout: layouts = get_input_layouts(args.layout, model.inputs) - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) for i in range(len(model.inputs)): - ppp.input(i).tensor().set_element_type(Type.f32) + ppp.input(i).tensor().set_element_type(ov.Type.f32) input_name = model.input(i).get_any_name() if args.layout and input_name in layouts.keys(): - ppp.input(i).tensor().set_layout(Layout(layouts[input_name])) - ppp.input(i).model().set_layout(Layout(layouts[input_name])) + ppp.input(i).tensor().set_layout(ov.Layout(layouts[input_name])) + ppp.input(i).model().set_layout(ov.Layout(layouts[input_name])) for i in range(len(model.outputs)): - ppp.output(i).tensor().set_element_type(Type.f32) + ppp.output(i).tensor().set_element_type(ov.Type.f32) model = ppp.build() @@ -103,7 +102,7 @@ def main(): batch_size = args.batch_size if args.context_window_left == args.context_window_right == 0 else 1 if any((not _input.node.layout.empty for _input in model.inputs)): - set_batch(model, batch_size) + ov.set_batch(model, batch_size) else: log.warning('Layout is not set for any input, so custom batch size is not set') diff --git a/src/bindings/python/docs/test_examples.md b/src/bindings/python/docs/test_examples.md index 5855f110fa263a..80808fdbfd5a6d 100644 --- a/src/bindings/python/docs/test_examples.md +++ b/src/bindings/python/docs/test_examples.md @@ -107,7 +107,7 @@ Let's add a test case for new class. Start with imports and simple test of the c ```python import pytest import numpy as np -import openvino.runtime as ov +import openvino as ov def test_mytensor_creation(): tensor = ov.MyTensor([1, 2, 3])